From 84ef8e4d724d2086fea6172994d3d29a9c8c2666 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Thu, 22 Oct 2026 23:16:35 -0400 Subject: [PATCH 001/392] drm/asahi: m3: restore the 7.1.12 T8122 runtime sources, unwired Bring back, unchanged, the M3 runtime that starts the T8122 (M3 MacBook Air) GPU on the release-20 kernel (c46d7dcdb871, the source of linux-aurora-m3gpu 7.1.12.aurora2-20261017.20): the runtime, its firmware admission, InitData, device, RTKit, memory, submission and client files, the T8122 start experiment, and the 7.1.12 hw tables (hw.rs, hw/t8122.rs, hw/agx3.rs, identity.rs). g15_raw.rs and initdata.rs are the G15 (G15, V14_8_3) expansions of 7.1.12's fw/initdata.rs and initdata.rs: every #[versions(AGX)] item replaced by its G15V14_8_3 instance, which 7.2's version matrix no longer generates. The expansion keeps the source text; its eleven G15 structures match rustc's own expansion of c46d7dcdb871 field for field. Nothing declares these modules yet, so this changes no build. The next commits adapt them to the 7.2 driver. Co-authored-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-authored-by: Eryk Wieliczko Co-authored-by: Satya Benson Co-authored-by: Naeem Malik Co-authored-by: garymeyerca Co-authored-by: DjDeveloperr Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/agx_attachments.rs | 71 + drivers/gpu/drm/asahi/m3/agx_compute.rs | 9 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 + drivers/gpu/drm/asahi/m3/agx_memory.rs | 12 + drivers/gpu/drm/asahi/m3/agx_resources.rs | 170 ++ drivers/gpu/drm/asahi/m3/agx_status.rs | 34 + drivers/gpu/drm/asahi/m3/agx_uapi.rs | 721 ++++++++ drivers/gpu/drm/asahi/m3/g15_raw.rs | 1800 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/hw.rs | 679 +++++++ drivers/gpu/drm/asahi/m3/hw/agx3.rs | 59 + drivers/gpu/drm/asahi/m3/hw/t8122.rs | 147 ++ drivers/gpu/drm/asahi/m3/identity.rs | 67 + drivers/gpu/drm/asahi/m3/initdata.rs | 1457 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 1491 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_board.rs | 559 ++++++ drivers/gpu/drm/asahi/m3/m3_client.rs | 228 +++ drivers/gpu/drm/asahi/m3/m3_completion.rs | 66 + drivers/gpu/drm/asahi/m3/m3_compute.rs | 379 ++++ drivers/gpu/drm/asahi/m3/m3_config.rs | 444 +++++ drivers/gpu/drm/asahi/m3/m3_device.rs | 315 ++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 210 +++ drivers/gpu/drm/asahi/m3/m3_firmware.rs | 441 +++++ drivers/gpu/drm/asahi/m3/m3_memory.rs | 163 ++ drivers/gpu/drm/asahi/m3/m3_params.rs | 900 ++++++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 51 + drivers/gpu/drm/asahi/m3/m3_pool.rs | 41 + drivers/gpu/drm/asahi/m3/m3_render.rs | 696 +++++++ drivers/gpu/drm/asahi/m3/m3_resources.rs | 51 + drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 327 ++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 1246 +++++++++++++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 297 +++ drivers/gpu/drm/asahi/m3/m3_thermal.rs | 499 +++++ drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 333 ++++ drivers/gpu/drm/asahi/m3/t8122_start.rs | 579 ++++++ 34 files changed, 14629 insertions(+) create mode 100644 drivers/gpu/drm/asahi/m3/agx_attachments.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_status.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_uapi.rs create mode 100644 drivers/gpu/drm/asahi/m3/g15_raw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/agx3.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/t8122.rs create mode 100644 drivers/gpu/drm/asahi/m3/identity.rs create mode 100644 drivers/gpu/drm/asahi/m3/initdata.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_adt_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_board.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_client.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_device.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_drm.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_firmware.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_params.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pass.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pool.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_render.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_rtkit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_runtime.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_submit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_thermal.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_knobs.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_start.rs diff --git a/drivers/gpu/drm/asahi/m3/agx_attachments.rs b/drivers/gpu/drm/asahi/m3/agx_attachments.rs new file mode 100644 index 0000000000000..5aeb3c7335c22 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_attachments.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +pub(crate) const SIZE: usize = 0x104; +pub(crate) const CAPACITY: usize = 16; + +#[derive(Clone, Copy)] +pub(crate) struct Attachments(pub(crate) [u8; SIZE]); + +impl Attachments { + pub(crate) const EMPTY: Self = Self([0; SIZE]); + pub(crate) fn count(&self) -> u8 { self.0[0x100] } + + /// Normalize byte ranges to the firmware's 128-byte cache-line units. + /// The caller must validate the entire normalized writable GPU range. + pub(crate) fn range(address: u64, size: u64) -> Result<(u64, u64), ()> { + if address == 0 || size == 0 { return Err(()); } + let end = address.checked_add(size).and_then(|v| v.checked_add(127)).ok_or(())? & !127; + let start = address & !127; + if start == 0 || end > 1 << 42 || (end - start) >> 7 > u32::MAX as u64 { + return Err(()); + } + Ok((start, end - start)) + } + + pub(crate) fn new(ranges: &[(u64, u64)]) -> Result { + if ranges.len() > CAPACITY { return Err(()); } + let mut out = Self::EMPTY; + for (record, &(address, size)) in out.0[..0x100].chunks_exact_mut(16).zip(ranges) { + let (address, size) = Self::range(address, size)?; + record[..8].copy_from_slice(&address.to_le_bytes()); + record[8..12].copy_from_slice(&((size >> 7) as u32).to_le_bytes()); + record[12..14].copy_from_slice(&0x18u16.to_le_bytes()); + record[14..16].copy_from_slice(&1u16.to_le_bytes()); + } + out.0[0x100..].copy_from_slice(&(ranges.len() as u32).to_le_bytes()); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn covers_both_partial_edge_lines() { + let a = Attachments::new(&[(0x100007f, 2)]).unwrap(); + assert_eq!(u64::from_le_bytes(a.0[..8].try_into().unwrap()), 0x1000000); + assert_eq!(u32::from_le_bytes(a.0[8..12].try_into().unwrap()), 2); + assert_eq!(&a.0[12..16], &[0x18, 0, 1, 0]); + assert_eq!(a.count(), 1); + assert!(a.0[16..0x100].iter().all(|v| *v == 0)); + } + #[test] + fn rejects_unrepresentable_ranges_and_count() { + for range in [(0, 128), (128, 0), (u64::MAX - 1, 8), + ((1 << 42) - 128, 129), (128, (u32::MAX as u64 + 1) << 7)] { + assert!(Attachments::new(&[range]).is_err()); + } + assert!(Attachments::new(&[(128, 128); CAPACITY + 1]).is_err()); + let full = Attachments::new(&[(128, 128); CAPACITY]).unwrap(); + assert_eq!(full.count(), 16); + assert_eq!(&full.0[0x100..], &[16, 0, 0, 0]); + assert_eq!(Attachments::new(&[]).unwrap().0, Attachments::EMPTY.0); + } + #[test] + fn matches_native_m4_writable_buffer_descriptor() { + let encoded = Attachments::new(&[(0x100000a8000, 0x400000)]).unwrap(); + assert_eq!(&encoded.0[..16], &[ + 0, 0x80, 0x0a, 0, 0, 1, 0, 0, 0, 0x80, 0, 0, 0x18, 0, 1, 0]); + assert_eq!(encoded.count(), 1); + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_compute.rs b/drivers/gpu/drm/asahi/m3/agx_compute.rs new file mode 100644 index 0000000000000..d58005c504f7a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_compute.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/// Complete client CDM range and its USC address window. +#[derive(Clone, Copy)] +pub(crate) struct Control { + pub base: u64, + pub end: u64, + pub usc_base: u64, +} diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs new file mode 100644 index 0000000000000..9c37cfa98dce9 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_fault.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Immutable fault-only snapshots of owned M3 firmware memory. +//! +//! Records contain name[32], firmware VA, length, begin/end monotonic ns, +//! then exactly `length` bytes. No MMIO or application BOs are read. Reads +//! are sequential observations, not atomic with running firmware. + +use kernel::{prelude::*, time::{ClockSource, Monotonic}}; + +pub(crate) struct Dump { + bytes: KVVec, + count: u32, + limit: usize, +} + +impl Dump { + + /// M3 snapshots have no scheduler job ID yet and contain only bounded + /// driver-owned firmware configuration and channel storage. + pub(crate) fn new_m3() -> Result { + Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) + } + + fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { + // Reserve before firmware/job publication, while healthy. Record appends + // must never enter reclaim waiting on the GPU that just faulted. + let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; + bytes.resize(40, 0, GFP_KERNEL)?; + bytes[..8].copy_from_slice(magic); + bytes[16..24].copy_from_slice(&job.to_le_bytes()); + bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + Ok(Self { bytes, count: 0, limit }) + } + + /// Start the one-shot capture; construction may precede the fault by hours. + pub(crate) fn begin(&mut self, job: u64) { + self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); + self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + } + + pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, + read: impl FnOnce(&mut [u8]) -> Result) -> Result { + if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } + let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; + let start = self.bytes.len(); + let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; + if end > self.limit { return Err(E2BIG); } + // Capacity was reserved at construction. NOWAIT is defense in depth; + // the checked limit above guarantees resize cannot allocate here. + self.bytes.resize(end, 0, GFP_NOWAIT)?; + self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); + self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); + self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); + let begin = Monotonic::ktime_get() as u64; + if let Err(error) = read(&mut self.bytes[start+64..end]) { + self.bytes.truncate(start); + return Err(error); + } + let finish = Monotonic::ktime_get() as u64; + self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); + self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); + self.count = count; + Ok(()) + } + + pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { + let length = self.bytes.len(); + self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); + self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); + self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); + let format = "M3FWD001"; + let owned = KBox::new(self, GFP_NOWAIT)?; + kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, + kernel::devcoredump::DEFAULT_TIMEOUT); + dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); + Ok(()) + } +} + +impl kernel::devcoredump::DevCoreDump for Dump { + fn read(&self, output: &mut [u8], offset: usize) -> Result { + if offset >= self.bytes.len() { return Ok(0); } + let length = output.len().min(self.bytes.len()-offset); + output[..length].copy_from_slice(&self.bytes[offset..offset+length]); + Ok(length) + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_memory.rs b/drivers/gpu/drm/asahi/m3/agx_memory.rs new file mode 100644 index 0000000000000..521990b4aad21 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_memory.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +// Firmware validates Timestamp user destinations against a dedicated 64 MiB +// interval published in HwDataB+0x28. Reserve the last 64 MiB of the +// firmware VM selector, disjoint from MMIO/command allocations above. +pub(crate) const TIMESTAMP_RANGE: core::ops::Range = + 0xffff_fc2f_fc00_0000..0xffff_fc30_0000_0000; + +pub(crate) fn publish() { + // SAFETY: Orders preceding WC buffer stores before a device doorbell. + unsafe { core::arch::asm!("dsb oshst", options(nostack, preserves_flags)) }; +} diff --git a/drivers/gpu/drm/asahi/m3/agx_resources.rs b/drivers/gpu/drm/asahi/m3/agx_resources.rs new file mode 100644 index 0000000000000..8fdf87bdb046d --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_resources.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J713 bootloader resource handoff. Reading this metadata does not map memory, +//! claim UAT ownership, authenticate firmware, or grant permission to start it. +//! Firmware ABI admission and an owned runtime are separate steps. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Region { + pub(crate) base: u64, + pub(crate) size: u64, +} + +impl Region { + fn end(self) -> Option { + self.base.checked_add(self.size) + } + + fn valid(self) -> bool { + self.base != 0 + && self.size != 0 + && (self.base | self.size) & 0x3fff == 0 + && self.end().is_some_and(|end| end <= 1 << 42) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResourceError { + Version, + Region, + Overlap, + FirmwareLayout, +} + +#[derive(Debug)] +pub(crate) struct Resources { + /// ttbs, pagetables, handoff, shared-l2, fw-text, fw-data. + pub(crate) regions: [Region; 6], + pub(crate) firmware_vas: [u64; 2], +} + +impl Resources { + pub(crate) fn validate( + version: u32, + regions: [Region; 6], + firmware_vas: [u64; 2], + firmware_flags: [u32; 2], + ) -> Result { + if version != 1 { + return Err(ResourceError::Version); + } + for (i, region) in regions.iter().enumerate() { + if !region.valid() { + return Err(ResourceError::Region); + } + for other in ®ions[..i] { + if region.base < other.end().unwrap() && other.base < region.end().unwrap() { + return Err(ResourceError::Overlap); + } + } + } + // The inherited 42-bit page-table roots and all 64 16-byte TTBAT + // entries fit their reserved regions. No panel-sized allocation pool. + if firmware_flags != [1, 0] + || firmware_vas[0] != 0xffff_fc00_0000_0000 + || firmware_vas[0].checked_add(regions[4].size) != Some(firmware_vas[1]) + || firmware_vas[1].checked_add(regions[5].size).is_none() + { + return Err(ResourceError::FirmwareLayout); + } + Ok(Self { + regions, + firmware_vas, + }) + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + + fn observed() -> [Region; 6] { + [ + (0x103fffb8000, 0x4000), + (0x103fff78000, 0x40000), + (0x103fff70000, 0x4000), + (0x103fff74000, 0x4000), + (0x10000cc0000, 0x60000), + (0x10001df4000, 0x13c000), + ] + .map(|(base, size)| Region { base, size }) + } + const VAS: [u64; 2] = [0xfffffc0000000000, 0xfffffc0000060000]; + + #[test] + fn observed_and_relocated_boot_addresses() { + let res = Resources::validate(1, observed(), VAS, [1, 0]).unwrap(); + assert_eq!(res.firmware_vas, VAS); + assert_eq!(res.regions[2].base, 0x103fff70000); + let relocated = observed().map(|r| Region { + base: r.base + 0x400000000, + ..r + }); + assert!(Resources::validate(1, relocated, VAS, [1, 0]).is_ok()); + } + + /// Relocating the UAT reservations preserves their sizes and ownership checks. + #[test] + fn relocated_uat_regions_keep_firmware_segments() { + let mut regions = observed(); + for region in &mut regions[..4] { + region.base -= 0x40000; + } + let res = Resources::validate(1, regions, VAS, [1, 0]).unwrap(); + assert_eq!(res.regions[0].base, 0x103fff78000); + assert_eq!(res.regions[4], observed()[4]); + assert_eq!(res.regions[5], observed()[5]); + } + + #[test] + fn rejects_missing_overlapping_unaligned_and_out_of_range_memory() { + for r in [ + Region { base: 0, size: 0 }, + Region { + base: 0x12345, + size: 0x4000, + }, + Region { + base: 1 << 42, + size: 0x4000, + }, + Region { + base: u64::MAX & !0x3fff, + size: 0x4000, + }, + ] { + let mut bad = observed(); + bad[0] = r; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Region) + )); + } + let mut bad = observed(); + bad[0] = bad[1]; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Overlap) + )); + } + + #[test] + fn rejects_unknown_handoff_and_segment_layouts() { + assert!(matches!( + Resources::validate(0, observed(), VAS, [1, 0]), + Err(ResourceError::Version) + )); + for (vas, flags) in [ + (VAS, [0, 1]), + ([VAS[0], VAS[1] + 0x4000], [1, 0]), + ([0xfffffe0000000000, VAS[1]], [1, 0]), + ] { + assert!(matches!( + Resources::validate(1, observed(), vas, flags), + Err(ResourceError::FirmwareLayout) + )); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_status.rs b/drivers/gpu/drm/asahi/m3/agx_status.rs new file mode 100644 index 0000000000000..1d83086a1d155 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_status.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! CPU-owned sticky VM failure reporting. No GPU mapping or firmware access. +use core::sync::atomic::{AtomicI32, Ordering}; + +pub(crate) const PARAM_GROUP_VM_STATUS: u32 = 1; +pub(crate) const FEATURE_VM_STATUS: u64 = 1 << 1; +pub(crate) const VM_STATUS_SIZE: usize = 16; + +/// Shared by the VM and accepted jobs; never cleared by queue destruction or +/// successful work. A newly created VM receives a new status object. +pub(crate) struct VmStatus { error: AtomicI32 } +impl VmStatus { + pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0) } } + pub(crate) fn record(&self, error: i32) { + if error < 0 { + let _ = self.error.compare_exchange(0, error, Ordering::AcqRel, Ordering::Acquire); + } + } + pub(crate) fn get(&self) -> i32 { self.error.load(Ordering::Acquire) } +} + +/// GET_PARAMS group 1 is an in/out request: vm_id, flags=0, error(out), pad=0. +/// Decode once, before lookup, so a racing userspace edit cannot change owner. +pub(crate) fn decode_request(bytes: &[u8; VM_STATUS_SIZE]) -> Option { + let word = |i| u32::from_ne_bytes(bytes[i..i+4].try_into().unwrap()); + if word(4) != 0 || word(12) != 0 { return None; } + Some(word(0)) +} +pub(crate) fn encode_response(vm_id: u32, error: i32) -> [u8; VM_STATUS_SIZE] { + let mut bytes = [0; VM_STATUS_SIZE]; + bytes[0..4].copy_from_slice(&vm_id.to_ne_bytes()); + bytes[8..12].copy_from_slice(&error.to_ne_bytes()); + bytes +} diff --git a/drivers/gpu/drm/asahi/m3/agx_uapi.rs b/drivers/gpu/drm/asahi/m3/agx_uapi.rs new file mode 100644 index 0000000000000..60107f8833edc --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_uapi.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Shared modern-Asahi UAPI parsing. +//! +//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`, +//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates +//! the queue's 4 GiB USC window and every userspace range before constructing +//! M3 render parameters or compute-entry operands. This module allocates no +//! object, publishes no queue state, and performs no firmware or MMIO action. + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; +pub(crate) const UAPI_RENDER_SIZE: usize = 256; +pub(crate) const UAPI_COMPUTE_SIZE: usize = 64; +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; +const COMMAND_RENDER: u16 = 0; +const COMMAND_COMPUTE: u16 = 1; +const SET_VERTEX_ATTACHMENTS: u16 = 2; +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT; + +const SAMPLER_SIZE: u64 = 8; +const MAX_SAMPLERS: u32 = 1024; +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + StencilClearBits, +} + +impl UapiTranslateError { + /// Stable short name for one rejection, so a failing submit reports which + /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no + /// `Debug` formatting) to stay cheap on the ioctl path. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::HeaderTruncated => "header-truncated", + Self::PayloadTruncated => "payload-truncated", + Self::PayloadTail => "payload-tail", + Self::CommandType => "command-type", + Self::AttachmentPayloadSize => "attachment-payload-size", + Self::AttachmentCount => "attachment-count", + Self::AttachmentFields => "attachment-fields", + Self::AttachmentBarrier => "attachment-barrier", + Self::HardwareCommandCount => "hardware-command-count", + Self::BarrierFuture => "barrier-future", + Self::QueueUscAlignment => "queue-usc-alignment", + Self::QueueUscRange => "queue-usc-range", + Self::AddressRange => "address-range", + Self::AddressAlignment => "address-alignment", + Self::FlagsUnknownBits => "flags-unknown-bits", + Self::FlagsUnsupported => "flags-unsupported", + Self::Dimensions => "dimensions", + Self::Layers => "layers", + Self::UtileDimensions => "utile-dimensions", + Self::Samples => "samples", + Self::TilebufferSize => "tilebuffer-size", + Self::SamplerCount => "sampler-count", + Self::SamplerPair => "sampler-pair", + Self::HelperProgram => "helper-program", + Self::Add3ProofAttachments => "add3-proof-attachments", + Self::ProgramOffset => "program-offset", + Self::ZlsFields => "zls-fields", + Self::StencilClearBits => "stencil-clear-bits", + + + } + } +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + pub(crate) fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +pub(crate) fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +pub(crate) fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) timestamps: UapiTimestamps, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + Ok(TranslatedComputeCommand { + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + timestamps: command.timestamps, + }) +} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs new file mode 100644 index 0000000000000..a876cad1ec283 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -0,0 +1,1800 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + /// Host->FW ring descriptor of the 14.x firmware ABI. + /// + /// Instead of one `ChannelState` pointer, each host->FW ring (pipes, DeviceControl) is + /// described by the FW VAs of its three index words and of the ring itself. On G15 the + /// indices (and the pipe rings) live in Fender scratch SRAM. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct ScratchRingDesc { + /// FW VA of the u32 read index (FW writes; scratch part B). +0x00 + pub(crate) read_index: U64, + /// FW VA of the u32 "CFI" index (FW writes; scratch part B). +0x08 + pub(crate) cfi_index: U64, + /// FW VA of the u32 write index (host writes; scratch part A). +0x10 + pub(crate) write_index: U64, + /// FW VA of the ring entries. +0x18 + pub(crate) ring: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannelsG15V14_8_3 { + + // 14.x ABI: {TA, 3D, CL} descriptors of 0x20 bytes, 0x60 per priority + pub(crate) vtx: ScratchRingDesc, + pub(crate) frag: ScratchRingDesc, + pub(crate) comp: ScratchRingDesc, + } + default_zeroed!(PipeChannelsG15V14_8_3); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + /// P0+0x45c4. Non-zero makes the firmware install its work-scheduler callbacks + /// Left 0, the first compute kick branches to address 0 (observed on J516S). + pub(crate) sched_callbacks: u32, + __pad4: Pad<0x8>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + /// G15 "P0" globals-2/status block, 0xc3d0 bytes, pointed to by InitData+0xb0. It embeds the + /// G13 `FwStatus` at +0x4568 (FWCtl ring pointers, 8 bytes + /// of padding, then the halt/resume flags at +0x4580), which is why the + /// FWCtl ring and flags keep their `FwStatus` field names here. + #[repr(C)] + pub(crate) struct StatusBlock { + /// 0 at init; also written by the firmware register dump. +0x0 + pub(crate) unk_0: u32, + /// Noise-suppression idle power-off state. +0x4 + pub(crate) noise_suppression_idle_pwroff: u32, + /// 0 at init. +0x8 + pub(crate) unk_8: u32, + /// Not touched by the host (0x0008..0x4017). + /// TODO: firmware-private, or tables (pending stamps?) the host must fill? + pub(crate) unk_c: Array<0x400c, u8>, + /// Perf-counter sampler control, 0x4018..0x40a0. Unused by drm/asahi. + pub(crate) perfctr: Array<0x88, u8>, + /// FW-initiated recovery info (FW-written). +0x40a0/+0x40a4 + pub(crate) fw_recovery_info_a: u32, + pub(crate) fw_recovery_info_b: u32, + pub(crate) unk_40a8: Array<0x3f8, u8>, + /// Recovery-packet details read on event type 4. +0x44a0 + pub(crate) recovery_packet_info: Array<0x14, u8>, + pub(crate) unk_44b4: Array<0xc, u8>, + /// Read while draining events. +0x44c0 + pub(crate) unk_44c0: U64, + pub(crate) unk_44c8: Array<0x90, u8>, + /// Read while draining events. +0x4558/+0x4560 + pub(crate) unk_4558: U64, + pub(crate) unk_4560: U64, + /// FWCtl `{state, ring}` FW VAs. +0x4568/+0x4570 + pub(crate) fwctl_channel: ChannelRing, + /// 8 bytes of padding put the flags on a 16-byte boundary. + pub(crate) __pad_4578: Pad<0x8>, + /// halt_count/halted/resume/... at +0x4580/+0x4590/+0x45a0. + /// +0x45c4 (`flags.sched_callbacks`) must be non-zero or the work scheduler + /// is left without its callbacks. + pub(crate) flags: FwStatusFlags, + /// Not touched by the host (0x45c8..0xc3c7). + pub(crate) unk_45f0: Array<0x7dd8, u8>, + /// QoS mode, 0 at init. +0xc3c8 + pub(crate) qos_mode: u32, + /// TODO: read-modify-written at shared-data init; left 0. +0xc3cc + pub(crate) unk_c3cc: u32, + } + default_zeroed!(StatusBlock); + no_debug!(StatusBlock); + + /// G15 "P1" power-controller shared block, 0x238 bytes, InitData+0xb8. Its 0x0c..0xd4 part + /// keeps the order of the G13 13.5 `Globals` power + /// fields (power zones, fast-die and PPM controllers), so the same field names are used. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PowerCtlBlock { + /// FW-published power values. +0x0/+0x4 + pub(crate) fw_power_a: u32, + pub(crate) fw_power_b: u32, + /// Power-zone reset request. +0x8 + pub(crate) power_zone_reset: u32, + /// +0xc..+0x5f: G13 Globals `power_zone_count`..`power_zones`, same order. + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + /// CLVR loop override / control effort. +0x60..+0x68 + pub(crate) clvr_loop_override_a: u32, + pub(crate) clvr_ctrl_effort: u32, + pub(crate) clvr_loop_override_b: u32, + pub(crate) unk_6c: Array<0x38, u8>, + /// Lifetime-servo temperature target. +0xa4 + pub(crate) lifetime_servo_temp_a: u32, + /// +0xa8..+0xc8: G13 Globals `unk_89bc`..`unk_89e0`, same order. + pub(crate) fast_die_temp_target: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_b0: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: u32, + /// +0xc8..+0xdb: the fast-die/PPM enable flags, the PPM target (maximum power) and the + /// PPM controller gains, in G13 Globals order (`unk_89e0`, `max_power_2`, `ppm_kp`, + /// `ppm_ki_dt`). The two T6030 backends place them differently: the manager starts + /// with the enable flag at +0xc8 (the layout it was validated with), the runtime backend + /// with 0 at +0xc8 and the flag at +0xcc, as its InitData has them. See the builder. + pub(crate) ppm_words: Array<5, u32>, + pub(crate) unk_dc: Array<0xc, u8>, + /// "SE" controller override / engagement enable. +0xe8/+0xec + pub(crate) se_controller_override: u32, + pub(crate) se_engagement: u32, + pub(crate) unk_f0: Array<0x68, u8>, + /// DPE leakage update config. +0x158..+0x1ab + pub(crate) dpe_leak_cfg_a: u32, + pub(crate) unk_15c: u32, + pub(crate) unk_160: u32, + pub(crate) unk_164: u32, + pub(crate) dpe_leak_coefs: Array<0x40, u8>, + pub(crate) dpe_leak_cfg_b: u32, + pub(crate) unk_1ac: Array<0x8, u8>, + /// CLPC shared data (FW-written). +0x1b4..+0x1cc + pub(crate) clpc_shared_a: U64, + pub(crate) clpc_shared_b: U64, + pub(crate) clpc_split_ratio_a: u32, + pub(crate) clpc_split_ratio_b: u32, + pub(crate) clpc_shared_c: u32, + /// +0x1d0 (u8) / +0x1d1 (unaligned u32) + pub(crate) perf_ctrl_override_active: u8, + pub(crate) consistent_perf_state: U32, + pub(crate) unk_1d5: Array<0x3, u8>, + /// FW-written energy / filtered power. +0x1d8/+0x1e0/+0x1e4 + pub(crate) accumulated_energy: U64, + pub(crate) filtered_power: u32, + pub(crate) unk_1e4: u32, + /// SLC DSID configuration. +0x1e8..+0x22b + pub(crate) dsid_cfg_a: U64, + pub(crate) unk_1f0: Array<0x28, u8>, + pub(crate) dsid_cfg_b: U64, + pub(crate) dsid_cfg_c: U64, + pub(crate) dsid_cfg_d: u32, + pub(crate) unk_22c: u32, + /// +0x230 + pub(crate) submit_in_progress_gate: u32, + pub(crate) unk_234: u32, + } + default_zeroed!(PowerCtlBlock); + + /// G15 "P2" debug/trace block, 0x20 bytes, InitData+0xa8. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct DebugBlock { + /// Moved here from G13 `Globals.ktrace_enable`. +0x0 + pub(crate) ktrace_enable: u32, + /// TODO: written at init with an unknown value; left 0. +0x4 + pub(crate) unk_4: u32, + /// System-sleep notification in progress. +0x8 + pub(crate) system_sleep_in_progress: u32, + pub(crate) unk_c: u32, + /// Firmware-written. Read-only for the host. + pub(crate) fw_word_10: AtomicU32, + /// Firmware init progress: the firmware writes 0 once it consumes the INIT handshake and 1 + /// once its post-INIT initialisation is done, each followed by a barrier. The G15 driver + /// writes a sentinel here before MSG_INIT so the three states (not consumed / stuck / done) + /// are distinguishable. + pub(crate) init_state: AtomicU32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + } + default_zeroed!(DebugBlock); + + /// One Globals register-override record, 0x18 bytes. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct RegisterOverride { + pub(crate) addr: U64, + pub(crate) value: U64, + pub(crate) id: u32, + pub(crate) __pad: u32, + } + + /// One InitSeq register-initialisation record, 0x18 bytes. + /// + /// The InitSeq buffer (InitData+0x08) is a list of these. A zeroed buffer is a valid empty + /// list, which is what drm/asahi passes; this type only documents the format. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct InitSeqRecord { + pub(crate) value: U64, + pub(crate) reg_id: u32, + pub(crate) extra: u32, + /// 1 = 32-bit, 2 = 64-bit, 3 = 64-bit physical address. + pub(crate) kind: u32, + pub(crate) __pad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + /// G15 (14.x) shader-engine controller block of HwDataA, at +0x10b4. + /// + /// It holds the fields of [`HwDataA130Extra`] in a different order: the filter, gain and + /// time-constant words move 0x14 bytes down, and the threshold/criteria header moves behind + /// them, to +0xd8. Offsets in the field names are G15 block offsets. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA140Extra { + pub(crate) unk_0: Array<0x40, u8>, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_64: F32, + pub(crate) unk_68: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) unk_84: u32, + pub(crate) unk_88: F32, + /// Shader-engine target (ADT gpu-se-tgt). + pub(crate) se_target: u32, + pub(crate) unk_90: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: F32, + pub(crate) unk_b4: Array<0x24, u8>, + pub(crate) unk_d8: u32, + pub(crate) unk_dc: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) unk_e8: u32, + pub(crate) unk_ec: u32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_f4: Array<0x28, u8>, + pub(crate) unk_11c: F32, + pub(crate) unk_120: Array<0xc, u8>, + pub(crate) unk_12c: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_134: Array<0x84, u8>, + } + default_zeroed!(HwDataA140Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZoneG15V14_8_3 { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + // G15 HwDataA uses the G13 single-die layout with 8x8 coefficient tables at 0x970-0xb6f + //, so the G14X 16-core layout is G14X-only. + const MAX_CORES_PER_CLUSTERG15V14_8_3: usize = { + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[repr(C)] + pub(crate) struct HwDataAG15V14_8_3 { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + pub(crate) ppm_filter_tc_clks: u32, + + pub(crate) unk_730_4: u32, + + pub(crate) unk_730_8: u32, + + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + pub(crate) pad_bb8_0: Pad<0x200>, + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + pub(crate) avg_power_target_filter_tc_clks: u32, + + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZoneG15V14_8_3>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + pub(crate) avg_power_filter_tc_clks: u32, + + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + pub(crate) unk_e10_pad: Array<0x10, u8>, + pub(crate) unk_e10_0: HwDataA140Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + // G15 offsets match G13 13.5 up to ~0x1224 and are +0x10 from 0x1288 + // (`fast_die0_sensor_mask_2`, G13 0x1278). TODO: exact insertion point between + // 0x1224 and 0x1288 is not known; placed as late as possible. + pub(crate) unk_1278_g15: Array<0x10, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + pub(crate) aux_leak_coef: AuxLeakCoef, + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + + // G15 tail, 0x422c..0x4360. + pub(crate) unk_422c: Array<0x20, u8>, + + /// System counter value at init. +0x424c + /// The runtime backend fills it; the manager leaves it 0, as validated. + pub(crate) init_timestamp: U64, + pub(crate) unk_4254: Array<0x40, u8>, + + /// +0x4294/+0x4298 + pub(crate) unk_4294: u32, + pub(crate) unk_4298: u32, + pub(crate) unk_429c: Array<0x8, u8>, + + /// Three 8-entry per-cluster tables. +0x42a4 + /// TODO: source values unknown; left 0. + pub(crate) cluster_tables: Array<3, Array<8, u32>>, + + /// Eight words, source unknown. +0x4304..+0x4320 + pub(crate) unk_4304: Array<8, u32>, + pub(crate) unk_4324: Array<0x1c, u8>, + + /// Read-modify-written at init. +0x4340 + pub(crate) unk_4340: U64, + pub(crate) unk_4348: Array<0x10, u8>, + + /// GPU keep-alive mode (read by the host). +0x4358 + pub(crate) gpu_keepalive_mode: u32, + pub(crate) unk_435c: u32, + } + default_zeroed!(HwDataAG15V14_8_3); + no_debug!(HwDataAG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + const IO_MAPPING_COUNTG15V14_8_3: usize = { + // 31 IOMapping slots (0x3e0 / 0x20) + { + 0x1f + } + }; + + /// A G15 performance-state table: the highest entry, then per entry the frequency (MHz) + /// and the core and SRAM voltages (mV), eight voltage columns each. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBPStateTable { + pub(crate) max_pstate: u32, + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBG15V14_8_3 { + + pub(crate) unk_8: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + + /// G15: a pointer. +0x30 + /// TODO: what it points to; left 0. + pub(crate) unk_30_ptr: U64, + + // G15: two tables of 32 CSC matrices (3x4 i16) at 0x38..0x637 + pub(crate) yuv_matrices: Array<0x40, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + + // G15 0xa70..0xb3f. The G13 13.5 0x9b0..0xa73 group, + // with 12 bytes inserted at an unknown position; the words are named by their G15 offset. + // TODO: map these to the G13 names (`num_frags`, `unk_524` secure cache flush, ...) + // once their values are known. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: u32, + pub(crate) unk_a78: u32, + pub(crate) unk_a7c: U64, + pub(crate) unk_a84: u32, + pub(crate) unk_a88: u32, + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: u32, + pub(crate) unk_a94: u32, + pub(crate) unk_a98: u32, + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: u32, + pub(crate) unk_aa4: u32, + pub(crate) unk_aa8: u32, + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: u32, + pub(crate) unk_ab4: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: U64, + pub(crate) unk_ac4: u32, + pub(crate) unk_ac8: Array<0x4, u32>, + pub(crate) unk_ad8: u32, + pub(crate) unk_adc: u32, + pub(crate) unk_ae0: u32, + pub(crate) unk_ae4: u32, + pub(crate) unk_ae8: U64, + pub(crate) unk_af0: u32, + pub(crate) unk_af4: U64, + pub(crate) unk_afc: u32, + pub(crate) unk_b00: u32, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: Array<0x4, u32>, + pub(crate) unk_b18: u32, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: U64, + pub(crate) unk_b2c: Array<0x14, u8>, + + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + + /// G15: second 16-entry frequency table in MHz (meaning inferred). +0xf9c + pub(crate) frequencies_2: Array<0x10, u32>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + pub(crate) unk_arr_0: Array<32, u32>, + + pub(crate) aux_ps: HwDataBAuxPStates, + + // G15 0x134c..0x17eb: a third p-state table at 0x134c, then the + // words before `timer_offset`. G15 is +4 relative to G13 here (+0xd0 at `timer_offset`). + + /// Third p-state table: the secondary (lowest frequency per voltage) table, with one + /// voltage column. +0x134c + pub(crate) aux_ps_3: HwDataBPStateTable, + pub(crate) aux_ps_3_pad: Array<0xc, u8>, + pub(crate) unk_179c: U64, + pub(crate) unk_17a4: Array<0x10, u8>, + pub(crate) unk_17b4: u32, + pub(crate) unk_17b8: u32, + pub(crate) unk_17bc: u32, + + /// Unit enable bit masks. +0x17c0/+0x17c8 + pub(crate) unit_mask_a: U64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_17cc: u32, + pub(crate) unk_17d0: u32, + pub(crate) unk_17d4: Array<0x10, u8>, + pub(crate) unk_17e4: u32, + pub(crate) unk_17e8: u32, + + pub(crate) timer_offset: U64, + + // G15 0x17f4..0x1867. + pub(crate) unk_17f4: U64, + pub(crate) unk_17fc: u32, + pub(crate) unk_1800: u32, + pub(crate) unk_1804: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_180c: u32, + + /// +0x1810 and +0x1814 are both 1 (written as one 64-bit store); the firmware copies them + /// to separate globals. + pub(crate) unk_1810: u32, + pub(crate) unk_1814: u32, + pub(crate) unk_1818: u32, + pub(crate) unk_181c: u32, + + /// 48 bytes of 0xff ("all units enabled" masks?). +0x1820 + pub(crate) all_ones_masks: Array<0x30, u8>, + + /// FSTP override enable. +0x1850 + pub(crate) fstp_override: u32, + pub(crate) unk_1854: U64, + pub(crate) unk_185c: u32, + pub(crate) unk_1860: u32, + pub(crate) unk_1864: u32, + } + default_zeroed!(HwDataBG15V14_8_3); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFragG15V14_8_3 { + // G15: the two -1 fields are at E2+0xc18/+0xc30 + pub(crate) unk1_0: Array<0xb10, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + // G15 E2 is 0x1248 bytes in total + pub(crate) unk3: Array<0x614, u8>, + } + + impl Default for GpuStatsFragG15V14_8_3 { + fn default() -> Self { + Self { + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFragG15V14_8_3 { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFragG15V14_8_3, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + // Not part of the 0x490-byte 14.x RuntimePointers, so the G15V14_8_3 + // instantiation is unused. + #[derive(Debug)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct RuntimeScratchG15V14_8_3 { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + default_zeroed!(RuntimeScratchG15V14_8_3); + + #[repr(C)] + pub(crate) struct RuntimePointersG15V14_8_3<'a> { + // G15 / 14.8.3: 0x490 bytes, packed (HwDataA pointer at +0x441). GpuPointer/U64 are + // packed(1), so repr(C) gives the firmware offsets. + + /// +0x000 + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataBG15V14_8_3>, + + /// Hardware-erratum (BRN) table. +0x008 + pub(crate) brn_table: GpuPointer<'a, &'a [u8]>, + + /// E7 (0x88 bytes, zeroed). +0x010 + pub(crate) unkptr_e7: GpuPointer<'a, &'a [u8]>, + + pub(crate) pipes: Array<4, PipeChannelsG15V14_8_3>, + + /// +0x198 + pub(crate) device_control: ScratchRingDesc, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + + /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 + pub(crate) fwlog_buf: Option>, + pub(crate) __pad_200: Pad<0x30>, + + /// 0 at init. +0x230 + pub(crate) unk_230: u32, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFragG15V14_8_3>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + + + /// E5 (0x60 bytes, zeroed; FW status word at +0). +0x24c + pub(crate) unkptr_e5: GpuPointer<'a, &'a [u8]>, + pub(crate) __pad_254: Pad<0x54>, + + /// Optional pointer, left 0. +0x2a8 + pub(crate) unkptr_2a8: U64, + + /// PB descriptor table GPU/FW VA. +0x2b0/+0x2b8 + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + + /// UMA page-pool descriptor table GPU/FW VA. +0x2c0/+0x2c8 + pub(crate) uma_pool_desc_gpu_addr: U64, + pub(crate) uma_pool_desc_fw_addr: U64, + + /// Configuration words, left 0. +0x2d0/+0x2d4 + pub(crate) unk_2d0: u32, + pub(crate) unk_2d4: u32, + pub(crate) __pad_2d8: Pad<0xd8>, + + /// 0xff at init (context-switch histogram control). +0x3b0 + pub(crate) cswitch_hist_ctl: u8, + pub(crate) unk_3b1: Array<0x90, u8>, + + // G15: +0x441 (unaligned) + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataAG15V14_8_3>, + + + /// 16 configuration bytes, left 0. +0x449 + pub(crate) unk_449: Array<0x10, u8>, + pub(crate) __pad_459: Pad<0x14>, + + /// Zero at init. +0x46d + pub(crate) unk_46d: Array<0x20, u8>, + pub(crate) __pad_48d: Pad<0x3>, + } + no_debug!(RuntimePointersG15V14_8_3<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GlobalsG15V14_8_3 { + // G15 / 14.8.3 layout: 0xe00 bytes, cached, packed. + // It re-packs the G13 field groups; the large G13 arrays are gone. + + /// Timer context-switch mode, 0 or 7. +0x0 + pub(crate) timer_cswitch: u32, + pub(crate) unk_4: u32, + + /// +0x8 + pub(crate) dm_pause_mode: u32, + pub(crate) unk_c: u32, + + /// +0x10 + pub(crate) dm_pause_timer: u32, + pub(crate) unk_14: u32, + + /// +0x18 + pub(crate) disable_cswitch: u32, + + /// Configuration word. +0x1c + pub(crate) unk_1c: u32, + + /// +0x20/+0x24 + pub(crate) relaxed_cl_kill_timeout: u32, + pub(crate) frg_task_timeout: u32, + + /// G13 `unk_24`. +0x28 + pub(crate) unk_28: u32, + + /// Debug flags bit 0 (G13 `debug`). +0x2c + pub(crate) debug: u32, + + /// +0x30 + pub(crate) smart_idle_off_enable: u32, + pub(crate) unk_34: u32, + + /// Constant 0x78. +0x38 + pub(crate) unk_38: u32, + + /// +0x3c/+0x40 + pub(crate) cpms_window_size: u32, + pub(crate) cpms_tfca_size: u32, + pub(crate) unk_44: u32, + + /// Kick channel QoS pair. +0x48/+0x4c + pub(crate) kick_channel_qos_b: u32, + pub(crate) kick_channel_qos_a: u32, + + /// G13 `unk_54`/`unk_56`/`unk_58` moved -0x14. +0x50..+0x54 (inferred) + pub(crate) unk_50: u16, + pub(crate) unk_52: u16, + pub(crate) unk_54: u16, + + /// G13 `unk_5a`/`unk_5e`/`unk_62` moved -0x14 (unaligned). +0x56..+0x5e + pub(crate) unk_56: U32, + pub(crate) unk_5a: U32, + pub(crate) cswitch_timer_multiplier: U32, + + /// +0x62 (G13 `unk_66`, tentative) + pub(crate) cdm_cs_mode_change: U32, + pub(crate) unk_66: Array<0xa, u8>, + + /// Command submission enable; must be non-zero to submit work. +0x70 + pub(crate) command_submission_enabled: u32, + + /// Pending-submission counter (G13 `pending_submissions`). +0x74 + pub(crate) pending_submissions: AtomicU32, + + /// +0x78..+0x80 + pub(crate) power_if_target: Array<3, u32>, + + /// +0x84..+0x8c + pub(crate) controller_override_0: u32, + pub(crate) controller_override_1: u32, + pub(crate) perf_state_cap: u32, + + /// Perf-controller block. +0x90..+0xc7 + pub(crate) perf_controller: Array<0x38, u8>, + + /// +0xc8/+0xcc + pub(crate) deadline_control_effort: u32, + pub(crate) deadline_controller_override: u32, + pub(crate) unk_d0: Array<0xa0, u8>, + + /// DPE (dynamic power estimation) block. +0x170..+0x69f + pub(crate) dpe: Array<0x530, u8>, + + /// +0x6a0 + pub(crate) dpe_sl_config: U64, + pub(crate) unk_6a8: Array<0x100, u8>, + + /// Smart idle-off config. +0x7a8 + pub(crate) smart_idle_off_cfg: Array<10, u32>, + + /// +0x7d0..+0x7dc + pub(crate) ut_engagement: u32, + pub(crate) clvr_engagement: u32, + pub(crate) keepalive_perf_threshold_rd: u32, + pub(crate) keepalive_off_threshold_rd: u32, + pub(crate) unk_7e0: u32, + + /// Register overrides. +0x7e4 + pub(crate) reg_overrides: Array<16, RegisterOverride>, + + /// +0x964 + pub(crate) reg_override_count: u32, + pub(crate) unk_968: u32, + + /// Display power-management params. +0x96c/+0x974 + pub(crate) dpm_param_a: U64, + pub(crate) dpm_param_b: U64, + + /// +0x97c..+0x984 + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + /// +0x988..+0x99c: the G13 `unk_1102c_0`..`unk_1102c` group (inferred) + pub(crate) unk_1102c_0: u32, + pub(crate) unk_1102c_4: u32, + pub(crate) unk_1102c_8: u32, + pub(crate) unk_1102c_c: u32, + pub(crate) unk_1102c_10: u32, + pub(crate) unk_1102c: u32, + + /// +0x9a0..+0x9a8 + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + + /// +0x9ac + pub(crate) gvdm_timer_interval: u32, + + /// +0x9b0/+0x9b4 + pub(crate) cl_context_switch_timeout_ms: u32, + pub(crate) cl_kill_timeout_ms: u32, + pub(crate) unk_9b8: u32, + + /// +0x9bc + pub(crate) cdm_backoff_timeout: u8, + pub(crate) unk_9bd: Array<0x4, u8>, + + /// +0x9c1/+0x9c3 + pub(crate) fwutil_default_fab_pstate: Array<0x2, u8>, + pub(crate) fwutil_timer_period: u8, + pub(crate) unk_9c4: Array<0x19, u8>, + + /// Unaligned. +0x9dd + pub(crate) mtr_sensor_ptd_override_mask: U64, + pub(crate) unk_9e5: Array<0x400, u8>, + + /// Read by the host when it creates command queues. +0xde5 + pub(crate) unk_de5: U32, + + /// Keep-alive overrides/thresholds. +0xde9..+0xdf5 + pub(crate) gpu_keepalive_override: U32, + pub(crate) gfxc_keepalive_override: U32, + pub(crate) keepalive_perf_threshold: U32, + pub(crate) keepalive_off_threshold: U32, + + /// Soft-fault settings (possibly G13 `fault_control`). +0xdf9 + pub(crate) soft_fault_settings: U32, + pub(crate) unk_dfd: Array<0x3, u8>, + + /// The G13 pending-stamp table is not in the 0xe00-byte G15 Globals. This + /// zero-length field keeps `GpuManager::mark_pending_events()` compiling as a no-op. + /// TODO: find the G15 pending-stamp table (possibly in P0 0x0008..0x4017). + pub(crate) pending_stamps: Array<0, PendingStamp>, + } + default_zeroed!(GlobalsG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitDataG15V14_8_3<'a> { + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointersG15V14_8_3>, + pub(crate) globals: GpuPointer<'a, super::GlobalsG15V14_8_3>, + + /// G15: no FwStatus pointer; u32 0 then u32 1 (possibly a host-mapped FW allocations + /// flag). +0x28/+0x2c + pub(crate) unk_28: u32, + pub(crate) unk_2c: u32, + // 0x30..0x97 on G15 is a 0x68-byte copy of the UAT pmap descriptor (+0x18..+0x7f). Its + // first 0x64 bytes have the G13 layout (3 levels, 0x20-byte records). + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + + + /// G15: last 4 bytes of the pmap descriptor copy (descriptor +0x7c). + /// TODO: meaning unknown; left 0. + pub(crate) unk_94: u32, + pub(crate) __pad_98: Pad<0x10>, + + /// P2 debug block (0x20). +0xa8 + pub(crate) debug_block: GpuPointer<'a, super::DebugBlock>, + + /// P0 status block (0xc3d0). +0xb0 + pub(crate) status_block: GpuPointer<'a, super::StatusBlock>, + + /// P1 power-controller block (0x238). +0xb8 + pub(crate) power_ctl_block: GpuPointer<'a, super::PowerCtlBlock>, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(StatusBlock); +trivial_gpustruct!(PowerCtlBlock); +trivial_gpustruct!(DebugBlock); +trivial_gpustruct!(GpuGlobalStatsVtx); +trivial_gpustruct!(GpuGlobalStatsFragG15V14_8_3); +trivial_gpustruct!(GpuStatsComp); + +trivial_gpustruct!(HwDataAG15V14_8_3); + +trivial_gpustruct!(HwDataBG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct StatsG15V14_8_3 { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[derive(Debug)] +pub(crate) struct RuntimePointersG15V14_8_3 { + pub(crate) stats: StatsG15V14_8_3, + + pub(crate) hwdata_a: GpuObject, + pub(crate) hwdata_b: GpuObject, + + // G15 group-A regions referenced from RuntimePointers + pub(crate) brn_table: GpuArray, + pub(crate) unkptr_e7: GpuArray, + pub(crate) unkptr_e5: GpuArray, + pub(crate) uma_pool_desc: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +impl GpuStruct for RuntimePointersG15V14_8_3 { + type Raw<'a> = raw::RuntimePointersG15V14_8_3<'a>; +} + +trivial_gpustruct!(GlobalsG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct InitDataG15V14_8_3 { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + + /// G15: the P0 status block, which embeds the FwStatus FWCtl ring and halt flags. It keeps + /// the `fw_status` name so the FwStatus users see the same fields. + pub(crate) fw_status: GpuObject, + pub(crate) power_ctl_block: GpuObject, + pub(crate) debug_block: GpuObject, +} + +impl GpuStruct for InitDataG15V14_8_3 { + type Raw<'a> = raw::InitDataG15V14_8_3<'a>; +} + +// G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. +mod g15_layout { + use super::channels; + use super::raw::*; + use core::mem::{offset_of, size_of}; + use kernel::static_assert; + + // InitData, 0xc0 bytes used + static_assert!(size_of::>() == 0xc0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_buf) == 0x08); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, runtime_pointers) == 0x18); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, globals) == 0x20); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_2c) == 0x2c); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, uat_page_size) == 0x30); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, debug_block) == 0xa8); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, status_block) == 0xb0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, power_ctl_block) == 0xb8); + + // host->FW ring descriptor + static_assert!(size_of::() == 0x20); + static_assert!(size_of::() == 0x60); + + // RuntimePointers: 0x490 bytes, packed + static_assert!(size_of::>() == 0x490); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_b) == 0x000); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, brn_table) == 0x008); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e7) == 0x010); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, pipes) == 0x018); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, device_control) == 0x198); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, event) == 0x1b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fw_log) == 0x1c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, ktrace) == 0x1d8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats) == 0x1e8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fwlog_buf) == 0x1f8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_230) == 0x230); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_vtx) == 0x234); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_frag) == 0x23c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_comp) == 0x244); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e5) == 0x24c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_2a8) == 0x2a8); + static_assert!( + offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_gpu_addr) == 0x2b0 + ); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_fw_addr) == 0x2b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_gpu_addr) == 0x2c0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_fw_addr) == 0x2c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_2d0) == 0x2d0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, cswitch_hist_ctl) == 0x3b0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_a) == 0x441); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_449) == 0x449); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_46d) == 0x46d); + + // Globals: 0xe00 bytes, packed + static_assert!(size_of::() == 0xe00); + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); + static_assert!(offset_of!(GlobalsG15V14_8_3, cswitch_timer_multiplier) == 0x5e); + static_assert!(offset_of!(GlobalsG15V14_8_3, command_submission_enabled) == 0x70); + static_assert!(offset_of!(GlobalsG15V14_8_3, pending_submissions) == 0x74); + static_assert!(offset_of!(GlobalsG15V14_8_3, perf_controller) == 0x90); + static_assert!(offset_of!(GlobalsG15V14_8_3, deadline_control_effort) == 0xc8); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe) == 0x170); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe_sl_config) == 0x6a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, smart_idle_off_cfg) == 0x7a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, ut_engagement) == 0x7d0); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_overrides) == 0x7e4); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_override_count) == 0x964); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpm_param_a) == 0x96c); + static_assert!(offset_of!(GlobalsG15V14_8_3, progress_check_interval_3d) == 0x97c); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_1102c_0) == 0x988); + static_assert!(offset_of!(GlobalsG15V14_8_3, idle_off_delay_ms) == 0x9a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, fender_idle_off_delay_ms) == 0x9a4); + static_assert!(offset_of!(GlobalsG15V14_8_3, fw_early_wake_timeout_ms) == 0x9a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, cl_context_switch_timeout_ms) == 0x9b0); + static_assert!(offset_of!(GlobalsG15V14_8_3, cdm_backoff_timeout) == 0x9bc); + static_assert!(offset_of!(GlobalsG15V14_8_3, fwutil_default_fab_pstate) == 0x9c1); + static_assert!(offset_of!(GlobalsG15V14_8_3, mtr_sensor_ptd_override_mask) == 0x9dd); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_de5) == 0xde5); + static_assert!(offset_of!(GlobalsG15V14_8_3, soft_fault_settings) == 0xdf9); + + // P0 status block + static_assert!(size_of::() == 0xc3d0); + static_assert!(offset_of!(StatusBlock, fw_recovery_info_a) == 0x40a0); + static_assert!(offset_of!(StatusBlock, recovery_packet_info) == 0x44a0); + static_assert!(offset_of!(StatusBlock, unk_44c0) == 0x44c0); + static_assert!(offset_of!(StatusBlock, unk_4558) == 0x4558); + static_assert!(offset_of!(StatusBlock, fwctl_channel) == 0x4568); + static_assert!(offset_of!(StatusBlock, flags) == 0x4580); + static_assert!(offset_of!(StatusBlock, flags.halted) == 0x4590); + static_assert!(offset_of!(StatusBlock, flags.resume) == 0x45a0); + static_assert!(offset_of!(StatusBlock, flags.unk_40) == 0x45b0); + static_assert!(offset_of!(StatusBlock, flags.sched_callbacks) == 0x45c4); + static_assert!(offset_of!(StatusBlock, qos_mode) == 0xc3c8); + + // P1 power-controller block + static_assert!(size_of::() == 0x238); + static_assert!(offset_of!(PowerCtlBlock, power_zone_count) == 0x0c); + static_assert!(offset_of!(PowerCtlBlock, power_zones) == 0x24); + static_assert!(offset_of!(PowerCtlBlock, clvr_loop_override_a) == 0x60); + static_assert!(offset_of!(PowerCtlBlock, lifetime_servo_temp_a) == 0xa4); + static_assert!(offset_of!(PowerCtlBlock, fast_die0_release_temp) == 0xac); + static_assert!(offset_of!(PowerCtlBlock, ppm_words) == 0xc8); + static_assert!(offset_of!(PowerCtlBlock, unk_dc) == 0xdc); + static_assert!(offset_of!(PowerCtlBlock, se_controller_override) == 0xe8); + static_assert!(offset_of!(PowerCtlBlock, se_engagement) == 0xec); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_a) == 0x158); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_b) == 0x1a8); + static_assert!(offset_of!(PowerCtlBlock, clpc_shared_a) == 0x1b4); + static_assert!(offset_of!(PowerCtlBlock, perf_ctrl_override_active) == 0x1d0); + static_assert!(offset_of!(PowerCtlBlock, consistent_perf_state) == 0x1d1); + static_assert!(offset_of!(PowerCtlBlock, accumulated_energy) == 0x1d8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_a) == 0x1e8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_b) == 0x218); + static_assert!(offset_of!(PowerCtlBlock, submit_in_progress_gate) == 0x230); + + // P2 debug block + static_assert!(size_of::() == 0x20); + static_assert!(offset_of!(DebugBlock, system_sleep_in_progress) == 0x08); + static_assert!(offset_of!(DebugBlock, fw_word_10) == 0x10); + static_assert!(offset_of!(DebugBlock, init_state) == 0x14); + + // InitSeq record + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(InitSeqRecord, kind) == 0x10); + + // HwDataB: 0x1868 bytes + static_assert!(size_of::() == 0x1868); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_30) == 0x20); + static_assert!(offset_of!(HwDataBG15V14_8_3, timestamp_area_base) == 0x28); + static_assert!(offset_of!(HwDataBG15V14_8_3, yuv_matrices) == 0x38); + static_assert!(offset_of!(HwDataBG15V14_8_3, pad_1c8) == 0x638); + static_assert!(offset_of!(HwDataBG15V14_8_3, io_mappings) == 0x640); + static_assert!(offset_of!(HwDataBG15V14_8_3, sgx_sram_ptr) == 0xa20); + static_assert!(offset_of!(HwDataBG15V14_8_3, chip_id) == 0xa28); + static_assert!(offset_of!(HwDataBG15V14_8_3, base_clock_khz) == 0xa68); + static_assert!(offset_of!(HwDataBG15V14_8_3, power_sample_period) == 0xa6c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_a9c) == 0xa9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_ac8) == 0xac8); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_b24) == 0xb24); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_554) == 0xb40); + static_assert!(offset_of!(HwDataBG15V14_8_3, uat_ttb_base) == 0xb44); + static_assert!(offset_of!(HwDataBG15V14_8_3, gpu_core_id) == 0xb4c); + static_assert!(offset_of!(HwDataBG15V14_8_3, num_cores) == 0xb54); + static_assert!(offset_of!(HwDataBG15V14_8_3, max_pstate) == 0xb58); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies) == 0xb5c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages) == 0xb9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages_sram) == 0xd9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies_2) == 0xf9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, sram_k) == 0xfdc); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_9f4) == 0x101c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_arr_0) == 0x10dc); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps) == 0x115c); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps_3) == 0x134c); + static_assert!(size_of::() == 0x444); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_179c) == 0x179c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_17e8) == 0x17e8); + static_assert!(offset_of!(HwDataBG15V14_8_3, timer_offset) == 0x17ec); + static_assert!(offset_of!(HwDataBG15V14_8_3, all_ones_masks) == 0x1820); + static_assert!(offset_of!(HwDataBG15V14_8_3, fstp_override) == 0x1850); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_1860) == 0x1860); + static_assert!(size_of::() == 0x20); + + // HwDataA: 0x4360 bytes, G13 13.5 layout, +0x10 from 0x1288 + static_assert!(size_of::() == 0x4360); + static_assert!(offset_of!(HwDataAG15V14_8_3, clocks_per_period) == 0x4); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_status) == 0x10); + static_assert!(offset_of!(HwDataAG15V14_8_3, actual_pstate) == 0x2c); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled) == 0x54); + static_assert!(offset_of!(HwDataAG15V14_8_3, sram_k) == 0x80); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_64c) == 0x658); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_filter_a_neg) == 0x664); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_power_2) == 0x6a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization) == 0x788); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_filter_drop_threshold) == 0x7a8); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization_2) == 0x7ec); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled_4) == 0x860); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask) == 0x8ac); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_release_temp_cc) == 0x8b4); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_ki_dt) == 0x8d8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_kp) == 0x8e8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_prop_tgt_delta) == 0x900); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_10) == 0x938); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_coef_a1) == 0x970); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_11) == 0xbe0); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_c2c) == 0xe80); + static_assert!(offset_of!(HwDataAG15V14_8_3, power_zone_count) == 0xe84); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_target_filter_a_neg) == 0xe9c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_periods_x4) == 0xf7c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_kp) == 0xfa8); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_ms) == 0xfd0); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_14) == 0xff8); + // The builder relies on this base for the G15 +0x10f4/+0x11d0 stores + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e10_0) == 0x10b4); + static_assert!(size_of::() == 0x1b8); + static_assert!(offset_of!(HwDataA140Extra, unk_40) == 0x40); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_a_neg) == 0x4c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_kp) == 0x6c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_1) == 0x7c); + static_assert!(offset_of!(HwDataA140Extra, se_target) == 0x8c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_time_constant_clks) == 0x9c); + static_assert!(offset_of!(HwDataA140Extra, unk_b0) == 0xb0); + static_assert!(offset_of!(HwDataA140Extra, unk_d8) == 0xd8); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_reset_criteria) == 0xf0); + static_assert!(offset_of!(HwDataA140Extra, unk_11c) == 0x11c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_2) == 0x130); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_2) == 0x1288); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e28) == 0x1294); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_alt) == 0x1a98); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_3640) == 0x3aa4); + static_assert!(offset_of!(HwDataAG15V14_8_3, init_timestamp) == 0x424c); + static_assert!(offset_of!(HwDataAG15V14_8_3, cluster_tables) == 0x42a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_4340) == 0x4340); + static_assert!(offset_of!(HwDataAG15V14_8_3, gpu_keepalive_mode) == 0x4358); + + // E2 StatsFrag: 0x1248 bytes, -1 fields at +0xc18/+0xc30 + static_assert!(size_of::() == 0x1248); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, cur_stamp_id) + == 0xc18 + ); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, unk_id) + == 0xc30 + ); + + // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). + // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these + // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X + // sizes. + static_assert!(size_of::>() == 0x6bc0); + static_assert!(size_of::>() == 0xb3b0); + static_assert!(size_of::() == 0x12394); + static_assert!(size_of::() == 0x1715c); + static_assert!(size_of::() == 0x421c); + static_assert!(size_of::() == 0x6c34); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xb4); + static_assert!(size_of::() == 0x30); + static_assert!(size_of::() == 0x1124); + static_assert!(size_of::() == 0x1a34); + static_assert!(size_of::>() == 0x30); +} diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs new file mode 100644 index 0000000000000..c90156bd1244b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::fw::types::*; +pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod agx3; +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t6030; +pub(crate) mod t6031; +pub(crate) mod t8103; +pub(crate) mod t8112; +pub(crate) mod t8122; +pub(crate) mod t8140; + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + /// The chip revision T6031 (M3 Max) reports (/arm-io chip-revision 0x12). Only the T6031 + /// table names it; no identification path maps a register read to it yet. + #[allow(dead_code)] + B2 = 0x12, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D + // G15 firmware core type IDs (G15S = 0x17). + // G15M = 19, + // G15P_AGX2 = 20, + // G15P = 21, + G15G = 22, + G15S = 23, + /// Not confirmed on hardware yet; only the T6031 table uses it. + #[allow(dead_code)] + G15C = 24, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + /// + /// This is a firmware-ABI value (renumbered per firmware build). It is + /// `None` where no firmware GpuCore id has been recovered (AGX3 parts): + /// consumers must fail closed rather than substitute a number. + pub(crate) gpu_core: Option, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Input-address bits translated by each UAT root. + pub(crate) uat_ias: u8, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// Unified shader core generation. + pub(crate) usc_generation: u32, + /// Apple GPU HAL generation (0 for the pre-numbered generations). + pub(crate) gpu_hal_generation: GpuHalGeneration, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Number of GPU dies read from the ID register. + pub(crate) num_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, + +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, + /// Shader-engine controller target (G15 runtime layout). + pub(crate) se_target: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &kernel::device::Device, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev, + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + // Round up: a voltage is never published below the device tree's. + volt_uv.iter_mut().for_each(|a| *a = if cfg.gpu_gen == GpuGen::G15 { a.div_ceil(1000) } else { *a / 1000 }); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree node of `dev`. + pub(crate) fn load(dev: &kernel::device::Device, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev, "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { + // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. + // Tolerate a DT without the property on G15 only. + prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + } else { + prop!("apple,min-sram-microvolt") + }, + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + se_target: if cfg.gpu_gen == GpuGen::G15 { prop!("apple,se-target", 1400) } else { 1400 }, + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs new file mode 100644 index 0000000000000..cb0c4fdbfa012 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +#[cfg(not(test))] +use crate::identity::{GpuGen, GpuVariant}; +#[cfg(test)] +#[path = "../identity.rs"] +mod identity; +#[cfg(test)] +use identity::{GpuGen, GpuVariant}; + +/// Identification-only configuration for one AGX3 SoC. +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +pub(crate) const T8122: SocConfig = agx3(0x8122, GpuGen::G15, GpuVariant::G, 0x7, 2, 1); +/// Apple M3 Pro (G15S). +pub(crate) const T6030: SocConfig = agx3(0x6030, GpuGen::G15, GpuVariant::S, 0x7, 3, 1); +/// Apple M3 Max (G15C). +pub(crate) const T6031: SocConfig = agx3(0x6031, GpuGen::G15, GpuVariant::C, 0x7, 4, 1); +/// Apple M3 Ultra (G15D, two dies). +pub(crate) const T6032: SocConfig = agx3(0x6032, GpuGen::G15, GpuVariant::D, 0x7, 4, 2); diff --git a/drivers/gpu/drm/asahi/m3/hw/t8122.rs b/drivers/gpu/drm/asahi/m3/hw/t8122.rs new file mode 100644 index 0000000000000..5ba6b0ef6acd6 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/t8122.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for T8122 (M3, G15G): one die, one cluster of ten core slots. +//! +//! Only the M3 runtime reads this table (`m3_soc::T8122`), and it refuses a T8122 until the +//! rest of the SoC's runtime configuration is known (`m3_soc::Soc::require_complete`). The +//! G15 manager backend never starts on T8122. + +use crate::f32; + +use super::*; + +/// Fender: ADT sgx reg[1] translated through the /arm-io ranges (SGX + 0xd00000). +const FENDER_BASE: usize = 0x2_90d0_0000; +/// The GPU register window: ADT sgx reg[0] translated through the /arm-io ranges. +const SGX_BASE: usize = 0x2_9000_0000; + +/// The firmware IO mappings, one per HwDataB slot (31 on G15). A slot is `None` when T8122 has +/// no block for it; its record then stays zero. Every address is a CPU physical address. +const IO_MAPPINGS: [Option; 31] = [ + // 0: Fender. The window is 0x104000 bytes: the ADT sgx reg[1] size (0x12c000) less 0x28000, + // the same difference as on T6030. + Some(IOMapping::new(FENDER_BASE, false, 1, 0x10_4000, 0, true)), + // 1: AIC timer, the same 16 KiB page on every SoC. + Some(IOMapping::new(0x2_0e10_0000, false, 1, 0x4000, 0, false)), + // 2: AIC software interrupts: AIC (0x2d1000000) + 0x14000, the page that holds the ADT's + // meta-sw-interrupt registers. + Some(IOMapping::new(0x2_d101_4000, false, 1, 0x4000, 0, true)), + // 3: RGX: the first 128 KiB of the GPU register window. + Some(IOMapping::new(SGX_BASE, false, 1, 0x2_0000, 0, true)), + None, // 4: UVD: the ADT sgx node has no third reg range + None, // 5: unused + None, // 6: display underrun workaround + None, // 7: analog temperature sensors + None, // 8: PMP doorbell + // 9: metrology sensors, SGX + 0xe08000. + Some(IOMapping::new(SGX_BASE + 0xe0_8000, false, 1, 0x8000, 0, true)), + // 10: GM GIFAF registers, a 4 KiB page at Fender + 0xd000. + Some(IOMapping::new(FENDER_BASE + 0xd000, false, 1, 0x1000, 0, true)), + // 11: memory cache, two instances of 0x58000 bytes (a multiple of 16 KiB), at 0x220000000 + // and 0x222000000. + Some(IOMapping::new(0x2_2000_0000, false, 2, 0x5_8000, 0x200_0000, true)), + None, // 12: AIC banked registers + None, // 13: PMGR scratch + None, // 14: NIA idle register, die 0 + None, // 15: NIA idle register, die 1 + None, // 16: CRE registers + None, // 17: streaming codec registers + // 18: telemetry dashboard, written by the firmware: the 4 KiB at +0x10000 of the PMP's + // telemetry window (0x2d03c0000, 0x14000 bytes). + Some(IOMapping::new(0x2_d03d_0000, false, 1, 0x1000, 0, true)), + // 19: telemetry dashboard, read by the firmware: the first 8 KiB of the same window. + Some(IOMapping::new(0x2_d03c_0000, false, 1, 0x2000, 0, false)), + None, // 20: telemetry dashboard configuration + None, // 21 + None, // 22 + None, // 23: AFR registers + None, // 24 + // 25: ANE doorbell, ANE + 0x45c000. + Some(IOMapping::new(0x3_1145_c000, false, 1, 0x4000, 0, true)), + // 26: PMS metrology sensors. + Some(IOMapping::new(0x2_d028_0000, false, 1, 0x8000, 0, false)), + None, // 27 + None, // 28 + // 29: GPU clock generator, SGX + 0xe1c000. + Some(IOMapping::new(SGX_BASE + 0xe1_c000, false, 1, 0x4000, 0, false)), + None, // 30 +]; + +/// T8122 (M3, G15G): one die, one cluster of ten core slots, eight or ten of them active. +pub(crate) const HWCONFIG_T8122: super::HwConfig = HwConfig { + chip_id: 0x8122, + gpu_gen: GpuGen::G15, // ID_VERSION[31:24] == 7 + gpu_variant: GpuVariant::G, // ID_VERSION[23:16] == 2 + gpu_core: Some(GpuCore::G15G), + + // The 24 MHz reference clock (HwDataB base_clock_khz = 24000). + base_clock_hz: 24_000_000, + // 42-bit UAT roots and output addresses, as on every AGX3 part (hw::agx3::T8122). + uat_ias: 42, + uat_oas: 42, + num_dies: 1, // ID_COUNTS_1[19:16] + max_num_clusters: 1, // one cluster of ten core slots + max_num_cores: 10, + max_num_frags: 10, // one fragment unit per core slot + max_num_gps: 4, // ID_COUNTS_2[23:16] + + // Not read by the M3 runtime, which keeps its own buffer layout. + preempt1_size: 0, + preempt2_size: 0, + preempt3_size: 0, + // Compute preemption buffer: 0xd80 bytes plus 0x700 per cluster. + compute_preempt1_size: 0x1480, + clustering: None, + render: HwRenderConfig { tiling_control: 0 }, + + da: HwConfigA { + // Cleared by the M3 runtime's InitData in any case. + unk_87c: 0, + // Fast-die temperature target, in centidegrees Celsius (110 C, as on T6030). The J613 + // ADT's gpu-fast-die0-target is 88; how the two relate is not known. + unk_8cc: 11_000, + unk_e24: 0, + }, + db: HwConfigB { + // The chip revision's major number: /arm-io chip-revision 0x20 >> 4. + unk_454: 2, + // The G15 InitData builders read none of the words below. + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0, + unk_abc: 0, + unk_b30: 1, + }, + // G15 leaves the hws1/hws2/hws3 curves zero, as on T6030. + shared1_tab: &[], + shared1_a4: 0, + shared2_tab: &[], + shared2_unk_508: 0, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + // Idle-off standby timer when the device tree gives none: the J613 ADT has no + // gpu-idleoff-standby-timer. + idle_off_standby_timer_default: 700, + // The smart idle-off curve and count, the same as T6030's. + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + unk_hws2_24: 6, + // The M3 runtime writes 0xffff to this Globals word in any case. + global_unk_54: 0, + // SRAM power scale 1.02 for every performance state, as on T6030. + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + // No FwUtil performance tunables are written. + global_tab: None, + has_csafr: false, + // The fast-die sensor mask is each machine's: the M3 runtime takes it from the boot + // loader's apple,fast-die0-sensor-mask (the ADT's gpu-fast-die0-sensor-mask), not from here. + fast_sensor_mask: [0, 0], + fast_sensor_mask_alt: [0, 0], + fast_die0_sensor_present: 0, + io_mappings: &IO_MAPPINGS, + // No Fender scratch SRAM window. + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/m3/identity.rs b/drivers/gpu/drm/asahi/m3/identity.rs new file mode 100644 index 0000000000000..f468272c5680e --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/identity.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +/// Supported GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, + G15 = 15, + G17 = 17, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuHalGeneration { + Legacy = 0, + Hal200 = 200, +} + +/// Public architecture axes for the G15 ID-register decoder. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) struct GpuIdentity { + pub(crate) gpu_gen: GpuGen, + pub(crate) gpu_variant: GpuVariant, + pub(crate) usc_generation: u32, + pub(crate) gpu_hal_generation: GpuHalGeneration, +} + +pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option { + use GpuVariant::*; + if family != 7 { return None; } + let (gpu_variant, usc_generation) = match variant { + 0 => (P, 2), + 2 => (G, 3), + 3 => (S, 3), + 4 => match num_dies { 1 => (C, 3), 2 => (D, 3), _ => return None }, + _ => return None, + }; + Some(GpuIdentity { gpu_gen: GpuGen::G15, gpu_variant, usc_generation, + gpu_hal_generation: GpuHalGeneration::Legacy }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn g15_variants_keep_their_public_identity() { + assert_eq!(decode_gpu_identity(7, 3, 1).unwrap().gpu_variant, GpuVariant::S); + assert_eq!(decode_gpu_identity(7, 4, 1).unwrap().gpu_variant, GpuVariant::C); + assert_eq!(decode_gpu_identity(7, 4, 2).unwrap().gpu_variant, GpuVariant::D); + assert_eq!(decode_gpu_identity(7, 2, 1).unwrap().usc_generation, 3); + assert!(decode_gpu_identity(7, 4, 0).is_none()); + assert!(decode_gpu_identity(10, 2, 1).is_none()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs new file mode 100644 index 0000000000000..dfeffd900eb12 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. + +/// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live +/// in Fender scratch SRAM without a backup/restore path. +/// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the +/// firmware treats the value as signed, and is never smaller than u32::MAX would be if it +/// converts ms to 24 MHz ticks in 32-bit arithmetic (both wrap to ~179 s); with 64-bit +/// arithmetic it is ~24 days. +#[allow(dead_code)] +pub(crate) const G15_KEEP_POWERED_MS: u32 = i32::MAX as u32; + +/// Which T6030 backend a G15 InitData is built for. The two differ where the manager backend +/// keeps the values it was validated with on J516S. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum G15Layout { + /// The G15 GpuManager backend: rings in Fender SRAM, the GPU never idles off. + Manager, + /// The serialized M3 runtime backend: rings in DRAM with finite idle-off, its own GPU and + /// firmware VA layout, and the values its InitData carries for the power, idle and + /// shader-engine controllers. + Runtime, +} + +/// G15 InitData choices that are not part of the hardware or device-tree configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct G15Options { + /// The backend the InitData is built for. + pub(crate) layout: G15Layout, + /// Publish the performance states as two voltage-sorted tables (see [`PStateTables`]). + pub(crate) split_pstates: bool, + /// Highest performance state the firmware may use, as an index into the voltage-sorted + /// primary table (1 = lowest). None publishes the whole table. + pub(crate) cap: Option, + /// Firmware VA of the timestamp arena (HwDataB+0x28); None uses the manager's kernel range. + pub(crate) timestamp_base: Option, + /// Manager layout: write the PPM words and the Globals power targets as the runtime layout + /// does (`asahi.g15_debug` bit 55). The runtime layout always does. + pub(crate) reference_ppm: bool, + /// Runtime layout: the SoC's HwDataB configuration words and unit masks + /// (`m3_soc::Soc::hwdata_b`). None for the manager layout, which does not write them. + pub(crate) runtime_hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// Accept operating points whose clusters have different voltages + /// (`m3_soc::Soc::per_cluster_voltages`); see [`PStateTables`]. + pub(crate) per_cluster_voltages: bool, +} + +/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor +/// device-tree values: configuration words where the manager layout keeps 0 or its validated +/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values +/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. +#[derive(Debug)] +pub(crate) struct G15RuntimeHwDataB { + pub(crate) unk_454: u32, + pub(crate) unk_464: u32, + pub(crate) unk_a7c: u64, + pub(crate) unk_a98: u32, + pub(crate) unk_abc: u64, + pub(crate) unk_ae4: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u64, + /// +0xb40, the "UAT enabled" word of the manager layout. + pub(crate) unk_554: u32, + pub(crate) unk_17b8: u32, + /// Unit enable masks, +0x17c0/+0x17c8. + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_1818: u32, +} + +impl G15Options { + /// The options of the G15 manager backend: its validated defaults, the performance-state + /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. + fn manager(cfg: &'static hw::HwConfig) -> G15Options { + let g15 = cfg.gpu_gen == hw::GpuGen::G15; + G15Options { + layout: G15Layout::Manager, + split_pstates: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), + cap: g15.then(|| { + crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) + }), + timestamp_base: None, + reference_ppm: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), + runtime_hwdata_b: None, + per_cluster_voltages: false, + } + } + + /// The published performance-state tables for `pwr`. + pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { + PStateTables::new(pwr, self.split_pstates, self.cap, self.per_cluster_voltages) + } + + /// RuntimePointers+0x2d0, a configuration word: 4 in the runtime backend's InitData, 0 (its + /// validated value) for the manager. + pub(crate) fn runtime_pointers_2d0(&self) -> u32 { + match self.layout { + G15Layout::Manager => 0, + G15Layout::Runtime => 4, + } + } +} + +/// Start of the USC (shader code) window in the runtime backend's GPU VA layout. +const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; +/// The runtime backend's unknown-page address, the end of its 42-bit user VA range. +const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; + +/// Maximum number of entries of a firmware performance-state table, including the off state. +const PSTATE_TABLE_ENTRIES: usize = 16; + +/// The performance-state tables as published to the G15 firmware. +/// +/// The device tree lists the performance states in the bootloader's order: several states can +/// share one voltage, and the frequencies are not sorted. The firmware takes two tables sorted +/// by voltage: the primary one with the highest frequency of each voltage, the secondary one with +/// the lowest, plus the device-tree index of every entry. Entry 0 is the off state in both. +/// Every frequency and voltage comes from the device tree; nothing is interpolated. +/// +/// With per-cluster voltages (T6031, whose clusters have their own binned voltage tables), the +/// states are grouped by their whole voltage list instead: states that share their highest +/// cluster voltage must have the same voltage on every cluster, the groups are ordered by that +/// highest voltage, and every cluster's voltage must rise from one entry to the next. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PStateTables { + /// Number of entries, including the off state. + pub(crate) len: usize, + /// Device-tree index of each primary-table entry. + pub(crate) primary: [u8; PSTATE_TABLE_ENTRIES], + /// Device-tree index of each secondary-table entry (split tables only). + pub(crate) secondary: Option<[u8; PSTATE_TABLE_ENTRIES]>, + /// Highest primary-table entry the firmware may select. + pub(crate) max: u32, +} + +impl PStateTables { + /// The primary/secondary split of `states`: one entry per distinct voltage, ascending. + fn split( + states: &[hw::PState], + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + Self::split_for(states, false) + } + + /// [`Self::split`], accepting per-cluster voltages when `per_cluster` is set. + fn split_for( + states: &[hw::PState], + per_cluster: bool, + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + let mut secondary = [0u8; PSTATE_TABLE_ENTRIES]; + if states.len() < 2 || states.len() > u8::MAX as usize || states[0].freq_hz != 0 { + return Err(EINVAL); + } + let mut len = 1; + let mut last_mv = 0; + loop { + // The next voltage above `last_mv`, and its fastest and slowest states. + let mut next: Option<(u32, usize, usize)> = None; + for (i, ps) in states.iter().enumerate().skip(1) { + let mv = ps.max_volt_mv(); + // Every state after the off state runs, with one voltage for all clusters + // unless the SoC gives each cluster its own. + if ps.freq_hz == 0 || (!per_cluster && ps.volt_mv.iter().any(|v| *v != mv)) { + return Err(EINVAL); + } + if mv <= last_mv { + continue; + } + // States of one entry run every cluster at the same voltages. + if let (true, Some((best, hi, _))) = (per_cluster, next) { + if mv == best && states[hi].volt_mv[..] != ps.volt_mv[..] { + return Err(EINVAL); + } + } + next = match next { + Some((best, hi, lo)) if mv == best => Some(( + mv, + if ps.freq_hz > states[hi].freq_hz { i } else { hi }, + if ps.freq_hz < states[lo].freq_hz { i } else { lo }, + )), + Some((best, _, _)) if mv > best => next, + _ => Some((mv, i, i)), + }; + } + let Some((mv, hi, lo)) = next else { + break; + }; + if len >= PSTATE_TABLE_ENTRIES + || states[hi].freq_hz <= states[primary[len - 1] as usize].freq_hz + || states[lo].freq_hz <= states[secondary[len - 1] as usize].freq_hz + { + // Faster states must need a higher voltage in both tables. + return Err(EINVAL); + } + // With per-cluster voltages, every cluster's voltage rises with the entry. + if per_cluster && len > 1 { + let previous = &states[primary[len - 1] as usize].volt_mv; + let current = &states[hi].volt_mv; + if previous.len() != current.len() + || previous.iter().zip(current.iter()).any(|(p, c)| c <= p) + { + return Err(EINVAL); + } + } + primary[len] = hi as u8; + secondary[len] = lo as u8; + len += 1; + last_mv = mv; + } + // At least one running state must have a nonzero voltage; callers + // index the table up to len - 1 and clamp caps to 1..=len - 1. + if len < 2 { + return Err(EINVAL); + } + Ok((primary, secondary, len)) + } + + /// Build the published tables. `cap` is an index into the voltage-sorted primary table, + /// also when the states are published in device-tree order: the firmware may then use any + /// device-tree state up to the capped frequency, and the cap is translated to the highest + /// device-tree index whose states all stay at or below it. + /// `per_cluster` accepts per-cluster voltages (see [`PStateTables`]). + pub(crate) fn new( + pwr: &hw::PwrConfig, + split: bool, + cap: Option, + per_cluster: bool, + ) -> Result { + let states = &pwr.perf_states; + if split { + let (primary, secondary, len) = Self::split_for(states, per_cluster)?; + let top = len as u64 - 1; + let max = cap.map_or(top, |c| c.clamp(1, top)) as u32; + return Ok(PStateTables { + len, + primary, + secondary: Some(secondary), + max, + }); + } + if states.len() > PSTATE_TABLE_ENTRIES { + return Err(EINVAL); + } + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + for (i, slot) in primary.iter_mut().enumerate().take(states.len()) { + *slot = i as u8; + } + let top = states.len() as u32 - 1; + let max = match cap { + // No voltage-sorted table can have more entries than the device tree has states, + // so such a cap leaves the whole table. + None => top, + Some(c) if c >= u64::from(top) => top, + Some(c) => { + let (sorted, _, sorted_len) = Self::split(states)?; + let limit_hz = states[sorted[c.clamp(1, sorted_len as u64 - 1) as usize] as usize].freq_hz; + // The longest device-tree prefix that stays at or below the capped frequency. + let mut max = 1; + while max < top && states[max as usize + 1].freq_hz <= limit_hz { + max += 1; + } + max + } + }; + Ok(PStateTables { + len: states.len(), + primary, + secondary: None, + max, + }) + } + + /// The device-tree state of primary-table entry `i`. + pub(crate) fn primary_state<'s>(&self, pwr: &'s hw::PwrConfig, i: usize) -> &'s hw::PState { + &pwr.perf_states[self.primary[i] as usize] + } + + /// The frequency of the highest entry the firmware may select, in kHz. + pub(crate) fn max_frequency_khz(&self, pwr: &hw::PwrConfig) -> u32 { + self.primary_state(pwr, self.max as usize).freq_hz / 1000 + } +} + +/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its +/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. +pub(crate) fn g15_manager_max_frequency_khz( + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, +) -> Option { + if cfg.gpu_gen != hw::GpuGen::G15 { + return None; + } + let tables = G15Options::manager(cfg).tables(pwr).ok()?; + Some(tables.max_frequency_khz(pwr)) +} + +pub(crate) struct InitDataBuilderG15V14_8_3<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: G15Options, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilderG15V14_8_3<'a> { + InitDataBuilderG15V14_8_3 { + dev, + alloc, + cfg, + dyncfg, + g15: G15Options::manager(cfg), + } + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + } + _ => (), + } + Ok(()) + }) + } + + /// Build the HwDataA contents (mostly power-related configuration) without allocating the + /// object, so that they can fill a firmware object or a plain buffer alike. + pub(crate) fn hwdata_a_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let tables = g15.tables(pwr)?; + let max_ps = tables.max; + let runtime = g15.layout == G15Layout::Runtime; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = if cfg.gpu_gen == hw::GpuGen::G15 { + max_ps.checked_sub(base_ps).ok_or(EINVAL)? + } else { max_ps - base_ps }; + + #[allow(unused_variables)] + let base_clock_khz = cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + Ok( + try_init!(raw::HwDataAG15V14_8_3 { + clocks_per_period: v_clocks_per_period, + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + unk_163c: 1, + unk_3644: 0, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for i in 0..tables.len { + raw.sram_k[i] = cfg.sram_k; + } + + // The only constants the G15 firmware expects in the HwDataA 0x10b4 block + // (+0x10f4 = 0x32, +0x11cc = {0, 65536.0}). + // unk_e10_0 sits at 0x10b4 (asserted in fw/initdata.rs), so +0x40 is 0x10f4 + // and +0x11c is 0x11d0. The runtime layout fills the whole block below. + { + raw.unk_e10_0.unk_40 = 0x32; + raw.unk_e10_0.unk_11c = f32!(65536.0); + } + + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..dyncfg.id.num_clusters as usize { + if let Some(coef_a) = cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + raw.power_zones[i].unk_10 = 1320000000; + } + + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if runtime { + Self::hwdata_a_runtime(raw, pwr, max_ps_scaled, clocks_per_period_coarse); + } + + Ok(()) + })) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataAG15V14_8_3 as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(dev.as_ref(), "!!! Hwdata A match"); + } + } + } + Ok(()) + }) + }) + } + + /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the + /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device + /// tree), and the leakage coefficient and flag words of the G15 tail. Words that the + /// runtime's InitData keeps at zero are cleared. + fn hwdata_a_runtime( + raw: &mut raw::HwDataAG15V14_8_3, + pwr: &hw::PwrConfig, + max_ps_scaled: u32, + clocks_per_period_coarse: u32, + ) { + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + let se = &mut raw.unk_e10_0; + se.unk_40 = 50; + se.unk_44 = 1; + se.gpu_se_filter_a_neg = f32!(1.0) - filter_a; + se.gpu_se_filter_1_a_neg = f32!(1.0) - filter_1_a; + se.gpu_se_filter_a = filter_a; + se.gpu_se_filter_1_a = filter_1_a; + se.gpu_se_ki_dt = pwr.se_ki * period_s; + se.gpu_se_ki_1_dt = pwr.se_ki_1 * period_s; + se.unk_68 = f32!(65536.0); + se.gpu_se_kp = pwr.se_kp; + se.gpu_se_kp_1 = pwr.se_kp_1; + se.unk_78 = 100; + se.max_pstate_scaled_1 = max_ps_scaled; + se.min_pstate_scaled = 100; + se.unk_88 = f32!(8000.0); + // The shader-engine target (ADT gpu-se-tgt, 1400 on J514S and J516S; the device tree + // may override it). + se.se_target = pwr.se_target; + se.gpu_se_filter_time_constant_ms = pwr.se_filter_time_constant * period_ms; + se.gpu_se_filter_time_constant_1_ms = pwr.se_filter_time_constant_1 * period_ms; + se.gpu_se_filter_time_constant_clks = + U64((pwr.se_filter_time_constant * clocks_per_period_coarse).into()); + se.gpu_se_filter_time_constant_1_clks = + U64((pwr.se_filter_time_constant_1 * clocks_per_period_coarse).into()); + se.unk_b0 = f32!(65536.0); + se.unk_d8 = 8000; + se.unk_dc = 4; + se.gpu_se_inactive_threshold = pwr.se_inactive_threshold; + se.gpu_se_engagement_criteria = pwr.se_engagement_criteria; + se.unk_e8 = 2; + se.unk_ec = 4; + se.gpu_se_reset_criteria = pwr.se_reset_criteria; + se.unk_11c = f32!(65536.0); + se.unk_12c = 40; + se.max_pstate_scaled_2 = max_ps_scaled; + + // No fast-die release temperature, no +0x8b8 offset and no alternate sensor mask: the + // T6030 device tree has no source for them. + raw.fast_die0_release_temp_cc = 0; + raw.unk_87c = 0; + raw.fast_die0_sensor_mask_alt = U64(0); + raw.unk_3640 = f32!(5.0).to_bits(); + // The fused core leakage coefficient of this machine (device tree), first cluster table. + if let Some(coef) = pwr.core_leak_coef.first() { + raw.cluster_tables[0][0] = coef.to_bits(); + } + raw.unk_4298 = 1; + raw.unk_429c[0] = 1; + } + + // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any + // (debug aid, same as the G13/G14 builders do inline). + fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { + if blob.is_empty() { + return; + } + // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. + let sla = unsafe { + core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) + }; + if sla.len() != blob.len() { + dev_err!( + dev.as_ref(), + "!!! {} size mismatch: {} {}", + what, + sla.len(), + blob.len() + ); + } + match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { + Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), + None => dev_info!(dev.as_ref(), "!!! {} match", what), + } + } + + // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose + // source is not known yet are left 0 and marked TODO. + // (Plain comments: the versions macro needs `fn` right after a gating #[ver].) + fn hwdata_b_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let runtime_hwdata_b = g15.runtime_hwdata_b; + let timestamp_base = match g15.timestamp_base { + Some(base) => base, + None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + }; + Ok(try_init!(raw::HwDataBG15V14_8_3 { + // VA-layout words, +0x0/+0x8 + unk_8: U64(0x6f_00000000), + unk_18: U64(0xffc00000), + // USC start: the G15 firmware takes 0x10_00000000 at +0x10/+0x18. Like the G13 + // builder, the manager leaves 0: drm/asahi passes the USC base per queue. + unk_20: U64(0), + unk_28: U64(0), + // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its + // 42-bit user layout; the manager keeps a 39-bit user range and maps the page + // at IOVA_UNK_PAGE, so it passes that. + unk_30: U64(mmu::IOVA_UNK_PAGE), + // FW VA of the timestamp area, +0x28 + timestamp_area_base: U64(timestamp_base), + // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) + // are left 0, as on G13 ("TODO: yuv matrices"). + // +0xa28..+0xa34: chip id + 3 words (G13 unk_454..45c) + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + // +0xa38..+0xa64 (G13 unk_460..unk_48c). +0xa48/+0xa54/+0xa64 are 0. + // +0xa38 (GPC present, 1 on t6030) and +0xa40 (PIO supported) are 1: the + // firmware runs its power management, and so powers up the GPU cores for + // a kicked job, only if both are non-zero. +0xa50/+0xa58/+0xa60 are 1. + // Tested on J516S: an empty compute job completes with these values. + unk_460: 1, + unk_468: 1, + unk_478: 1, + unk_480: 1, + unk_488: 1, + // +0xa68: 0x5dc0 = 24000 + base_clock_khz: cfg.base_clock_hz / 1000, + // +0xa6c: 200 on slow simulation models, otherwise 1, so 1 on t6030. The + // HwDataA filter constants follow +0xa70 (the sample period in ms), which + // stays the DT value. + power_sample_period: 1, + // +0xa70: the value HwDataA +0xea8/+0xeac multiply the filter time + // constants by, i.e. the sample period in ms (inferred) + unk_a70: pwr.power_sample_period, + // TODO: +0xa74, +0xa88, +0xa90, +0xaa0, +0xac4 are board/config values whose + // source is not known yet; left 0. + unk_a78: 1, + unk_a7c: U64(1), + unk_a98: 1, + unk_a9c: 0x1f, + unk_ac8: Array::new([0, 1, 1, 0]), + // TODO: +0xaa8 and +0xb04 are condition flags; left 0. + // "UAT enabled" flag, G13 unk_554 = 1. +0xb40 + unk_554: 0x1, + // +0xb44..+0xb58 + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core.ok_or(ENODEV)? as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: tables.len as u32 - 1, + // +0x17e8/+0x1800/+0x1804/+0x1810; + // +0x1814 = 1 as well. + unk_17e8: 1, + timer_offset: U64(0), + unk_1800: 1, + unk_1804: 1, + unk_1810: 1, + unk_1814: 1, + // +0x17fc = 1 on t6030. + unk_17fc: 1, + // +0x1820: 48 x 0xff + all_ones_masks: Array::new([0xff; 0x30]), + // +0x1860 = 1 & ~(debug global bit 0), so 1 without debug. (inferred) + unk_1860: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + // Rounded up, like the operating-point voltages. + let min_sram_mv = pwr.min_sram_microvolt.div_ceil(1000); + // Core and SRAM voltage rows of one state: every column holds the state's + // device-tree voltage (columns beyond the device tree's repeat the first one), and + // the SRAM rail never goes below the device tree's floor. + let rows = |ps: &hw::PState| -> ([u32; 8], [u32; 8]) { + let mut volts = [0u32; 8]; + for (j, v) in volts.iter_mut().enumerate() { + *v = *ps.volt_mv.get(j).unwrap_or(&ps.volt_mv[0]); + } + (volts, volts.map(|mv| mv.max(min_sram_mv))) + }; + + // p-state tables at +0xb58..+0xf9b, +0xfdc, same formulas as G13. Every + // frequency and voltage comes from the device tree. + for i in 0..tables.len { + let ps = tables.primary_state(pwr, i); + let (volts, sram) = rows(ps); + raw.frequencies[i] = ps.freq_hz / 1000000; + raw.voltages[i] = volts; + raw.voltages_sram[i] = sram; + raw.sram_k[i] = cfg.sram_k; + } + + match tables.secondary { + None => { + // One table in device-tree order: the second frequency table repeats it, + // and the index map is the identity. + for i in 0..tables.len { + raw.frequencies_2[i] = raw.frequencies[i]; + } + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + } + Some(secondary) => { + // The secondary table (+0xf9c), the device-tree index of every entry of + // both tables (+0x10dc, +0x111c) and the third table (+0x134c, the + // secondary table with one voltage column). + // The base state is a device-tree index, and the HwDataA base words use it as + // an index into the published table: the two must name the same state. + if pwr.perf_base_pstate as usize >= tables.len + || u32::from(tables.primary[pwr.perf_base_pstate as usize]) + != pwr.perf_base_pstate + { + return Err(EINVAL); + } + let top = tables.primary_state(pwr, tables.len - 1); + let base = tables.primary_state(pwr, pwr.perf_base_pstate as usize); + if top.pwr_mw == 0 || top.freq_hz <= base.freq_hz { + return Err(EINVAL); + } + let aux = &mut raw.aux_ps_3; + aux.max_pstate = tables.len as u32 - 1; + for i in 0..tables.len { + let ps = &pwr.perf_states[secondary[i] as usize]; + let (volts, sram) = rows(ps); + raw.frequencies_2[i] = ps.freq_hz / 1000000; + raw.unk_arr_0[i] = tables.primary[i] as u32; + raw.unk_arr_0[16 + i] = secondary[i] as u32; + aux.frequencies[i] = ps.freq_hz / 1000000; + aux.voltages[i][0] = volts[0]; + aux.voltages_sram[i][0] = sram[0]; + + // Maximum power and boost frequency of each primary entry, in percent + // of the top entry (boost relative to the base state). + let ps = tables.primary_state(pwr, i); + if i > 0 { + raw.rel_max_powers[i] = + (100 * u64::from(ps.pwr_mw) / u64::from(top.pwr_mw)) as u32; + } + if ps.freq_hz > base.freq_hz { + let f = u64::from(ps.freq_hz / 1000000 - base.freq_hz / 1000000); + let span = u64::from(top.freq_hz / 1000000 - base.freq_hz / 1000000); + raw.rel_boost_freqs[i] = (100 * f / span) as u32; + } + } + } + } + + if let Some(csafr) = pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // +0x17c0 (u64) / +0x17c8 (u32): masks with the low N bits set + // (per-core / per-cluster enable masks, inferred). + // TODO: which counts these are; default total cores / clusters. + let low_bits = |n: u32| -> u64 { + if n >= 64 { + u64::MAX + } else { + (1u64 << n) - 1 + } + }; + raw.unit_mask_a = U64(low_bits(dyncfg.id.num_cores * dyncfg.id.num_clusters)); + raw.unit_mask_b = low_bits(dyncfg.id.num_clusters.min(32)) as u32; + + if runtime { + Self::hwdata_b_runtime(raw, runtime_hwdata_b.ok_or(EINVAL)?); + } + Ok(()) + })) + } + + // The runtime backend's HwDataB words: its GPU VA layout (USC base, unknown page), and the + // SoC's configuration words and unit masks (`words`, see `G15RuntimeHwDataB`). Those are a + // runtime InitData's values, not derived from the device tree: the T6030 ones hold for the + // boards that InitData was checked on (J514S, J516S), not necessarily for other + // configurations. + fn hwdata_b_runtime(raw: &mut raw::HwDataBG15V14_8_3, words: &G15RuntimeHwDataB) { + raw.unk_20 = U64(RUNTIME_USC_BASE); + raw.unk_28 = U64(RUNTIME_USC_BASE); + raw.unk_30 = U64(RUNTIME_UNKNOWN_PAGE); + raw.unk_454 = words.unk_454; + raw.unk_464 = words.unk_464; + raw.unk_a7c = U64(words.unk_a7c); + raw.unk_a98 = words.unk_a98; + raw.unk_abc = U64(words.unk_abc); + raw.unk_ae4 = words.unk_ae4; + raw.unk_b20 = words.unk_b20; + raw.unk_b24 = U64(words.unk_b24); + raw.unk_554 = words.unk_554; + raw.unk_17b8 = words.unk_17b8; + raw.unit_mask_a = U64(words.unit_mask_a); + raw.unit_mask_b = words.unit_mask_b; + raw.unk_1808 = words.unk_1808; + raw.unk_1818 = words.unk_1818; + } + + // Create the G15 (14.8.3) HwDataB structure. + fn hwdata_b(&mut self) -> Result> { + let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); + Ok(()) + }) + }) + } + + // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields + // with a known init value are set; the rest start at 0. + fn globals_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = g15.reference_ppm; + Ok(try_init!(raw::GlobalsG15V14_8_3 { + // TODO: 0 or 7 depending on a board condition (+0x0); 0 = timer + // context switch off. + timer_cswitch: 0, + // G13 unk_24 = 0, debug = 0 (inferred) + unk_28: 0, + debug: 0, + // +0x38: constant 0x78 + unk_38: 0x78, + // G13 unk_54/56/58 moved to +0x50..+0x54 (inferred) + // TODO: these are board configuration values; G13 values for now. + unk_50: cfg.global_unk_54, + unk_52: 40, + unk_54: 0xffff, + // G13 unk_5e = 1 moved to +0x5a (a flag on G15) (inferred) + unk_5a: U32(1), + // +0x5e = max(HwDataB+0xa6c, 1) = 1. + cswitch_timer_multiplier: U32(1), + // Must be non-zero before work is submitted. + // Non-zero at boot has been accepted by the firmware on J516S. + command_submission_enabled: 1, + pending_submissions: AtomicU32::new(0), + // G13 values. TODO: the G15 defaults for +0x97c..+0x984 are not + // known. + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + // G13 unk_1102c_0.._c values; same group at +0x988 (inferred) + unk_1102c_0: 1, + unk_1102c_4: 1, + unk_1102c_8: 100, + unk_1102c_c: 1, + // +0x9a0..+0x9a8 + // TODO: with the rings in Fender SRAM the GPU and Fender must stay + // powered from firmware boot on, until an SRAM backup/restore exists. + // gpu.rs update_globals() keeps idle_off_delay_ms at the same value on + // every submit. + idle_off_delay_ms: AtomicU32::new(G15_KEEP_POWERED_MS), + fender_idle_off_delay_ms: G15_KEEP_POWERED_MS, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + // G13 values (+0x9b0/+0x9b4, set by setters) + cl_context_switch_timeout_ms: 40, + cl_kill_timeout_ms: 50, + // The manager leaves the FwUtil perf tunables (cfg.global_tab) and + // `soft_fault_settings` (+0xdf9) unwritten; the runtime layout sets them below. + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + if runtime { + Self::globals_runtime(raw, cfg, pwr, &tables)?; + } else if reference_ppm { + Self::globals_power_targets(raw, pwr, &tables); + } + Ok(()) + })) + } + + // The Globals power-interface targets (maximum power from the device tree, then the highest + // usable performance state scaled by 100, twice) and the performance-state cap. + fn globals_power_targets(raw: &mut raw::GlobalsG15V14_8_3, pwr: &hw::PwrConfig, tables: &PStateTables) { + let max_ps_scaled = 100 * tables.max; + raw.power_if_target = Array::new([pwr.max_power_mw, max_ps_scaled, max_ps_scaled]); + raw.perf_state_cap = max_ps_scaled; + } + + // The runtime backend's Globals: DRAM rings with finite idle-off, so the idle policy + // (imported by DeviceControl 0x13) and smart idle-off are on and the device-tree + // idle-off delays apply; the power interface targets and the performance-state cap + // (maximum power from the device tree, pstates scaled by 100); the smart idle-off + // configuration (standby timer from the device tree); and the FwUtil tunables. + fn globals_runtime( + raw: &mut raw::GlobalsG15V14_8_3, + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, + tables: &PStateTables, + ) -> Result { + raw.relaxed_cl_kill_timeout = 3000; + raw.debug = 1; + raw.smart_idle_off_enable = 1; + raw.unk_50 = 0xffff; + // The runtime enables command submission through its own queue protocol. + raw.command_submission_enabled = 0; + Self::globals_power_targets(raw, pwr, tables); + + let mut idle = [0u32; 10]; + idle[0] = pwr.idle_off_standby_timer; + if let Some(curve) = cfg.unk_hws2_4 { + for (slot, k) in idle[1..9].iter_mut().zip(curve.iter()) { + *slot = k.to_bits(); + } + } + idle[9] = cfg.unk_hws2_24; + raw.smart_idle_off_cfg = Array::new(idle); + raw.ut_engagement = 1; + raw.clvr_engagement = 1; + raw.keepalive_perf_threshold_rd = 100; + raw.keepalive_off_threshold_rd = 100; + + raw.idle_off_delay_ms = AtomicU32::new(pwr.idle_off_delay_ms); + raw.fender_idle_off_delay_ms = pwr.fender_idle_off_delay_ms; + raw.fw_early_wake_timeout_ms = pwr.fw_early_wake_timeout_ms; + + // CDM backoff 4, then a u32 1 and the FwUtil perf tunables from +0x9c1. + raw.cdm_backoff_timeout = 4; + raw.unk_9bd = Array::new([1, 0, 0, 0]); + if let Some(tab) = cfg.global_tab { + let mut bytes = [0u8; 0x1c]; + bytes.get_mut(..tab.len()).ok_or(EINVAL)?.copy_from_slice(tab); + raw.fwutil_default_fab_pstate = Array::new([bytes[0], bytes[1]]); + raw.fwutil_timer_period = bytes[2]; + (*raw.unk_9c4)[..0x19].copy_from_slice(&bytes[3..0x1c]); + } + raw.soft_fault_settings = U32(3); + Ok(()) + } + + // Create the G15 (14.8.3) Globals structure. + fn globals(&mut self) -> Result> { + let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); + Ok(()) + }) + }) + } + + // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by + // the GPU manager. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); + let rp_2d0 = self.g15.runtime_pointers_2d0(); + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointersG15V14_8_3 { + // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 + // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. + stats <- { + let alloc = &mut *alloc; + try_init!(StatsG15V14_8_3 { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + hwdata_b: hwb, + + // BRN (erratum) table. + // TODO: size and contents unknown; one zeroed page (no errata). + brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, + // E7: 0x88 bytes, zeroed + unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, + // E5: 0x60 bytes, zeroed + unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, + // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). + // TODO: size/format unknown; one zeroed page. The firmware takes a + // GPU VA and a FW VA; both point at this kernel mapping for now. + uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointersG15V14_8_3 { + hwdata_b: inner.hwdata_b.gpu_pointer(), + brn_table: inner.brn_table.gpu_pointer(), + unkptr_e7: inner.unkptr_e7.gpu_pointer(), + + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + fwlog_buf: None, + __pad_200: Default::default(), + // FWLog enable: the firmware writes no log records while this is 0. + // Opt-in via asahi.g15_debug bit 43. + unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + unkptr_e5: inner.unkptr_e5.gpu_pointer(), + __pad_254: Default::default(), + // Optional pointer, left 0 + unkptr_2a8: U64(0), + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), + uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), + uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), + // TODO: board configuration words; +0x2d4 left 0. + unk_2d0: rp_2d0, + unk_2d4: 0, + __pad_2d8: Default::default(), + // +0x3b0: 0xff, then 0x90 zero bytes + cswitch_hist_ctl: 0xff, + unk_3b1: Default::default(), + hwdata_a: inner.hwdata_a.gpu_pointer(), + // TODO: 16 board configuration bytes; left 0. + unk_449: Default::default(), + __pad_459: Default::default(), + unk_46d: Default::default(), + __pad_48d: Default::default(), + }) + }, + ) + } + + // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the + // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle + // power-off state (+0x4), which the runtime's InitData starts at 1. + fn status_block_fill(raw: &mut raw::StatusBlock, g15: &G15Options) { + raw.flags.sched_callbacks = 1; + if g15.layout == G15Layout::Runtime { + raw.noise_suppression_idle_pwroff = 1; + } + } + + // Create the G15 P0 status block. + fn status_block(&mut self) -> Result> { + let g15 = self.g15; + let mut block = self.alloc.shared.new_default::()?; + block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); + Ok(block) + } + + // Build the G15 P1 power-controller block without allocating it. The fields keep the order + // and the formulas of the G13 13.5 Globals power fields (inferred). + fn power_ctl_block_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> impl Init + 'a { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = runtime || g15.reference_ppm; + + try_init!(raw::PowerCtlBlock { + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: pwr.avg_power_filter_tc_ms / period_ms, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + // G13 Globals unk_89bc..unk_89e0 order (inferred) + fast_die_temp_target: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_b0: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + // +0xc0/+0xc4 are board values at init (also mirrored at HwDataA + // +0x934/+0x93c, G13 unk_8e8/unk_8f0, which the G13 path leaves 0). + // TODO: real values unknown; default 0 as in the G13 HwDataA mirror. + unk_c0: 0, + unk_c4: 0, + // +0xc8..+0xd8, manager layout: the G13 unk_89e0 = 1 flag (on G15 a + // condition flag), then the PPM target and the PPM kp. The PPM controller + // normally receives them at runtime; the G13 Globals values keep it from + // starting with a zero target. + ppm_words: Array::new([1, pwr.max_power_mw, pwr.ppm_kp.to_bits(), 0, 0]), + // +0xec: 1 at init + se_engagement: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + if runtime { + // Runtime layout: no fast-die release temperature or +0xb0 offset (the + // T6030 device tree has no source for them). + raw.fast_die0_release_temp = 0; + raw.unk_b0 = 0; + } + if reference_ppm { + // 0 at +0xc8, then the enable flag, the PPM target and the PPM kp and ki*dt. + let ppm_ki_dt = pwr.ppm_ki * period_s; + raw.ppm_words = Array::new([ + 0, + 1, + pwr.max_power_mw, + pwr.ppm_kp.to_bits(), + ppm_ki_dt.to_bits(), + ]); + } + Ok(()) + }) + } + + // Create the G15 P1 power-controller block. + fn power_ctl_block(&mut self) -> Result> { + let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); + self.alloc + .shared + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) + } + + // Create the G15 P2 debug block. ktrace stays disabled, as on G13. + fn debug_block(&mut self) -> Result> { + self.alloc.shared.new_default::() + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// The three UatLevelInfo records of InitData: L1 ("page catalogue") entries are 8 with a + /// 39-bit IAS and 64 with the 42-bit G15 IAS (index mask 0x3f0_0000_0000). + pub(crate) fn uat_levels(cfg: &'static hw::HwConfig) -> [raw::UatLevelInfo; 3] { + let uat_l1_entries = 1usize << (cfg.uat_ias as usize - 36); + [ + Self::uat_level_info(cfg, 36, uat_l1_entries), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ] + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + { + let pwr = &self.dyncfg.pwr; + let tables = self.g15.tables(pwr)?; + dev_info!( + self.dev.as_ref(), + "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", + self.g15.cap.unwrap_or(0), + if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, + tables.max, + tables.len - 1, + tables.max_frequency_khz(pwr) / 1000 + ); + } + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + // 14.x: FwStatus lives inside the P0 status block + let fw_status = self.status_block()?; + let power_ctl_block = self.power_ctl_block()?; + let debug_block = self.debug_block()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitDataG15V14_8_3 { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + power_ctl_block, + debug_block, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitDataG15V14_8_3 { + ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + // InitData +0x28/+0x2c: u32 0, u32 1 + unk_28: 0, + unk_2c: 1, + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new(Self::uat_levels(cfg)), + // TODO: pmap descriptor +0x7c, meaning unknown; left 0. + unk_94: 0, + __pad_98: Default::default(), + debug_block: inner.debug_block.gpu_pointer(), + status_block: inner.fw_status.gpu_pointer(), + power_ctl_block: inner.power_ctl_block.gpu_pointer(), + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} + +/// G15 InitData contents built into plain memory, for a backend that places the firmware objects +/// itself (the M3 runtime backend, whose object layout is fixed). +pub(crate) struct G15Contents { + pub(crate) hwdata_a: KVBox, + pub(crate) hwdata_b: KVBox, + pub(crate) globals: KVBox, + pub(crate) power_ctl_block: KVBox, + pub(crate) status_block: KVBox, + pub(crate) uat_levels: [raw::UatLevelInfo; 3], + /// The published performance-state tables. + pub(crate) tables: PStateTables, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Build the G15 InitData contents with the same builders the manager backend uses, into + /// plain memory. Pointer fields stay zero; the caller relocates them. + pub(crate) fn g15_contents( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result { + let mut status_block = KVBox::init(pin_init::init_zeroed(), GFP_KERNEL)?; + Self::status_block_fill(&mut status_block, g15); + Ok(G15Contents { + hwdata_a: KVBox::init(Self::hwdata_a_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + hwdata_b: KVBox::init(Self::hwdata_b_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + globals: KVBox::init(Self::globals_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + power_ctl_block: KVBox::init(Self::power_ctl_block_init(cfg, dyncfg, g15), GFP_KERNEL)?, + status_block, + uat_levels: Self::uat_levels(cfg), + tables: g15.tables(&dyncfg.pwr)?, + }) + } +} + +/// The bytes of a firmware structure built by this module. +/// +/// The structures are plain `repr(C)` data, and every builder starts from a zeroed object, so +/// no byte is uninitialized. +pub(crate) fn raw_bytes(value: &T) -> &[u8] { + // SAFETY: `value` is a live, initialized `T` (see above); the slice covers exactly it and + // borrows it for its lifetime. + unsafe { core::slice::from_raw_parts((value as *const T).cast::(), size_of::()) } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs new file mode 100644 index 0000000000000..2fa384c910688 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -0,0 +1,1491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use core::mem::{offset_of, size_of}; + +use kernel::{ + c_str, device, + device::Core, + platform, + prelude::*, + time::{delay::fsleep, Delta}, +}; + +use crate::{ + float::F32, + fw::initdata::raw, + hw, initdata, + initdata::{G15Layout, G15Options}, + m3_config::Records, + m3_firmware::Firmware, + m3_init_layout as init, + m3_init_storage as storage, + m3_memory::Buffer, + m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3_soc::{IoMapping, PstateTable, Soc}, + t8122_hwdata, + t8122_start::Experiment, +}; + +// The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage +// coefficient fields from core_leak_coef[4] to the end of aux_leak_coef, which the G15 builder +// leaves zero, and nothing the builder fills. +const _: () = { + type A = raw::HwDataAG15V14_8_3; + assert!(offset_of!(A, core_leak_coef) == 0x417c); + assert!(offset_of!(A, sram_leak_coef) == 0x419c); + assert!(offset_of!(A, aux_leak_coef) == 0x41bc); + assert!(t8122_hwdata::OFFSET == offset_of!(A, core_leak_coef) + 4 * size_of::()); + assert!( + t8122_hwdata::OFFSET + t8122_hwdata::SIZE + == offset_of!(A, aux_leak_coef) + size_of::() + ); + assert!(t8122_hwdata::OFFSET + t8122_hwdata::SIZE <= size_of::()); +}; + +/// Owner indices (`m3_init_storage`) of the objects the typed builders generate. +pub(crate) const INITDATA: usize = storage::ROOT; +pub(crate) const RUNTIME_POINTERS: usize = storage::RUNTIME_POINTERS; +pub(crate) const HWDATA: usize = storage::HARDWARE_DATA; +pub(crate) const GLOBALS: usize = storage::GLOBALS; +pub(crate) const GLOBALS_POWER: usize = storage::GLOBALS_POWER; +pub(crate) const STATUS: usize = storage::CONTROL_REGION; +pub(crate) const DEBUG: usize = storage::RUNTIME_FLAGS; +/// HwDataA follows HwDataB inside the HwData object. +pub(crate) const HWDATA_A: usize = 0x4580; + +/// Offset of HwDataB's IO-mapping table and the size of one entry. +const IO_MAPPINGS: usize = offset_of!(raw::HwDataBG15V14_8_3, io_mappings); +const IO_MAPPING_SIZE: usize = size_of::(); +/// Number of IO-mapping slots of HwDataB. +const IO_MAPPING_SLOTS: usize = 31; + +/// The T6030 firmware IO mappings of the runtime backend (HwDataB slot, physical address, total +/// size, element size, writable). The runtime maps IO with its own IO-mapping table +/// (`m3_init_storage::T6030_IOMAPS`), which must cover every entry (checked when the contents +/// are built). The physical addresses are T6030 SoC register blocks, the same ones the G15 manager +/// maps (`hw::t6030`). +pub(crate) const T6030_IO_MAPPINGS: [IoMapping; 15] = [ + (0, 0x2_90d0_0000, 0x14_4000, 0x14_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x3_5101_4000, 0x4000, 0x4000, true), // AIC software interrupts + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (7, 0x3_502b_c000, 0x1000, 0x1000, false), // analog temperature sensors + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (11, 0x2_2000_0000, 0x12_a000, 0x9_5000, true), // memory cache, two instances + (12, 0x3_5104_c000, 0x4000, 0x4000, false), // AIC banked registers + (18, 0x3_503d_0000, 0x4000, 0x4000, true), // telemetry dashboard + (19, 0x3_503c_0000, 0x4000, 0x4000, false), // telemetry dashboard (read) + (20, 0x3_503d_8000, 0x4000, 0x4000, true), // telemetry dashboard (config) + (23, 0x2_9300_0000, 0x40_0000, 0x40_0000, true), // AFR registers + (25, 0x3_0945_c000, 1, 1, true), // ANE doorbell + (26, 0x3_5028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e5_c000, 0x4000, 0x4000, false), // GFX clock generator +]; + +/// The T8122 firmware IO mappings of the runtime backend, in the same form: the blocks of +/// `hw::t8122`, with the exact register, total size and element size the G15G firmware takes in +/// each slot. The AIC timer entry is the T6030 one, the same register on every SoC. +/// `fill_io_mappings` checks every entry against the runtime's IO maps of the SoC +/// ([`T8122_IOMAPS`]) before writing it. +pub(crate) const T8122_IO_MAPPINGS: [IoMapping; 12] = [ + (0, 0x2_90d0_0000, 0x10_4000, 0x10_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x2_d101_4048, 1, 1, true), // AIC software interrupt register + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (10, 0x2_90d0_d000, 0x1000, 0x1000, true), // GM GIFAF registers + (11, 0x2_2000_0000, 0xa_a000, 0x5_5000, true), // memory cache, two instances + (18, 0x2_d03d_0000, 0x1000, 0x1000, true), // telemetry dashboard + (19, 0x2_d03c_0000, 0x2000, 0x2000, false), // telemetry dashboard (read) + (25, 0x3_1145_c000, 1, 1, true), // ANE doorbell + (26, 0x2_d028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e1_c000, 0x4000, 0x4000, false), // GPU clock generator +]; + +/// The T8122 runtime's IO maps (`m3_soc::T8122.iomaps`): its IO mappings, laid out as the T6030 +/// ones are. +pub(crate) static T8122_IOMAPS: [storage::IoMap; 12] = + storage::pack_iomaps(&T8122_IO_MAPPINGS, storage::IOMAP_BASE); + +/// Whether every entry of `mappings` is a whole number of elements in the HwConfig block of the +/// same slot, with the same writability: the check `fill_io_mappings` makes when it builds the +/// InitData. +const fn same_blocks(mappings: &[IoMapping], cfg: &hw::HwConfig) -> bool { + let mut i = 0; + while i < mappings.len() { + let (slot, phys, total, element, writable) = mappings[i]; + if slot >= cfg.io_mappings.len() || element == 0 || total % element != 0 { + return false; + } + match cfg.io_mappings[slot] { + Some(block) + if block.base as u64 & !0x3fff == phys & !0x3fff && block.writable == writable => {} + _ => return false, + } + i += 1; + } + true +} + +const _: () = assert!(same_blocks(&T8122_IO_MAPPINGS, &hw::t8122::HWCONFIG_T8122)); + +// The T6030 IO maps are the packed IO maps of its IO mappings, the layout the T8122 start +// experiment derives its own from (`storage::pack_iomaps`). +const _: () = assert!(storage::same_iomaps( + &storage::pack_iomaps(&T6030_IO_MAPPINGS, storage::IOMAP_BASE), + &storage::T6030_IOMAPS +)); + +/// Highest performance state device-tree InitData may use without the driver's thermal limit +/// (`asahi.m3_thermal=off`), and the runtime cap the thermal limit starts at and falls back to +/// (the "safe cap"). The InitData's fast-die temperature controller words stay 0, so the +/// firmware does not limit the die temperature by itself. +pub(crate) const ADT_MAX_PSTATE_LIMIT: u32 = 5; + +/// The performance states the runtime backend lets the firmware use. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PstatePolicy { + /// Highest entry of the published table. + pub(crate) table_max: u32, + /// Frequency of the highest entry of the published table, in MHz. + pub(crate) table_max_mhz: u32, + /// The ceiling: the highest entry the firmware is given (the InitData's highest-state + /// words). A reported state above it marks the GPU failed. + pub(crate) max: u32, + /// The runtime cap at boot (the Globals cap words): the highest entry the firmware may use + /// until the thermal limit changes it. Equal to `max` unless the thermal limit is on or + /// holding, which start here and never go below it. + pub(crate) safe: u32, + /// The entry requested at boot, at most `safe`. + pub(crate) boot: u32, + /// Frequency of the ceiling entry, in MHz. + pub(crate) max_mhz: u32, + /// Frequency of the `safe` entry, in MHz. + pub(crate) safe_mhz: u32, + /// Frequency of the boot entry, in MHz. + pub(crate) boot_mhz: u32, + /// Frequency of every entry of the published table, in MHz (entry 0 is the off state). + pub(crate) freqs: [u32; 16], + /// `asahi.m3_thermal`. + pub(crate) thermal: ThermalMode, +} + +impl PstatePolicy { + /// The highest frequency the GPU may run at this boot, for userspace: the ceiling's when + /// the thermal limit is on, the runtime cap's otherwise. + pub(crate) fn reported_max_mhz(&self) -> u32 { + match self.thermal { + ThermalMode::On => self.max_mhz, + ThermalMode::Off | ThermalMode::Hold => self.safe_mhz, + } + } + + /// Clamp the `asahi.m3_max_pstate` and `asahi.m3_boot_pstate` settings to the published table + /// of `hwdata` (the generated HwData object image), and to `ceiling` when given (the T8122 + /// start experiment's `asahi.t8122_pstate_cap`). + fn new(dev: &device::Device, hwdata: &[u8], ceiling: Option) -> Result { + let hwb_max = offset_of!(raw::HwDataBG15V14_8_3, max_pstate); + let hwb_freq = offset_of!(raw::HwDataBG15V14_8_3, frequencies); + let table_max = read_u32(hwdata, hwb_max)?; + if table_max == 0 || table_max >= 16 { + return Err(EINVAL); + } + let freq = |i: u32| read_u32(hwdata, hwb_freq + 4 * i as usize); + // Unset: the thermal limit is on. + let thermal = m3_params::thermal_param().unwrap_or(ThermalMode::On); + // Unset: the whole table when the thermal limit bounds the runtime cap, the default cap + // otherwise. + let requested = m3_params::max_pstate_param().unwrap_or(match thermal { + ThermalMode::Hold | ThermalMode::On => u64::from(table_max), + ThermalMode::Off => m3_params::MAX_PSTATE_DEFAULT, + }); + let mut max = requested.clamp(1, u64::from(table_max)) as u32; + if u64::from(max) != requested { + dev_info!( + dev, + "M3: asahi.m3_max_pstate={} clamped to {} (the table has states 1..={})\n", + requested, + max, + table_max + ); + } + if thermal == ThermalMode::Off && max > ADT_MAX_PSTATE_LIMIT { + dev_info!( + dev, + "M3: performance cap {} lowered to {}: device-tree InitData leaves the fast-die temperature controller off and the thermal limit is off (asahi.m3_thermal=off)\n", + max, + ADT_MAX_PSTATE_LIMIT + ); + max = ADT_MAX_PSTATE_LIMIT; + } + if let Some(ceiling) = ceiling { + if max > ceiling { + dev_info!( + dev, + "M3 G15G start: performance cap {} lowered to {} (asahi.t8122_pstate_cap)\n", + max, + ceiling + ); + max = ceiling; + } + } + // Without the thermal limit the runtime cap is the ceiling; with it, the runtime cap + // starts at (and never goes below) the cap used without it. + let safe = match thermal { + ThermalMode::Off => max, + ThermalMode::Hold | ThermalMode::On => max.min(ADT_MAX_PSTATE_LIMIT), + }; + let boot = m3_params::boot_pstate_param() + .unwrap_or(u64::from(safe)) + .clamp(1, u64::from(safe)) as u32; + let mut freqs = [0u32; 16]; + for (i, f) in freqs.iter_mut().enumerate().take(table_max as usize + 1) { + *f = freq(i as u32)?; + } + Ok(PstatePolicy { + table_max, + table_max_mhz: freqs[table_max as usize], + max, + safe, + boot, + max_mhz: freqs[max as usize], + safe_mhz: freqs[safe as usize], + boot_mhz: freqs[boot as usize], + freqs, + thermal, + }) + } +} + +/// The runtime backend's InitData contents: the generated object images and the +/// performance-state policy. +pub(crate) struct Contents { + /// Generated images, per owner; None for owners that stay zero. Only the three image + /// owners' images are uploaded; the records' are checked. + images: KVec>>, + /// The performance states the firmware may use. + pub(crate) pstates: PstatePolicy, + /// The runtime's IO maps of the SoC (`Soc::iomaps`), which cover the InitData IO mappings. + pub(crate) iomaps: &'static [storage::IoMap], + /// The accepted performance-state table (`Soc::pstates`): states above the off state, and + /// the frequency of the highest one in MHz. + pub(crate) table: (u32, u32), + /// The HwDataB slots whose IO maps the firmware gets read-only, as a bit mask. 0 (every IO + /// map read-write) except in the T8122 start experiment. + pub(crate) read_only_slots: u32, + /// The HwData object's placement and mapping (`Soc::hwdata_object`). + pub(crate) hwdata_object: Option, +} + +fn read_u32(bytes: &[u8], offset: usize) -> Result { + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + Ok(u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?)) +} + +fn write(bytes: &mut [u8], offset: usize, data: &[u8]) -> Result { + bytes + .get_mut(offset..offset + data.len()) + .ok_or(EINVAL)? + .copy_from_slice(data); + Ok(()) +} + +impl Contents { + /// Build the InitData contents from this board's device tree (`asahi.m3_initdata`), before + /// any GPU register is touched. + /// + /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 + /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an + /// armed T6031 (`t6031_start`). + pub(crate) fn select( + pdev: &platform::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + ) -> Result { + let dev = pdev.as_ref(); + let param = m3_params::initdata_param(); + check_version(dev, firmware, experiment, t6031)?; + let iomaps = match experiment { + Some(e) => e.iomaps(), + None => match t6031 { + Some(e) => e.iomaps(), + None => soc.iomaps.ok_or(ENODEV)?, + }, + }; + storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; + let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + let pstates = PstatePolicy::new( + dev, + hwdata, + experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), + )?; + let table = match soc.pstates { + PstateTable::Fixed { states, top_mhz } => (states, top_mhz), + PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + dev_err!( + dev, + "M3: the device tree's operating points give no performance-state table\n" + ); + EINVAL + })?, + }; + if (pstates.table_max, pstates.table_max_mhz) != table { + dev_err!( + dev, + "M3: performance-state table 1..={} up to {} MHz, the {} table is 1..={} up to {} MHz\n", + pstates.table_max, + pstates.table_max_mhz, + soc.name, + table.0, + table.1 + ); + return Err(EINVAL); + } + // The published states never go above the device tree's highest operating point. + if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if 1000 * pstates.max_mhz > dt_khz { + dev_err!( + dev, + "M3: performance cap {} MHz is above the device tree's highest operating point ({} kHz)\n", + pstates.max_mhz, + dt_khz + ); + return Err(EINVAL); + } + } + dev_info!( + dev, + "M3: InitData contents: generated from the device tree (asahi.m3_initdata={}); performance states 1..={}, cap {} ({} MHz), boot request {} ({} MHz); thermal limit {}, firmware highest state {} ({} MHz)\n", + match param { + InitDataParam::Auto => "auto", + InitDataParam::Adt => "adt", + }, + pstates.table_max, + pstates.safe, + pstates.safe_mhz, + pstates.boot, + pstates.boot_mhz, + match pstates.thermal { + ThermalMode::Off => "off", + ThermalMode::Hold => "holding", + ThermalMode::On => "on", + }, + pstates.max, + pstates.max_mhz + ); + let read_only_slots = experiment + .map(|e| e.read_only_slots()) + .or_else(|| t6031.map(|e| e.read_only_slots())) + .unwrap_or(0); + let hwdata_object = match t6031 { + Some(e) => e.hwdata_object(), + None => soc.hwdata_object, + }; + Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) + } + + /// Check the contents before they are uploaded. + pub(crate) fn check_for_upload(&self) -> Result { + if self.images.len() == storage::COUNT { + Ok(()) + } else { + Err(EINVAL) + } + } + + /// The initial contents of owner `index`, allocated as `a`: its generated image, or nothing + /// (zeroed) for owners without one. + pub(crate) fn initial<'a>(&'a self, index: usize, a: &storage::Allocation) -> Result<&'a [u8]> { + if a.initial == storage::Initial::Zero { + return Ok(&[]); + } + let image = self.images.get(index).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + if image.len() != a.size { + return Err(EINVAL); + } + Ok(image) + } +} + +/// The fixed allocation of owner `index`, as a firmware region moved by `delta`. +fn allocation_region(index: usize, delta: u64) -> Result { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + init::Region::new(a.address.checked_add(delta).ok_or(EINVAL)?, a.size).map_err(|_| EINVAL) +} + +/// The records the runtime constructs, encoded for its fixed allocation table moved by `delta`. +pub(crate) fn allocation_records(delta: u64) -> Result { + Records::encode(|index| allocation_region(index, delta)) +} + +/// The version word of the constructed InitData root. +pub(crate) fn constructed_version() -> Result { + let records = allocation_records(0)?; + Ok(u64::from_le_bytes(records.root[..8].try_into().map_err(|_| EINVAL)?)) +} + +/// Refuse to go on when the InitData root the runtime constructs carries another version than +/// the loaded firmware expects. The T8122 start experiment gives the firmware the version of +/// `asahi.t8122_initdata_version` instead (`m3_config::Config::new` writes it into the root). +fn check_version( + dev: &device::Device, + firmware: &Firmware, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> Result { + let version = constructed_version()?; + if version != firmware.initdata_magic && experiment.is_some() { + dev_warn!( + dev, + "M3 G15G start: InitData version {:#x} (asahi.t8122_initdata_version) replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic && t6031.is_some() { + dev_warn!( + dev, + "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: the constructed InitData version {:#x} is not the one the loaded firmware expects ({:#x})\n", + version, + firmware.initdata_magic + ); + return Err(ENODEV); + } + Ok(()) +} + +/// Whether the runtime probe stops before the GPU coprocessor starts (`asahi.g15_debug` bit +/// 53). Logs the stop. +pub(crate) fn stop_before_asc(dev: &device::Device) -> bool { + let stop = m3_params::g15_debug(G15Debug::StopBeforeAsc); + if stop { + dev_info!( + dev, + "M3: asahi.g15_debug bit 53: stopping before the GPU coprocessor starts; the firmware was not started\n" + ); + } + stop +} + +/// The GPU identity the runtime backend admits for `soc` (its SGX ID words, checked in +/// `m3_device` before the firmware starts): on T6030, G15, variant S, revision B1, one die with +/// two clusters of ten core slots. +fn soc_identity( + soc: &Soc, + cfg: &'static hw::HwConfig, + rev_id: Option, +) -> hw::GpuIdConfig { + hw::GpuIdConfig { + gpu_gen: hw::GpuGen::G15, + gpu_variant: soc.gpu_variant, + usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy, + gpu_rev: soc.gpu_revision, + gpu_rev_id: rev_id.unwrap_or(soc.gpu_revision_id), + num_dies: cfg.num_dies, + num_clusters: cfg.max_num_clusters, + num_cores: cfg.max_num_cores, + num_frags: cfg.max_num_frags, + num_gps: cfg.max_num_gps, + total_active_cores: 0, + core_masks: KVec::new(), + core_masks_packed: KVec::new(), + } +} + +pub(crate) const BOARD_DATA: [usize; 3] = [HWDATA, GLOBALS, GLOBALS_POWER]; + +/// The pointers of the constructed records that the typed firmware structures place: record +/// owner, offset, target owner, offset into the target. +const RECORD_POINTERS: [(usize, usize, usize, usize); 7] = { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + [ + (INITDATA, offset_of!(InitData, runtime_pointers), RUNTIME_POINTERS, 0), + (INITDATA, offset_of!(InitData, globals), GLOBALS, 0), + (INITDATA, offset_of!(InitData, debug_block), DEBUG, 0), + (INITDATA, offset_of!(InitData, status_block), STATUS, 0), + (INITDATA, offset_of!(InitData, power_ctl_block), GLOBALS_POWER, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_b), HWDATA, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_a), HWDATA, HWDATA_A), + ] +}; + +/// The bytes of a constructed record, by owner. +pub(crate) fn record_bytes(records: &Records, owner: usize) -> Option<&[u8]> { + match owner { + INITDATA => Some(&records.root[..]), + RUNTIME_POINTERS => Some(&records.runtime[..]), + STATUS => Some(&records.control[..]), + _ => None, + } +} + +fn check_layout() -> Result { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let records = allocation_records(0)?; + let pointer = |owner: usize, offset: usize| -> Option { + let bytes = record_bytes(&records, owner)?.get(offset..offset + 8)?; + Some(u64::from_le_bytes(bytes.try_into().ok()?)) + }; + let mut ok = true; + for (owner, offset, target, at) in RECORD_POINTERS { + let a = storage::allocation(target).map_err(|_| EINVAL)?; + ok &= at < a.size && pointer(owner, offset) == Some(a.address + at as u64); + } + for index in 0..storage::COUNT { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + let board = BOARD_DATA.contains(&index); + ok &= board == (a.initial != storage::Initial::Zero); + } + let size = |index: usize| storage::allocation(index).map_or(0, |a| a.size); + let ok = ok + && size(INITDATA) >= size_of::() + && size(RUNTIME_POINTERS) >= offset_of!(RuntimePointers, hwdata_a) + 8 + && HWDATA_A >= size_of::() + && size(HWDATA) >= HWDATA_A + size_of::() + && size(GLOBALS) >= size_of::() + && size(GLOBALS_POWER) >= size_of::() + && size(STATUS) >= size_of::() + && size(DEBUG) >= size_of::(); + if ok { Ok(()) } else { Err(EINVAL) } +} + +/// Fill HwDataB's IO-mapping table (the runtime fills the virtual addresses when it maps them), +/// checking every entry against the runtime's IO mapping of the same slot (`iomaps`). +fn fill_io_mappings( + dev: &device::Device, + cfg: &'static hw::HwConfig, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + mappings: &[IoMapping], + iomaps: &[storage::IoMap], + hwdata: &mut [u8], +) -> Result { + for (i, &(slot, phys, total, element, writable)) in mappings.iter().enumerate() { + if slot >= 31 || mappings[..i].iter().any(|&(other, ..)| other == slot) { + return Err(EINVAL); + } + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let virt = entry + offset_of!(raw::IOMapping, virt_addr); + // The same register block as the manager's table. + let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { + let base = match (experiment, t6031) { + (Some(e), _) => e.io_block(slot, m.base as u64), + (None, Some(e)) => e.io_block(slot, m.base as u64), + (None, None) => m.base as u64, + }; + base & !0x3fff == phys & !0x3fff && m.writable == writable + }); + let covered = iomaps.iter().any(|io| { + io.slot == slot + && io.pointer_field() == Ok(virt) + && io.covers(phys, total) + }); + if !same_block || !covered || element == 0 || total % element != 0 { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) does not match the object layout\n", slot, phys); + return Err(EINVAL); + } + write(hwdata, entry + offset_of!(raw::IOMapping, phys_addr), &phys.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, total_size), &total.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, element_size), &element.to_le_bytes())?; + write( + hwdata, + entry + offset_of!(raw::IOMapping, readwrite), + &u64::from(writable).to_le_bytes(), + )?; + } + // Every IO mapping the runtime makes must belong to one of the entries above. + for io in iomaps.iter() { + if !mappings.iter().any(|&(slot, ..)| slot == io.slot) { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) has no device-tree InitData entry\n", io.slot, io.physical); + return Err(EINVAL); + } + } + Ok(()) +} + +pub(crate) fn check_upload( + dev: &device::Device, + firmware: &Firmware, + iomaps: &[storage::IoMap], + objects: &mut [Buffer], +) -> Result { + let mut bad = 0u32; + let version = objects.get_mut(INITDATA).ok_or(EINVAL)?.read_u64(0)?; + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: InitData self-check: version {:#x}, the loaded firmware expects {:#x}\n", + version, + firmware.initdata_magic + ); + bad += 1; + } + for (owner, offset, target, at) in RECORD_POINTERS { + let expected = objects.get(target).ok_or(EINVAL)?.va() + at as u64; + let value = objects.get_mut(owner).ok_or(EINVAL)?.read_u64(offset)?; + if value != expected { + dev_err!( + dev, + "M3: InitData self-check: owner {} +{:#x} holds {:#x}, not owner {} +{:#x} ({:#x})\n", + owner, + offset, + value, + target, + at, + expected + ); + bad += 1; + } + } + let hwdata = objects.get_mut(HWDATA).ok_or(EINVAL)?; + for slot in 0..IO_MAPPING_SLOTS { + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let phys = hwdata.read_u64(entry + offset_of!(raw::IOMapping, phys_addr))?; + let virt = hwdata.read_u64(entry + offset_of!(raw::IOMapping, virt_addr))?; + let total = u64::from(hwdata.read_u32(entry + offset_of!(raw::IOMapping, total_size))?); + let ok = match iomaps.iter().find(|io| io.slot == slot) { + Some(io) => { + phys == io.physical + io.offset as u64 + && virt == io.address + io.offset as u64 + && total != 0 + && phys + total <= io.physical + io.size as u64 + } + None => phys == 0 && virt == 0, + }; + if !ok { + dev_err!( + dev, + "M3: InitData self-check: IO mapping slot {} describes {:#x}+{:#x} at {:#x}, which the runtime does not map there\n", + slot, + phys, + total, + virt + ); + bad += 1; + } + } + let region = hwdata.read_u64(storage::GPU_REGION_PHYSICAL)?; + if region != firmware.resources.regions[0].base { + dev_err!( + dev, + "M3: InitData self-check: GPU region {:#x}, the resources admitted {:#x}\n", + region, + firmware.resources.regions[0].base + ); + bad += 1; + } + if bad != 0 { + dev_err!(dev, "M3: InitData self-check failed ({} findings); not publishing the InitData\n", bad); + return Err(EINVAL); + } + dev_info!( + dev, + "M3: InitData self-check passed: version {:#x}, {} record pointers, {} IO mappings, GPU region {:#x}\n", + version, + RECORD_POINTERS.len(), + iomaps.len(), + region + ); + Ok(()) +} + +/// The whole part of a positive `F32`, for logs. +fn whole(v: F32) -> u32 { + let bits = v.to_bits(); + match ((bits >> 23) & 0xff) as i32 - 127 { + e if e < 0 => 0, + e if e > 31 => u32::MAX, + e if e > 23 => ((bits & 0x7f_ffff) | 0x80_0000) << (e - 23), + e => ((bits & 0x7f_ffff) | 0x80_0000) >> (23 - e), + } +} + +/// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported +/// them. Returns them when the boot loader's switch is set. With the switch set and no two +/// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. +fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { + let chosen = kernel::of::chosen(); + let switch = chosen + .as_ref() + .and_then(|c| c.get_property::>(lf.switch).ok()) + .is_some_and(|v| v.as_slice() == b"1\0"); + let values: Option> = chosen.as_ref().and_then(|c| c.get_property(lf.values).ok()); + // Positive, finite, nonzero. + let valid = |v: &F32| { + let b = v.to_bits(); + b != 0 && b >> 31 == 0 && (b >> 23) & 0xff != 0xff + }; + let pair = values + .as_ref() + .filter(|v| v.len() == 2 && v.iter().all(valid)) + .map(|v| [v[0], v[1]]); + let names = (lf.values.to_str().unwrap_or("?"), lf.switch.to_str().unwrap_or("?")); + match (pair, switch) { + (Some([a, b]), true) => { + dev_info!( + dev, + "M3: GPU leakage from the boot loader's fuse values: {} and {} (/chosen/{}, {} set)\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(Some([a, b])) + } + (Some([a, b]), false) => { + dev_info!( + dev, + "M3: the boot loader's GPU leakage fuse values are {} and {} (/chosen/{}); not used: {} is not set\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(None) + } + (None, true) => { + dev_err!( + dev, + "M3: {} is set, but /chosen/{} does not hold two valid values; GPU startup disabled\n", + names.1, + names.0 + ); + Err(EINVAL) + } + (None, false) => { + if values.is_some() { + dev_warn!(dev, "M3: /chosen/{} is malformed; ignored\n", names.0); + } + Ok(None) + } + } +} + +/// A zeroed image of owner `index`. +fn zeroed(index: usize) -> Result> { + let size = storage::allocation(index).map_err(|_| EINVAL)?.size; + Ok(KVVec::from_elem(0u8, size, GFP_KERNEL)?) +} + +/// Build the images of the generated owners from the device tree. +fn build_images( + dev: &device::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + iomaps: &[storage::IoMap], +) -> Result>>> { + check_layout().inspect_err(|_| { + dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") + })?; + let cfg: &'static hw::HwConfig = match t6031 { + Some(e) => e.hwcfg(), + None => soc.hwcfg.ok_or(ENODEV)?, + }; + let io_mappings = match experiment { + Some(e) => e.io_mappings(), + None => match t6031 { + Some(e) => e.io_mappings(), + None => soc.io_mappings.ok_or(ENODEV)?, + }, + }; + let runtime_hwdata_b = match experiment { + Some(e) => e.hwdata_b(), + None => match t6031 { + Some(e) => e.hwdata_b(), + None => soc.hwdata_b.ok_or(ENODEV)?, + }, + }; + let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { + dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) + })?; + // The boot loader's decoded leakage fuse, when its switch makes it this boot's leakage: value + // 1 is the core leakage coefficient (the boot loader then writes it as apple,core-leak-coef + // too), value 2 goes to HwDataA's third leakage table below. + let fuse = match soc.leak_fuse { + Some(lf) => leak_fuse(dev, &lf)?, + None => None, + }; + if let Some([core, _]) = fuse { + match pwr.core_leak_coef.first_mut() { + Some(c) if c.to_bits() != core.to_bits() => { + dev_info!( + dev, + "M3: core leakage {} replaces the device tree's {}\n", + whole(core), + whole(*c) + ); + *c = core; + } + Some(_) => {} + None => return Err(EINVAL), + } + } + // A SoC whose power model is still a stand-in caps the power target: every operating point's + // power is scaled by the same factor, so the relative powers stay the boot loader's. + let power_cap_mw = match t6031 { + Some(e) => Some(e.power_cap_mw()), + None => soc.power_target_cap_mw, + }; + if let Some(cap) = power_cap_mw { + let max = pwr.max_power_mw; + if max > cap { + for ps in pwr.perf_states.iter_mut() { + ps.pwr_mw = (u64::from(ps.pwr_mw) * u64::from(cap) / u64::from(max)) as u32; + } + pwr.max_power_mw = pwr.perf_states.iter().map(|ps| ps.pwr_mw).max().unwrap_or(cap); + dev_info!( + dev, + "M3: GPU power target capped at {} mW (the device tree's operating points reach {} mW)\n", + pwr.max_power_mw, + max + ); + } + } + let node = dev.of_node().ok_or(ENODEV)?; + let dyncfg = hw::DynConfig { + uat_ttb_base: firmware.resources.regions[0].base, + id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + pwr, + firmware_version: node + .get_property::>(c_str!("apple,firmware-version")) + .unwrap_or_default(), + hw_data_a: KVVec::new(), + hw_data_b: KVVec::new(), + hw_globals: KVVec::new(), + }; + let g15 = G15Options { + layout: G15Layout::Runtime, + split_pstates: true, + // The cap is applied to the uploaded objects, for either source (`publish_pstates`). + cap: None, + timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + reference_ppm: true, + runtime_hwdata_b: Some(runtime_hwdata_b), + per_cluster_voltages: soc.per_cluster_voltages, + }; + let c = initdata::InitDataBuilderG15V14_8_3::g15_contents(cfg, &dyncfg, &g15).inspect_err(|e| { + dev_err!(dev, "M3: cannot build InitData from the device tree ({:?})\n", e) + })?; + + let mut images = KVec::new(); + for _ in 0..storage::COUNT { + images.push(None, GFP_KERNEL)?; + } + + // InitData: the magic of the identified firmware, the host-mapped allocations flag and the + // UAT description; the pointers are the constructed record's. + type InitData = raw::InitDataG15V14_8_3<'static>; + let mut root = zeroed(INITDATA)?; + write(&mut root, offset_of!(InitData, ver_info), &firmware.initdata_magic.to_le_bytes())?; + write(&mut root, offset_of!(InitData, unk_2c), &1u32.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_size), &0x4000u16.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_bits), &[14, 3])?; + for (i, level) in c.uat_levels.iter().enumerate() { + let at = offset_of!(InitData, uat_level_info) + i * size_of::(); + write(&mut root, at, initdata::raw_bytes(level))?; + } + images[INITDATA] = Some(root); + + // RuntimePointers: only its configuration words; everything else is a pointer. + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let mut rt = zeroed(RUNTIME_POINTERS)?; + write(&mut rt, offset_of!(RuntimePointers, unk_2d0), &g15.runtime_pointers_2d0().to_le_bytes())?; + write(&mut rt, offset_of!(RuntimePointers, cswitch_hist_ctl), &[0xff])?; + images[RUNTIME_POINTERS] = Some(rt); + + // HwData: HwDataB, then HwDataA. + let mut hwdata = zeroed(HWDATA)?; + write(&mut hwdata, 0, initdata::raw_bytes(&*c.hwdata_b))?; + // The T8122 start experiment's unit masks (`asahi.t8122_unit_mask_a`/`_b`). + if let Some(e) = experiment { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } else if let Some(e) = t6031 { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } + write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; + // The system counter value at InitData creation, the base of the firmware's first power and + // energy interval. + let now: u64; + // SAFETY: reading the architectural counter has no side effects and is allowed at EL1/EL2. + unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) }; + write( + &mut hwdata, + HWDATA_A + offset_of!(raw::HwDataAG15V14_8_3, init_timestamp), + &now.to_le_bytes(), + )?; + // The SoC's MTR sensor masks, where its table gives them. + if let Some(m) = soc.mtr_masks { + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &m.fast_die.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &m.alarm.to_le_bytes())?; + dev_info!( + dev, + "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", + m.fast_die, + m.alarm + ); + } else if let Some(e) = t6031 { + let (fast, alarm) = e.mtr_masks(dev)?; + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; + dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); + } + // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first + // table holds the core coefficient, from the builder). + if let Some([_, second]) = fuse { + type A = raw::HwDataAG15V14_8_3; + let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); + write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; + } + // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies + // the T8122 words only when asahi.t6031_fw_words=t8122. + let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { + Some(e) if e.fw_words() => ( + &crate::m3_soc::T8122_HWDATA_WORDS, + &crate::m3_soc::T8122_GLOBALS_WORDS, + ), + Some(_) => (&[], &[]), + None => (soc.hwdata_words, soc.globals_words), + }; + for &(at, value) in hwdata_words { + write(&mut hwdata, at, &value.to_le_bytes())?; + } + // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 + // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + && cfg.gpu_gen == hw::GpuGen::G15 + { + if m3_params::t8122_hwdata_418c() { + let a = hwdata + .get_mut(HWDATA_A..HWDATA_A + size_of::()) + .ok_or(EINVAL)?; + t8122_hwdata::write(a).map_err(|()| EINVAL)?; + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db: wrote the 20 words (asahi.t8122_hwdata_418c=0 leaves them 0)\n" + ); + } else { + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db left 0 (asahi.t8122_hwdata_418c=0)\n" + ); + } + } + // On T8122, log and require the two HwDataB words the firmware's power management depends + // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. + if core::ptr::eq(soc, &crate::m3_soc::T8122) { + type B = raw::HwDataBG15V14_8_3; + let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; + let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; + dev_info!(dev, "M3: HwDataB +0xa38 {:#x}, +0xa40 {:#x}\n", a38, a40); + if a38 == 0 || a40 == 0 { + dev_err!( + dev, + "M3: HwDataB +0xa38 or +0xa40 is 0; the firmware would not power the GPU cores\n" + ); + return Err(EINVAL); + } + } + fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + images[HWDATA] = Some(hwdata); + + let mut globals = zeroed(GLOBALS)?; + write(&mut globals, 0, initdata::raw_bytes(&*c.globals))?; + // The gate of the firmware's frequency-feedback cap, per SoC. + let (ut, given) = m3_params::ut_engagement(soc); + write(&mut globals, offset_of!(raw::GlobalsG15V14_8_3, ut_engagement), &ut.to_le_bytes())?; + for &(at, value) in globals_words { + write(&mut globals, at, &value.to_le_bytes())?; + } + if !hwdata_words.is_empty() || !globals_words.is_empty() { + dev_info!( + dev, + "M3: {} HwData and {} Globals words set from the {} table\n", + hwdata_words.len(), + globals_words.len(), + soc.name + ); + } + if ut != 1 || given { + dev_info!( + dev, + "M3: Globals ut_engagement={}{}\n", + ut, + if given { " (asahi.m3_ut_engagement)" } else { "" } + ); + } + if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + .map_err(|_|EINVAL)? { + dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); + } + images[GLOBALS] = Some(globals); + + let mut power = zeroed(GLOBALS_POWER)?; + write(&mut power, 0, initdata::raw_bytes(&*c.power_ctl_block))?; + images[GLOBALS_POWER] = Some(power); + + let mut status = zeroed(STATUS)?; + write(&mut status, 0, initdata::raw_bytes(&*c.status_block))?; + images[STATUS] = Some(status); + + let top = c.tables.primary_state(&dyncfg.pwr, c.tables.len - 1); + dev_info!( + dev, + "M3: device-tree InitData: {} performance states in two voltage-sorted tables from {} device-tree states, top {} MHz at {} mV, SRAM floor {} mV, maximum power {} mW\n", + c.tables.len - 1, + dyncfg.pwr.perf_states.len() - 1, + top.freq_hz / 1_000_000, + top.max_volt_mv(), + dyncfg.pwr.min_sram_microvolt.div_ceil(1000), + dyncfg.pwr.max_power_mw + ); + log_tables(dev, &c.hwdata_b, c.tables.len); + Ok(images) +} + +/// Log the published performance-state tables, one line per entry: the primary table (fastest +/// state per voltage) and the secondary one (slowest), each with the device-tree index of the +/// entry, and the third table. +fn log_tables(dev: &device::Device, b: &raw::HwDataBG15V14_8_3, len: usize) { + let uniform = |row: &[u32; 8]| row.iter().all(|v| *v == row[0]); + for i in 1..len.min(16) { + let aux = &b.aux_ps_3; + let ok = uniform(&b.voltages[i]) && uniform(&b.voltages_sram[i]); + dev_info!( + dev, + "M3: performance state {}: primary dt {} {} MHz core {} mV SRAM {} mV; secondary dt {} {} MHz; third {} MHz core {} mV SRAM {} mV{}\n", + i, + b.unk_arr_0[i], + b.frequencies[i], + b.voltages[i][0], + b.voltages_sram[i][0], + b.unk_arr_0[16 + i], + b.frequencies_2[i], + aux.frequencies[i], + aux.voltages[i][0], + aux.voltages_sram[i][0], + if ok { "" } else { " (columns differ)" } + ); + } +} + +/// HwDataA words that hold the highest usable performance state scaled by 100. +const HWA_MAX_SCALED: [usize; 15] = { + type A = raw::HwDataAG15V14_8_3; + let se = offset_of!(A, unk_e10_0); + [ + offset_of!(A, max_pstate_scaled), + offset_of!(A, max_pstate_scaled_2), + offset_of!(A, max_pstate_scaled_3), + offset_of!(A, max_pstate_scaled_4), + offset_of!(A, max_pstate_scaled_5), + offset_of!(A, max_pstate_scaled_7), + offset_of!(A, max_pstate_scaled_8), + offset_of!(A, max_pstate_scaled_9), + offset_of!(A, max_pstate_scaled_10), + offset_of!(A, max_pstate_scaled_11), + offset_of!(A, max_pstate_scaled_12), + offset_of!(A, max_pstate_scaled_13), + offset_of!(A, max_pstate_scaled_14), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_1), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_2), + ] +}; + +/// HwDataA words that hold the base (lowest settled) performance state scaled by 100. +const HWA_BASE_SCALED: [usize; 4] = { + type A = raw::HwDataAG15V14_8_3; + [ + offset_of!(A, base_pstate_scaled), + offset_of!(A, base_pstate_scaled_2), + offset_of!(A, base_pstate_scaled_3), + offset_of!(A, base_pstate_scaled_4), + ] +}; + +/// Offsets in Globals of the runtime cap words: the second and third power-interface targets and +/// the performance-state cap, each the highest usable state scaled by 100. +const GLOBALS_RUNTIME_CAP: [usize; 3] = { + type G = raw::GlobalsG15V14_8_3; + let power_if_target = offset_of!(G, power_if_target); + [power_if_target + 4, power_if_target + 8, offset_of!(G, perf_state_cap)] +}; + +pub(crate) fn publish_pstates( + dev: &device::Device, + hwdata: &mut Buffer, + globals: &mut Buffer, + p: &PstatePolicy, +) -> Result { + use crate::float::F32; + type A = raw::HwDataAG15V14_8_3; + let a = |offset: usize| HWDATA_A + offset; + let full = 100 * p.table_max; + let base_scaled = hwdata.read_u32(a(offset_of!(A, base_pstate_scaled)))?; + let globals_max = GLOBALS_RUNTIME_CAP; + + let mut expected = true; + for off in HWA_MAX_SCALED { + expected &= hwdata.read_u32(a(off))? == full; + } + for off in HWA_BASE_SCALED { + expected &= hwdata.read_u32(a(off))? == base_scaled; + } + for off in globals_max { + expected &= globals.read_u32(off)? == full; + } + expected &= hwdata.read_u32(a(offset_of!(A, max_pstate_scaled_6)))? == F32::from(full).to_bits(); + let base = base_scaled / 100; + if !expected + || base_scaled % 100 != 0 + || base == 0 + || base > p.safe + || p.boot > p.safe + || p.safe > p.max + { + dev_err!(dev, "M3: unexpected performance-state words in the InitData; not starting the firmware\n"); + return Err(EINVAL); + } + + let ceiling = 100 * p.max; + let max_power = F32::from_bits(hwdata.read_u32(a(offset_of!(A, max_power_1)))?); + let min_duty = hwdata.read_u32(a(offset_of!(A, pwr_min_duty_cycle)))?; + for off in HWA_MAX_SCALED { + hwdata.u32(a(off), ceiling)?; + } + hwdata.u32(a(offset_of!(A, max_pstate_scaled_6)), F32::from(ceiling).to_bits())?; + hwdata.u32(a(offset_of!(A, pwr_pstate_related_k)), (-F32::from(ceiling) / max_power).to_bits())?; + hwdata.u32( + a(offset_of!(A, pwr_pstate_max_dc_offset)), + (min_duty as i32 - ceiling as i32) as u32, + )?; + hwdata.u32( + a(offset_of!(A, boost_state_unk_k)), + (F32::from(p.max - base) / crate::f32!(0.95)).to_bits(), + )?; + for off in globals_max { + globals.u32(off, 100 * p.safe)?; + } + + for off in [offset_of!(A, actual_pstate), offset_of!(A, tgt_pstate)] { + hwdata.u32(a(off), p.boot)?; + } + let settle = base; + dev_info!( + dev, + "M3: firmware performance request: state {} ({} MHz), base state {}, cap state {} ({} MHz) of 1..={}\n", + p.boot, + p.boot_mhz, + settle, + p.safe, + p.safe_mhz, + p.table_max + ); + if p.safe != p.max { + dev_info!( + dev, + "M3: firmware highest state {} ({} MHz); the runtime cap starts at {} ({} MHz)\n", + p.max, + p.max_mhz, + p.safe, + p.safe_mhz + ); + } + Ok(()) +} + +/// Set the runtime cap: the Globals runtime cap words, which the firmware keeps reading while it +/// runs, so the store alone moves the cap, with no message. `cap` must be a state of the +/// published table. +pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { + if cap == 0 || cap >= 16 { + return Err(EINVAL); + } + for off in GLOBALS_RUNTIME_CAP { + globals.u32(off, 100 * cap)?; + } + crate::agx_memory::publish(); + Ok(()) +} + +/// The integer part of a non-negative, finite F32 given by its bits; None otherwise. +fn f32_bits_to_u32(bits: u32) -> Option { + let exp = (bits >> 23) & 0xff; + if bits >> 31 != 0 || exp == 0xff { + return None; + } + if exp < 127 { + return Some(0); + } + let mant = u64::from((bits & 0x7f_ffff) | 0x80_0000); + let shift = exp - 127; + let value = if shift >= 23 { + mant.checked_shl(shift - 23)? + } else { + mant >> (23 - shift) + }; + u32::try_from(value).ok() +} + +/// One reading of the performance state: SGX+0xe01000, and the HwDataA actual and current +/// state and frequency (MHz, F32 bits) the firmware publishes. +#[derive(Copy, Clone, PartialEq, Eq)] +struct PstateSample { + register: u32, + actual: u32, + current: u32, + freq_bits: u32, +} + +/// Number of lines logging changes of the performance state per boot. +const PSTATE_LOG_LINES: u32 = 64; + +/// Number of lines warning about a reported state above the runtime cap per boot. +const RUNTIME_CAP_WARN_LINES: u32 = 16; + +/// Checks the performance state the firmware reports against the ceiling and the runtime cap, +/// after boot and after every job. +/// +/// The firmware publishes its actual and current state and its frequency in HwDataA. A +/// frequency above the ceiling entry's, or, when no running frequency is reported, a state in +/// the published table above the ceiling, that is still there when read again 1 ms later fails +/// the check. The same above the runtime cap (when it is below the ceiling) is reported to the +/// caller, which decides whether it persists too long (the thermal limit). Values outside the +/// published table are logged and not enforced: they are not states of this table. +/// SGX+0xe01000 is logged next to them but not enforced, as the index space of that register is +/// not established on every board. The published tables list device-tree operating points only, +/// so the check guards the cap, not the voltages. +pub(crate) struct PstateWatch { + policy: PstatePolicy, + last: Option, + lines: u32, + warn_lines: u32, + register_noted: bool, + unknown_noted: bool, + excused_noted: bool, +} + +impl PstateWatch { + pub(crate) fn new(policy: PstatePolicy) -> Self { + PstateWatch { + policy, + last: None, + lines: 0, + warn_lines: 0, + register_noted: false, + unknown_noted: false, + excused_noted: false, + } + } + + fn sample(hwdata: &mut Buffer, register: u32) -> Result { + type A = raw::HwDataAG15V14_8_3; + Ok(PstateSample { + register, + actual: hwdata.read_u32(HWDATA_A + offset_of!(A, actual_pstate))?, + current: hwdata.read_u32(HWDATA_A + offset_of!(A, cur_pstate))?, + freq_bits: hwdata.read_u32(HWDATA_A + offset_of!(A, freq_mhz))?, + }) + } + + /// Frequency of entry `state` of the published table, in MHz. + fn mhz(&self, state: u32) -> u32 { + self.policy.freqs.get(state as usize).copied().unwrap_or(0) + } + + /// The reported frequency in MHz, if it is a running frequency of the published table. + fn freq_mhz(&self, s: &PstateSample) -> Option { + f32_bits_to_u32(s.freq_bits).filter(|mhz| *mhz > 0 && *mhz <= self.policy.table_max_mhz) + } + + /// Whether a reported state lies within the published table but above `cap`. + fn state_above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + let above = |v: u32| v > cap && v <= self.policy.table_max; + if above(s.actual) { + Some(("actual state", s.actual)) + } else if above(s.current) { + Some(("current state", s.current)) + } else { + None + } + } + + /// The reported value above `cap` that lies within the published table, if any. The + /// frequency is the physical quantity: a state above the cap is excused when the reported + /// frequency is a running frequency within the cap (the state words then use another + /// index space), and counts otherwise. + fn above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + match self.freq_mhz(s) { + Some(mhz) if mhz > self.mhz(cap) => Some(("frequency (MHz)", mhz)), + Some(_) => None, + None => self.state_above(s, cap), + } + } + + /// Whether a reported value is not a state or frequency of the published table. + fn unknown(&self, s: &PstateSample) -> bool { + let p = &self.policy; + s.actual > p.table_max + || s.current > p.table_max + || f32_bits_to_u32(s.freq_bits).is_none_or(|mhz| mhz > p.table_max_mhz) + } + + fn log(&self, dev: &device::Device, what: &str, s: &PstateSample, cap: u32) { + dev_info!( + dev, + "M3: performance state {}: firmware actual {} current {} frequency {} MHz, sgx+0xe01000 {:#x}; cap state {} ({} MHz)\n", + what, + s.actual, + s.current, + f32_bits_to_u32(s.freq_bits).unwrap_or(u32::MAX), + s.register, + cap, + self.mhz(cap) + ); + } + + /// Read the existing firmware state words and SGX state register without + /// requesting a state, publishing a cap, or enforcing a sample. + pub(crate) fn log_active(&self, dev: &device::Device, hwdata: &mut Buffer, + register: u32, cap: u32, sequence: u32) -> Result { + let s = Self::sample(hwdata, register)?; + dev_info!(dev, + "M3_ACTIVE_POWER sample={} actual={} current={} freq_bits={:#x} sgx={:#x} cap={} cap_mhz={}\n", + sequence, s.actual, s.current, s.freq_bits, s.register, cap, self.mhz(cap)); + Ok(()) + } + + /// Check the reported performance state. `register` reads SGX+0xe01000; `what` names the + /// point of the check for the log; `cap` is the runtime cap the firmware may use now + /// (clamped to the ceiling). Returns ERANGE when the firmware stays above the ceiling; + /// otherwise whether it stays above `cap`, and the highest state it reports (0 if neither + /// state word is a state of the published table). + pub(crate) fn check( + &mut self, + dev: &device::Device, + hwdata: &mut Buffer, + register: impl Fn() -> Result, + what: &str, + cap: u32, + ) -> Result<(bool, u32)> { + let ceiling = self.policy.max; + let cap = cap.clamp(1, ceiling); + let mut s = Self::sample(hwdata, register()?)?; + let mut over = false; + if let Some((field, value)) = self.above(&s, cap) { + if cap == ceiling || self.warn_lines < RUNTIME_CAP_WARN_LINES { + self.warn_lines = self.warn_lines.saturating_add(u32::from(cap != ceiling)); + dev_warn!( + dev, + "M3: firmware reports {} {} above the cap {} ({} MHz) {}; reading again\n", + field, + value, + cap, + self.mhz(cap), + what + ); + } + fsleep(Delta::from_millis(1)); + s = Self::sample(hwdata, register()?)?; + if let Some((field, value)) = self.above(&s, ceiling) { + self.log(dev, what, &s, cap); + dev_err!( + dev, + "M3: firmware {} {} stays above the cap {} ({} MHz); marking the GPU failed\n", + field, + value, + ceiling, + self.mhz(ceiling) + ); + return Err(ERANGE); + } + over = self.above(&s, cap).is_some(); + } + if !self.excused_noted { + if let Some((field, value)) = self.state_above(&s, cap) { + if !over { + self.excused_noted = true; + dev_info!( + dev, + "M3: firmware {} {} is above the cap {} but its frequency is within it {}; not enforced\n", + field, + value, + cap, + what + ); + } + } + } + if !self.unknown_noted && self.unknown(&s) { + self.unknown_noted = true; + dev_info!( + dev, + "M3: firmware performance words outside the published table (states 1..={}, up to {} MHz) {}; not enforced\n", + self.policy.table_max, + self.policy.table_max_mhz, + what + ); + self.log(dev, what, &s, cap); + } + if !self.register_noted && s.register & 0xf > cap { + self.register_noted = true; + dev_info!( + dev, + "M3: sgx+0xe01000 reads {:#x}, above the cap as a table index; logged only\n", + s.register + ); + } + let key = PstateSample { + register: s.register & 0xf, + ..s + }; + if self.last != Some(key) { + if self.lines < PSTATE_LOG_LINES { + self.log(dev, what, &s, cap); + } else if self.lines == PSTATE_LOG_LINES { + dev_info!(dev, "M3: further performance state changes are not logged\n"); + } + self.lines = self.lines.saturating_add(1); + self.last = Some(key); + } + let peak = [s.actual, s.current] + .into_iter() + .filter(|v| *v <= self.policy.table_max) + .max() + .unwrap_or(0); + Ok((over, peak)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs new file mode 100644 index 0000000000000..5ecfd820b4e4f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board facts for the M3 runtime, read from the hardware and the device tree instead of being +//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. + +use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; + +use crate::m3_resources::{ + reserved_resource, + Region, + Resources, // +}; +use crate::m3_soc::Soc; + +/// Whether the root node's compatible list contains `compatible`. +fn board_is(compatible: &[u8]) -> bool { + let Some(root) = kernel::of::root() else { + return false; + }; + let Ok(board) = root.get_property::>(c_str!("compatible")) else { + return false; + }; + board.split(|b| *b == 0).any(|s| s == compatible) +} + +/// Whether this is a board the M3 runtime is validated on (`soc.validated_boards`). +/// `asahi.m3_expose=auto` registers the render node and `asahi.m3_backend=auto` starts the +/// runtime only on these. +pub(crate) fn runtime_validated_board(soc: &Soc) -> bool { + soc.validated_boards.iter().any(|board| board_is(board)) +} + +/// The number of core slots of `soc` (2 x 10 on T6030, 4 x 10 on T6031). +pub(crate) fn core_slots(soc: &Soc) -> u32 { + soc.cores_per_cluster * soc.clusters +} + +/// Whether the core-enable mask describes a usable GPU of `soc`: at least one core, and no core +/// outside its core slots. The mask is the fused core-enable words from SGX+0xe01500, the second +/// word as bits 32..63 when the SoC has more than 32 core slots. It varies with the SKU (0x6f5fc +/// on a 14-core T6030, 0x7fdff on an 18-core one); absent cores must never be enabled. +pub(crate) fn core_mask_valid(soc: &Soc, mask: u64) -> bool { + let slots = core_slots(soc); + mask != 0 && slots <= 64 && (slots == 64 || mask >> slots == 0) +} + +/// Per-cluster UAPI core masks for a core-enable mask of `soc`: consecutive fields of +/// `cores_per_cluster` bits, cluster 0 lowest, across the word boundary (the packing the +/// identification code uses for every multi-word mask, `regs.rs`). +pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUSTERS as usize] { + let mut masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + let width = soc.cores_per_cluster.min(32); + let field = (1u64 << width) - 1; + for (cluster, slot) in masks.iter_mut().enumerate().take(soc.clusters as usize) { + let shift = cluster as u32 * soc.cores_per_cluster; + *slot = mask.checked_shr(shift).map_or(0, |m| (m & field) as u32); + } + masks +} + +/// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node +/// has no populated OPP table. +pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut max_hz: u64 = 0; + for opp in opps.children() { + if let Ok(hz) = opp.get_property::(c_str!("opp-hz")) { + max_hz = max_hz.max(hz); + } + } + u32::try_from(max_hz / 1000).ok().filter(|khz| *khz != 0) +} + +/// The shape of the performance-state table the device tree's operating points give: states +/// above the off state and the highest frequency in MHz (`t8122_admission::opp_table_shape`). +/// Every cluster of a point must have the same voltage, except on a SoC with per-cluster +/// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable +/// OPP table. +pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut points = KVec::new(); + for opp in opps.children() { + let hz = opp.get_property::(c_str!("opp-hz")).ok()?; + let microvolt = opp.get_property::>(c_str!("opp-microvolt")).ok()?; + let mv = microvolt.first()?.div_ceil(1000); + if microvolt.len() != soc.clusters as usize + || (!soc.per_cluster_voltages && microvolt.iter().any(|uv| uv.div_ceil(1000) != mv)) + { + return None; + } + // With per-cluster voltages a point is ordered by its highest cluster voltage, as the + // published tables are (`initdata::PStateTables`). + let uv = if soc.per_cluster_voltages { + *microvolt.iter().max()? + } else { + *microvolt.first()? + }; + points.push((hz, uv), GFP_KERNEL).ok()?; + } + crate::t8122_admission::opp_table_shape(points) +} + +/// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the +/// first 16 KiB of `asc` (`m3_device`); the first 16 MiB of `sgx` cover the ID block at +/// +0xd04000, the Fender MMU block at +0xd08000 and the core masks at +0xe01500. +fn reg_windows(soc: &Soc) -> [(&'static CStr, u64, u64); 2] { + [ + (c_str!("asc"), soc.windows.asc, 0x4000), + (c_str!("sgx"), soc.windows.sgx, 0x100_0000), + ] +} + +/// The `reg` cells (two address and two size cells) of the mailbox of `soc`: 16 KiB at its base. +fn expected_mailbox_reg(soc: &Soc) -> [u32; 4] { + [(soc.windows.mailbox >> 32) as u32, soc.windows.mailbox as u32, 0, 0x4000] +} + +/// Reserved regions of the firmware handoff, in `Resources::regions` order. +const REGIONS: [&CStr; 6] = [ + c_str!("ttbs"), + c_str!("pagetables"), + c_str!("handoff"), + c_str!("shared-l2"), + c_str!("fw-text"), + c_str!("fw-data"), +]; + +/// Static reserved-memory nodes that describe a handoff region when the GPU node does not list +/// it under its own name: the bootloader fills these nodes (the UAT regions of the firmware's +/// page-table handoff) whether or not the GPU node references them. The second-level table of +/// the firmware's upper page-table tree ("shared-l2") is reserved this way without being in the +/// GPU node's memory-region list. +const REGION_NODES: [(&CStr, &CStr); 4] = [ + (c_str!("ttbs"), c_str!("/reserved-memory/uat-ttbs")), + (c_str!("pagetables"), c_str!("/reserved-memory/uat-pagetables")), + (c_str!("handoff"), c_str!("/reserved-memory/uat-handoff")), + (c_str!("shared-l2"), c_str!("/reserved-memory/uat-pagetables-l2")), +]; + +/// Properties that describe the loaded firmware's segments on the GPU node. +const SEGMENT_PROPS: [&CStr; 3] = [ + c_str!("apple,m3-handoff-version"), + c_str!("apple,firmware-segment-vas"), + c_str!("apple,firmware-segment-flags"), +]; + +/// Power-management properties of a GPU node whose SoC takes them from the boot loader +/// (`Soc::power_from_boot_loader`), one 32-bit cell each: the ones the T6030 device tree carries +/// as static values, and the shader-engine target. The boot loader copies each from the ADT's GPU +/// node (`apple,X` from `gpu-X`; `apple,se-target` from `gpu-se-tgt`). +/// +/// `apple,idleoff-standby-timer` is not among them: the J613 ADT has no +/// `gpu-idleoff-standby-timer`, so the boot loader adds it only when the ADT has one, and the +/// HwConfig's default applies otherwise. +const BOOT_LOADER_POWER: [&CStr; 40] = [ + c_str!("apple,perf-base-pstate"), + c_str!("apple,min-sram-microvolt"), + c_str!("apple,avg-power-filter-tc-ms"), + c_str!("apple,avg-power-ki-only"), + c_str!("apple,avg-power-kp"), + c_str!("apple,avg-power-min-duty-cycle"), + c_str!("apple,avg-power-target-filter-tc"), + c_str!("apple,fast-die0-integral-gain"), + c_str!("apple,fast-die0-proportional-gain"), + c_str!("apple,perf-boost-ce-step"), + c_str!("apple,perf-boost-min-util"), + c_str!("apple,perf-filter-drop-threshold"), + c_str!("apple,perf-filter-time-constant"), + c_str!("apple,perf-filter-time-constant2"), + c_str!("apple,perf-integral-gain"), + c_str!("apple,perf-integral-gain2"), + c_str!("apple,perf-integral-min-clamp"), + c_str!("apple,perf-proportional-gain"), + c_str!("apple,perf-proportional-gain2"), + c_str!("apple,perf-tgt-utilization"), + c_str!("apple,power-sample-period"), + c_str!("apple,ppm-filter-time-constant-ms"), + c_str!("apple,ppm-ki"), + c_str!("apple,ppm-kp"), + c_str!("apple,pwr-filter-time-constant"), + c_str!("apple,pwr-integral-gain"), + c_str!("apple,pwr-integral-min-clamp"), + c_str!("apple,pwr-min-duty-cycle"), + c_str!("apple,pwr-proportional-gain"), + c_str!("apple,pwr-sample-period-aic-clks"), + c_str!("apple,se-engagement-criteria"), + c_str!("apple,se-filter-time-constant"), + c_str!("apple,se-filter-time-constant-1"), + c_str!("apple,se-inactive-threshold"), + c_str!("apple,se-ki"), + c_str!("apple,se-ki-1"), + c_str!("apple,se-kp"), + c_str!("apple,se-kp-1"), + c_str!("apple,se-reset-criteria"), + c_str!("apple,se-target"), +]; + +/// Check that the boot loader gave a GPU node of `soc` every per-machine property the runtime +/// would otherwise take from a static device tree or a driver default: [`BOOT_LOADER_POWER`], the +/// 64-bit `apple,fast-die0-sensor-mask`, one core and one SRAM leakage coefficient per cluster, +/// the firmware version (three cells, not all zero), and an operating-point table of at least two +/// entries, each with one voltage per cluster and a power, and a frequency after the first. Logs +/// every missing or malformed property. +fn check_boot_loader_power(dev: &device::Device, node: &of::Node, soc: &Soc) -> Result { + let mut bad = 0u32; + let mut expect = |name: &CStr, want: usize, nonzero: bool| { + match node.get_property::>(name) { + Ok(value) if value.len() == want && (!nonzero || value.iter().any(|b| *b != 0)) => {} + Ok(value) => { + dev_info!( + dev, + "M3: not admitted: {:?} is {} bytes, {} expected{}\n", + name, + value.len(), + want, + if nonzero { ", not all zero" } else { "" } + ); + bad += 1; + } + Err(_) => { + dev_info!(dev, "M3: not admitted: the boot loader did not add {:?}\n", name); + bad += 1; + } + } + }; + for name in BOOT_LOADER_POWER { + expect(name, 4, false); + } + expect(c_str!("apple,fast-die0-sensor-mask"), 8, false); + expect(c_str!("apple,core-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,sram-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,firmware-version"), 12, true); + + let mut opps = 0u32; + let mut opps_ok = true; + match node.parse_phandle(c_str!("operating-points-v2"), 0) { + Some(table) => { + for opp in table.children() { + let hz = opp.get_property::(c_str!("opp-hz")).unwrap_or(0); + let volts = opp.get_property::>(c_str!("opp-microvolt")).map_or(0, |v| v.len()); + let power = opp.get_property::(c_str!("opp-microwatt")).is_ok(); + opps_ok &= volts == soc.clusters as usize && power && (opps == 0 || hz != 0); + opps += 1; + } + } + None => opps_ok = false, + } + if !opps_ok || opps < 2 { + dev_info!( + dev, + "M3: not admitted: the operating points are not the boot loader's ({} entries; each needs opp-hz, {} opp-microvolt cells and opp-microwatt)\n", + opps, + soc.clusters + ); + bad += 1; + } + if bad != 0 { + return Err(EINVAL); + } + dev_info!(dev, "M3: boot loader power configuration accepted ({} operating points)\n", opps); + Ok(()) +} + +/// Whether the GPU node lists the memory region `name` in memory-region-names. +fn lists_region(node: &of::Node, name: &CStr) -> bool { + node.get_property::>(c_str!("memory-region-names")) + .is_ok_and(|names| names.split(|b| *b == 0).any(|n| n == name.to_bytes())) +} + +/// The static reserved-memory node that stands in for the handoff region `name`, if the GPU +/// node does not list the region itself and the device tree has such a node. +fn region_node_path(node: &of::Node, name: &CStr) -> Option<&'static CStr> { + if lists_region(node, name) { + return None; + } + REGION_NODES + .iter() + .find(|(region, _)| region.to_bytes() == name.to_bytes()) + .map(|(_, path)| *path) +} + +/// A reference to a device tree node found by path, released on drop. +struct NodeRef(*mut bindings::device_node); + +impl NodeRef { + fn find(path: &CStr) -> Option { + // SAFETY: `path` is a NUL-terminated string; the lookup takes a node reference or + // returns NULL. + let np = unsafe { bindings::of_find_node_opts_by_path(path.as_char_ptr(), core::ptr::null_mut()) }; + (!np.is_null()).then_some(NodeRef(np)) + } +} + +impl Drop for NodeRef { + fn drop(&mut self) { + // SAFETY: the reference was taken by `find`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +/// Look up the handoff region `name` through its static reserved-memory node (see +/// [`REGION_NODES`]). Returns None when the GPU node lists the region itself or the device tree +/// has no such node; otherwise the region, which must be an enabled reserved-memory node the +/// kernel reserved at boot. Returns whether the node is no-map as well. +pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option> { + let path = region_node_path(node, name)?; + let np = NodeRef::find(path)?; + // SAFETY: `np` holds a node reference for the duration of these read-only queries. + let (available, rmem, nomap) = unsafe { + ( + bindings::of_device_is_available(np.0), + bindings::of_reserved_mem_lookup(np.0), + bindings::of_property_read_bool(np.0, c_str!("no-map").as_char_ptr()), + ) + }; + if !available || rmem.is_null() { + return Some(Err(EINVAL)); + } + // SAFETY: the reserved-memory table entry lives as long as the kernel. + let (base, size) = unsafe { ((*rmem).base, (*rmem).size) }; + if base == 0 || size == 0 || base.checked_add(size).is_none_or(|end| end > 1 << 42) { + return Some(Err(EINVAL)); + } + let raw = bindings::resource { + start: base, + end: base + size - 1, + flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque. The descriptor is owned and + // has no pointers into temporary data; its range is checked above. + Some(Ok((unsafe { core::mem::transmute::(raw) }, nomap))) +} + +/// Admit a GPU of `soc` described by the device tree, either statically or by a runtime overlay. +/// +/// The device must be a GPU node of `soc` with the ASC and SGX windows at their addresses +/// (at least as large as the runtime maps), the GPU coprocessor mailbox, the six handoff regions +/// by name (from the reserved-memory registry or no-map overlay nodes), and the firmware segment +/// description. Every refusal is logged. +pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result { + let dev = pdev.as_ref(); + let refuse = |what: &str, err: Error| { + dev_info!(dev, "M3: not admitted: {}\n", what); + err + }; + let node = dev.of_node().ok_or(ENODEV)?; + + if !board_is(soc.board.as_bytes()) { + dev_info!(dev, "M3: not admitted: not a {} board\n", soc.name); + return Err(ENODEV); + } + let compatible: KVec = node.get_property(c_str!("compatible"))?; + if !compatible.split(|b| *b == 0).any(|s| s == soc.gpu.as_bytes()) { + dev_info!(dev, "M3: not admitted: GPU node is not {}\n", soc.gpu); + return Err(ENODEV); + } + // The runtime runs the firmware that the bootloader loaded, in place: without the + // bootloader's description of its segments there is nothing to admit. + if !lists_region(&node, c_str!("fw-text")) + || !lists_region(&node, c_str!("fw-data")) + || SEGMENT_PROPS.iter().any(|p| node.get_property::>(p).is_err()) + { + return Err(refuse( + "the device tree does not describe the loaded GPU firmware segments (fw-text/fw-data regions and apple,firmware-segment-* properties); the bootloader predates this", + ENODEV, + )); + } + dev_info!(dev, "M3: board compatible accepted\n"); + if soc.power_from_boot_loader { + check_boot_loader_power(dev, &node, soc)?; + } + + for name in REGIONS { + if lists_region(&node, name) { + continue; + } + match region_node_path(&node, name) { + Some(path) if NodeRef::find(path).is_some() => { + dev_info!(dev, "M3: {:?} region: static node {:?}\n", name, path) + } + _ => { + dev_info!(dev, "M3: not admitted: no {:?} memory region\n", name); + return Err(EINVAL); + } + } + } + dev_info!(dev, "M3: memory region list accepted\n"); + + for (name, base, min_size) in reg_windows(soc) { + let res = pdev + .resource_by_name(name) + .ok_or_else(|| refuse("missing register window", EINVAL))?; + if res.start() != base || res.size() < min_size { + dev_info!( + dev, + "M3: not admitted: {:?} window {:#x}+{:#x}, need {:#x}+{:#x} or larger\n", + name, + res.start(), + res.size(), + base, + min_size + ); + return Err(EINVAL); + } + } + dev_info!(dev, "M3: register resources accepted\n"); + + let mboxes: KVec = node.get_property(c_str!("mboxes"))?; + let mbox = (mboxes.len() == 1) + .then(|| node.parse_phandle(c_str!("mboxes"), 0)) + .flatten() + .ok_or_else(|| refuse("expected exactly one mailbox", EINVAL))?; + let compat: KVec = mbox.get_property(c_str!("compatible"))?; + let mbox_reg: KVec = mbox.get_property(c_str!("reg"))?; + let irq_names: KVec = mbox.get_property(c_str!("interrupt-names"))?; + let irqs: KVec = mbox.get_property(c_str!("interrupts"))?; + let cells: u32 = mbox.get_property(c_str!("#mbox-cells"))?; + if !soc.mailbox_compatibles.iter().any(|c| *c == compat.as_slice()) + || mbox_reg.as_slice() != expected_mailbox_reg(soc) + || cells != 0 + || irq_names.as_slice() != b"send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0" + || irqs.as_slice() != soc.mailbox_interrupts + { + return Err(refuse("unexpected GPU mailbox description", EINVAL)); + } + dev_info!(dev, "M3: mailbox resource accepted\n"); + + let mut regions = [Region { base: 0, size: 0 }; 6]; + for (i, name) in REGIONS.iter().enumerate() { + let res = reserved_resource(&node, name).map_err(|e| { + dev_info!(dev, "M3: not admitted: {:?} region unusable ({:?})\n", name, e); + e + })?; + regions[i] = Region { + base: res.start(), + size: res.size(), + }; + } + dev_info!(dev, "M3: reserved resources {:?}\n", regions); + + let version: u32 = node + .get_property(c_str!("apple,m3-handoff-version")) + .map_err(|e| refuse("no firmware handoff description", e))?; + let vas: KVec = node.get_property(c_str!("apple,firmware-segment-vas"))?; + let flags: KVec = node.get_property(c_str!("apple,firmware-segment-flags"))?; + Resources::validate( + version, + regions, + vas.as_slice().try_into().map_err(|_| EINVAL)?, + flags.as_slice().try_into().map_err(|_| EINVAL)?, + ) + .map_err(|e| { + dev_info!(dev, "M3: not admitted: handoff description rejected ({:?})\n", e); + EINVAL + }) +} + +/// Whether the GPU node links to a PMP instance (`apple,pmp`) through which the GPU power vote +/// is cast. Without one, GPU power is left to the power domain. +pub(crate) fn has_pmp_link(pdev: &platform::Device) -> bool { + pdev.as_ref() + .of_node() + .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) +} + +pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; +use crate::m3_firmware::image_info; + +fn sha256(data: &[u8]) -> [u8; 32] { + let mut digest = [0u8; 32]; + // SAFETY: `data` and `digest` are valid for the duration of the synchronous call. + unsafe { bindings::sha256(data.as_ptr(), data.len(), digest.as_mut_ptr()) }; + digest +} + +/// Lower-case hex of `bytes`, for logs. +fn hex(bytes: &[u8]) -> [u8; N] { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = [b'0'; N]; + for (i, b) in bytes.iter().take(N / 2).enumerate() { + out[2 * i] = DIGITS[(b >> 4) as usize]; + out[2 * i + 1] = DIGITS[(b & 0xf) as usize]; + } + out +} + +pub(crate) fn identify( + dev: &device::Device, + text: &[u8], + stkg_sha256: &[u8; 32], + images: &'static [KnownImage], +) -> Option<&'static KnownImage> { + let info = image_info(text); + let masked_sha256 = info.as_ref().and_then(|info| { + let mut copy = KVec::new(); + copy.extend_from_slice(text, GFP_KERNEL).ok()?; + copy.get_mut(info.patchbay.clone())?.fill(0); + Some(sha256(©)) + }); + let uuid = info.as_ref().map(|info| info.uuid); + let uuid_hex = hex::<32>(&uuid.unwrap_or([0; 16])); + let stkg_hex = hex::<64>(stkg_sha256); + let masked_hex = hex::<64>(&masked_sha256.unwrap_or([0; 32])); + fn as_str(b: &[u8]) -> &str { + core::str::from_utf8(b).unwrap_or("?") + } + dev_info!( + dev, + "M3: loaded GPU firmware: uuid {} (header {}), text sha256 {} (STKG zeroed), {} (patchbay zeroed)\n", + if uuid.is_some() { as_str(&uuid_hex) } else { "none" }, + if info.as_ref().is_some_and(|i| i.text_size == text.len()) { "ok" } else { "missing or unexpected" }, + as_str(&stkg_hex), + if masked_sha256.is_some() { as_str(&masked_hex) } else { "n/a" } + ); + let image = images.iter().find(|image| image.accepts_runtime(uuid, stkg_sha256)); + match image { + Some(image) => dev_info!(dev, "M3: GPU firmware identified as {}\n", image.name), + None => dev_err!(dev, "M3: loaded GPU firmware is not a known image\n"), + } + image +} + +/// Whether the reserved-memory node behind the GPU memory region `name` is no-map. +/// +/// A region without no-map is part of the kernel's linear map, which maps it write-back; any +/// other CPU mapping of it must use the same memory type, because mismatched aliases of the same +/// memory are not allowed. Only no-map regions may be mapped write-combined. Returns true when +/// the region cannot be looked up, which keeps the caller's own choice. +pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { + if let Some(found) = static_region(node, name) { + return found.map_or(true, |(_, nomap)| nomap); + } + let Ok(names) = node.get_property::>(c_str!("memory-region-names")) else { + return true; + }; + let Some(index) = names.split(|b| *b == 0).position(|n| n == name.to_bytes()) else { + return true; + }; + let Some(region) = node.parse_phandle(c_str!("memory-region"), index) else { + return true; + }; + region.get_property::>(c_str!("no-map")).is_ok() +} + +/// Whether the M3 render node is registered (`asahi.m3_expose`). +/// +/// Userspace drivers pick up any registered render node. Until the M3 runtime is validated on a +/// board, a render node there would let a stock userspace driver submit work the board may not +/// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. +pub(crate) fn expose_render_node(soc: &Soc) -> bool { + match *crate::module_parameters::m3_expose.value() { + 0 => false, + v if v > 0 => true, + _ => runtime_validated_board(soc), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs new file mode 100644 index 0000000000000..6661222530a38 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Safety checks for clients of the M3 (G15) render node. +//! +//! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for +//! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 +//! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at +//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks +//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore +//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a +//! VM, so a refused client can still query the parameters but never reaches the hardware. + +use core::sync::atomic::{ + AtomicU32, + AtomicU64, + Ordering, // +}; + +use kernel::{ + prelude::*, + uapi, +}; + +use crate::{ + driver::AsahiDevice, + agx_uapi::{ + GpuAccess, + GpuAddressSpace, // + }, + mmu, +}; + +/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. +const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; + +/// Refused clients that are logged per boot; later refusals only count. +const REFUSALS_LOGGED: u32 = 16; +static REFUSALS: AtomicU32 = AtomicU32::new(0); + +/// Per-file admission state. +pub(crate) struct ClientGate { + /// Largest size this file has asked GET_PARAMS for the global parameters with. + params_size: AtomicU64, +} + +impl ClientGate { + pub(crate) const fn new() -> Self { + Self { + params_size: AtomicU64::new(0), + } + } + + /// Record a successful GET_PARAMS for the global parameters. + pub(crate) fn note_params(&self, size: u64) { + self.params_size.fetch_max(size, Ordering::Relaxed); + } + + /// Whether this file may create a VM. Only G15 devices are gated. + pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { + let size = self.params_size.load(Ordering::Relaxed); + if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { + return Ok(()); + } + let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if refusals <= REFUSALS_LOGGED { + let task = kernel::current!(); + let comm = task_comm(task); + let len = comm.iter().position(|&b| b == 0).unwrap_or(comm.len()); + let name = core::str::from_utf8(&comm[..len]).unwrap_or("?"); + let last = if refusals == REFUSALS_LOGGED { + " (further refusals are not logged)" + } else { + "" + }; + dev_info!( + device.as_ref(), + "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", + name, + task.group_leader().pid(), + size, + AGX3_PARAMS_SIZE, + last + ); + } + Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32))) + } +} + +/// The name of `task`, for log lines only. +pub(crate) fn task_comm(task: &kernel::task::Task) -> [u8; 16] { + let mut name = [0u8; 16]; + // SAFETY: `comm` is a fixed-size array inside the live task. It is read byte by byte with + // volatile loads because a concurrent rename may change it; a torn name is only printed. + unsafe { + let comm = core::ptr::addr_of!((*task.as_ptr()).comm).cast::(); + for (i, byte) in name.iter_mut().enumerate() { + *byte = comm.add(i).read_volatile(); + } + } + name +} + +/// Bytes at the start of a compute control stream whose launch records are checked. +const LAUNCH_WINDOW: usize = 4096; +/// Bytes that must be readable at a launch's program pointer. The pointer is 64-byte aligned, +/// so this never crosses a page. +const PROGRAM_BYTES: u64 = 64; +/// CDM record types (bits 31:29 of a record's first word). +const CDM_LAUNCH: u32 = 0; +const CDM_BARRIER: u32 = 3; + +/// Refuse a compute command whose control stream starts with a launch the GPU cannot fetch. +/// +/// Walks the records at the start of the stream: launches (40 bytes direct, 36 or 24 bytes +/// indirect, told apart by bits 28:27) and barriers (4 bytes). The walk ends at the first link, +/// return, terminator or unknown record, at the end of the stream and after `LAUNCH_WINDOW` +/// bytes. Each launch names its program at `(word1[31:22] << 38) | (word2 << 6)`; that address, +/// and the one read with only word2's low 26 bits when VA[37:32] is set, must be mapped +/// readable in the submitting VM, outside the driver's own mappings, or the command is refused +/// with EINVAL before it is queued. Without the check the GPU would take a translation fault on +/// the fetch, which the M3 backends cannot recover from. +/// +/// This catches malformed or stale pointers, not a hostile client: userspace can rewrite the +/// stream after the check. +pub(crate) fn check_compute_launches( + vm: &mmu::Vm, + space: &A, + base: u64, + end: u64, +) -> Result { + let len = usize::try_from(end.checked_sub(base).ok_or(EINVAL)?) + .unwrap_or(usize::MAX) + .min(LAUNCH_WINDOW); + let mut stream = KVec::from_elem(0u8, len, GFP_KERNEL)?; + if let Err(error) = read_through_cache(vm, base, &mut stream) { + pr_err!( + "M3 compute rejected: control stream {:#x} cannot be read ({:?})\n", + base, + error + ); + return Err(EINVAL); + } + let word = |offset: usize| { + u32::from_le_bytes([ + stream[offset], + stream[offset + 1], + stream[offset + 2], + stream[offset + 3], + ]) + }; + let mut offset = 0; + let mut launch = 0u32; + while offset + 4 <= len { + let head = word(offset); + match head >> 29 { + CDM_LAUNCH => { + if offset + 12 > len { + break; + } + let high = u64::from(word(offset + 4) >> 22) << 38; + let low = u64::from(word(offset + 8)); + // Whether the GPU takes VA[37:32] from word2's top six bits is not + // established, so the program must be mapped under both readings. They + // agree whenever those bits are clear, as in every working launch. + for program in [high | (low << 6), high | ((low & 0x3ff_ffff) << 6)] { + if !space.covers(program, PROGRAM_BYTES, GpuAccess::Read) { + pr_err!( + "M3 compute rejected: launch {} at {:#x} names program {:#x}, which is not mapped readable in this VM\n", + launch, + base + offset as u64, + program + ); + return Err(EINVAL); + } + } + launch += 1; + offset += match (head >> 27) & 3 { + 0 => 40, + 1 => 36, + 2 => 24, + _ => break, + }; + } + CDM_BARRIER => offset += 4, + _ => break, + } + } + Ok(()) +} + +/// Read `out.len()` bytes at `iova` in `vm` through the kernel's linear map. +/// +/// Userspace normally writes command streams through a write-combined mapping, which does not +/// update cache lines the kernel's cacheable alias may hold from an earlier read or a +/// speculative fetch. Each line is cleaned and invalidated before it is read, so the bytes +/// come from memory, as the GPU will fetch them. Cleaning never discards data: a dirty line from +/// a cached mapping is written back first. +/// +/// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the +/// pages cannot be unmapped and freed while they are cleaned and read. +fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { + vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) +} + +/// Clean and invalidate the data cache lines covering `start..start + len` to the point of +/// coherency, completing before later loads. +fn clean_invalidate(start: usize, len: usize) { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and has no side effects. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, options(nomem, nostack, preserves_flags)); + } + // DminLine: log2 of the smallest data cache line, in 4-byte words. + let line = 4usize << ((ctr >> 16) & 0xf); + let end = start.saturating_add(len); + let mut address = start & !(line - 1); + // SAFETY: the range lies inside one page of the linear map that the caller has mapped. + // Clean-and-invalidate writes dirty lines back before dropping them, so no data is lost. + unsafe { + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc civac, {address}", address = in(reg) address, options(nostack, preserves_flags)); + address += line; + } + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs new file mode 100644 index 0000000000000..08114c4d8cc8a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Completion work owned only by the M3 runtime. + +use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; +use crate::{driver, m3_drm::Shared}; + +#[pin_data] +pub(crate) struct Completion { + shared: Shared, + #[pin] + work: Work, +} +impl Completion { + pub(crate) fn new(shared: Shared) -> Result> { + Arc::pin_init(try_pin_init!(Self { + shared, + work <- new_work!("M3 completion"), + }), GFP_KERNEL) + } +} +impl_has_work! { impl HasWork for Completion { self.work } } +impl WorkItem for Completion { + type Pointer = Arc; + fn run(owner: Arc) { + use crate::m3_runtime::Service; + // Snapshots after a notification before sleeping for the next one: the completion + // stamps can trail the firmware's event message. + const TRAILING_POLLS: u32 = 4; + let (mut trailing, mut waited_at) = (0, u64::MAX); + loop { + let (outcome, events) = { + let mut guard = owner.shared.lock(); + let Some(runtime) = Option::as_mut(&mut *guard) else { return; }; + let outcome=runtime.service_job(); + let events=runtime.events(); + // Preserve the root-only CPU-overlap diagnostic: disabled waits retain + // the runtime lock so VM/timestamp preparation cannot overlap the GPU. + if let Service::Waiting(messages)=outcome { + if !events.cpu_overlap() {events.wait_past(messages);continue;} + } + (outcome,events) + }; + match outcome { + Service::Idle => break, + Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Waiting(messages) => { + if messages != waited_at { waited_at = messages; trailing = 0; } + if trailing < TRAILING_POLLS { + trailing += 1; + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); + } else { + // Until the next firmware event, or a bounded 100 us resnapshot. + events.wait_past(messages); + trailing = 0; + } + } + } + } + } +} +pub(crate) fn queue(dev: &driver::AsahiDevice) { + if let Some(work) = dev.completion.as_ref() { + let _ = workqueue::system_highpri().enqueue(work.clone()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs new file mode 100644 index 0000000000000..e7469d31b496f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_state_layout as state; +use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, + m3_pool_layout::GpuRegion, m3_init_layout::Region}; +use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; +use crate::m3_timeline as timeline; +use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, + TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; +const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { + backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, + manager:storage::POOL,counter:storage::COUNTER,table_index:2, +}; +// Retain at most four VM views, including the active one, as the render +// graph does. Game and compositor compute jobs otherwise remap the whole +// private arena every time they alternate. Each entry pins its VM binding. +struct CachedViews { + command: mmu::KernelMapping, + views: KVec<(usize, mmu::KernelMapping)>, + // Release cached mappings before returning their ASID lease. + binding: mmu::VmBind, +} +pub(crate) struct Compute { + _client_command: mmu::KernelMapping, + objects: KVec, + command_bytes: KVec, + sequence_bytes: KVec, + _binding: mmu::VmBind, + stats: Region, + sequence: u64, + last_progress_ns: i64, + head: u16, + cached_views: KVec, + slot: usize, + batch_count: usize, + checkpoint: (u64,u16,usize), + /// Firmware-only barrier storage, allocated only for mixed publications. + barriers: Option, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers: command::RegisterSet, +} +impl Compute { + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::agx_compute::Control, + registers: command::RegisterSet) -> Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new(); + let mut client_command=None; + let mut command_bytes = KVec::new(); + command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + let mut sequence_bytes = KVec::new(); + sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; + // A private compute aperture, disjoint from the retained render graph. + // Address-bearing graph fields are constructed from their live owners. + for index in 0..storage::COUNT { + let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let mut b=match a.mapping { + Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, + Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, + Mapping::FirmwareGpuShared=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,true)?, + Mapping::FirmwareOnly=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,false)?, + }; + if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} + objects.push(b,GFP_KERNEL)?; + } + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; + UMA_POOL.initialize(&mut objects)?; + Self::initialize_state(&mut objects, binding.slot())?; + // Queue references are firmware VAs from live allocations. In contrast + // to render's unpublished seed entries, compute dispatches entry zero + // immediately, so its initial producer cursor remains one. + let address = |index: usize| queue::FirmwareVa::new(objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { state: address(storage::QUEUE_STATE)?, ring: address(storage::RING)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, + context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; + let command = address(storage::COMMAND)?; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + objects[storage::QUEUE].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } + .encode(&mut state).map_err(|_| EINVAL)?; + objects[storage::QUEUE_STATE].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &[command]).map_err(|_| EINVAL)?; + objects[storage::RING].write(index as usize * 8, &entry)?; + } + dev_info!(dev.as_ref(),"M3 compute graph: context={} queue={:#x} command={:#x} low={:#x} cdm={:#x}..{:#x} pool={:#x}\n", + binding.slot(),objects[storage::QUEUE].va(),objects[storage::COMMAND].va(), + objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); + // The M3 compute stream uses absolute IOTO addresses and does not + // include the USC_EXEC_BASE_CP register. + let mut stream=[0u8;48]; + vm.read_bytes(control.base,&mut stream)?; + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + agx_memory::publish(); + // Finish the write-combining page-table stores before broadcasting + // the invalidation. + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + } + fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, + control: crate::agx_compute::Control, sequence: u64, slot: usize, + register_set: command::RegisterSet) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let preemption = &objects[PREEMPTION]; + let value = command::Command { + context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, + cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, + gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, + stamp_value: timeline::stamp(0xc1000000,sequence), + timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], + pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, + register_set, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) + } + fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, + context: u32, stamp_value: u32, slot: usize) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let value = microsequence::Sequence { + command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, + queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, + fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], + fw_stamp: fw(storage::FW_STAMP)?, stamp_value, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) + } + fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + objects[JOB_LIST].write(0, &list)?; + for (owner, value) in [(storage::COUNTER, 1), (storage::EVENT, 1), (storage::STAMP, timeline::stamp(0xc1000000,0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + objects[owner].write(0, bytes)?; + } + // Fill the preemption buffer with 0xcc, including when a compute + // command is submitted again. + objects[PREEMPTION].fill(0xcc) + } + pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, + control: crate::agx_compute::Control) -> Result { + if !self.complete()? { return Err(EBUSY); } + // All prior flushed stamps/events and engine idle were checked by + // Runtime before entry. Keep pool/queue ownership, remap only client + // views and patch the context when switching Vulkan processes. + if !self._binding.matches(vm) { + let next = if let Some(index) = self.cached_views.iter().position(|v| v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_| EIO)? + } else { + let binding = uat.bind(vm)?; + let mut views = KVec::new(); + for i in 0..storage::COUNT { + if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + let command = self.objects[storage::COMMAND].map_gpu_view(vm)?; + CachedViews {binding, command, views} + }; + // Complete allocations before moving the active views. Eviction + // drops only an idle cached view; no arena storage is duplicated. + let old_views = KVec::with_capacity(next.views.len(), GFP_KERNEL)?; + self.cached_views.reserve(1, GFP_KERNEL)?; + if self.cached_views.len() == 3 { + drop(self.cached_views.remove(0).map_err(|_| EIO)?); + } + let mut old = CachedViews { + binding: core::mem::replace(&mut self._binding, next.binding), + command: core::mem::replace(&mut self._client_command, next.command), + views: old_views, + }; + for (i, view) in next.views { + old.views.push((i, self.objects[i].swap_gpu_view(view)?), GFP_KERNEL)?; + } + self.cached_views.push(old, GFP_KERNEL)?; + } + self.checkpoint=(self.sequence,self.head,self.batch_count); + self.batch_count=0; + if let Err(error)=self.append(control) { + self.abort_batch()?; + return Err(error); + } + Ok(()) + } + /// Append only while the previous batch is fully retired and this one + /// remains unpublished. Firmware serializes each command, including its + /// original completion flush, in this one compute queue. + pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + if self.batch_count>=storage::SLOTS {return Err(E2BIG);} + self.slot=self.batch_count; + for i in storage::RESET { + let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; + } + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; + self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + let stamp=self.stamp(); + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; + self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; + self.batch_count+=1; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok(()) + } + /// Roll back producer state before any SubmitQueue publication. Old ring + /// entries beyond the restored cursor are unreachable and get overwritten. + pub(crate) fn abort_batch(&mut self)->Result { + (self.sequence,self.head,self.batch_count)=self.checkpoint; + self.slot=self.batch_count.saturating_sub(1); + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();Ok(()) + } + fn offset(&self,owner:usize)->Result { + storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + } + /// Same cache-generation contract as M4: StartCompute and the persistent + /// notifier must use the same tag before the firmware flushes user stamps. + /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. + pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + let command=self.offset(storage::COMMAND)?; + self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; + self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + agx_memory::publish();Ok(()) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { + let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + self.objects[storage::COMMAND].u64(offset,addresses[0])?; + self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; + let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; + let sequence=self.offset(storage::MICROSEQUENCE)?; + for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; + } + agx_memory::publish();Ok(()) + } + pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + agx_memory::publish();Ok(()) + } + fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3_sync_layout::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, + value:self.stamp(),event:2, + }) + } + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + if self.barriers.is_none() { + self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); + } + Ok(()) + } + /// Insert a dependency before the last appended, still unpublished CDM. + /// Its command ordinal and timestamp slot are unchanged; only the ring + /// gains an entry. The whole batch must retire before this storage is reused. + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { + return Err(EINVAL); + } + let value=self.stamp(); + let barriers=self.barriers.as_mut().ok_or(EINVAL)?; + let offset=self.slot*64; + let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; + crate::m3_sync_layout::Barrier { + stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, + self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + }.encode(&mut bytes).map_err(|_|EINVAL)?; + barriers.write(offset,&bytes)?; + let address=barriers.va()+offset as u64; + let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; + let command=self.objects[storage::COMMAND].va()+self.offset(storage::COMMAND)? as u64; + self.objects[storage::RING].u64(usize::from(slots[0])*8,address)?; + self.objects[storage::RING].u64(usize::from(slots[1])*8,command)?; + self.head=next; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();Ok(()) + } + pub(crate) fn gpu_ns(&mut self)->Result { + let mut total=0; + for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + Ok(total) + } + pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { + if slot>=self.batch_count {return Err(EINVAL);} + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// A batch span includes gaps between its compute commands, unlike gpu_ns. + /// Read only after stamps, queues, events and idle state prove retirement. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + } + pub(crate) fn head(&self) -> u16 { self.head } + pub(crate) fn ordinal(&self)->u64 {self.sequence} + pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + // Report real completions by elapsed time as well as sequence count; + // batches need not land on a multiple of 128. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.sequence != 1 && self.sequence % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(), "M3_COMPUTE_RETIRED sequence={} context={} stamp={:#x}/{:#x} queue={}/{} head={} idle=1 cache_flush=1\n", + self.sequence, self._binding.slot(), self.objects[storage::STAMP].read_u32(0)?, + self.objects[storage::FW_STAMP].read_u32(0)?, self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?, + self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?, self.head); + Ok(()) + } + + pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} + pub(crate) fn complete(&mut self)->Result { + Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + for off in (0..512).step_by(128) { + let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; + dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); + } + let stamp=self.objects[storage::STAMP].read_u32(0)?; + let fw=self.objects[storage::FW_STAMP].read_u32(0)?; + let done=self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?; + let read=self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?; + dev_info!(dev.as_ref(),"M3 compute: stamps={:#x}/{:#x} done={} read={}\n",stamp,fw,done,read); + let mut record=[0u8;0x84];self.objects[storage::POOL].read(0,&mut record)?; + dev_info!(dev.as_ref(),"M3 compute pool={:02x?}\n",record); + let mut roots=[(0,0);64];self._binding.vm().context_roots(&mut roots)?; + dev_info!(dev.as_ref(),"M3 context {} roots={:x?}, bootstrap={:x?}\n",self._binding.slot(),roots[self._binding.slot() as usize],roots[0]); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs new file mode 100644 index 0000000000000..59277aaefb681 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419.140.12 configuration with named initialization references. +use kernel::prelude::*; +use crate::m3_init_layout as init; +use crate::m3_init_storage as storage; +use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, + UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, + FW_CONTROL_STATE,FW_CONTROL_RING}; +use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); +kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); +kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); + +/// The root, runtime-pointer and firmware-control records, encoded for the owners +/// `region` describes. Config uploads them for its own allocations; the device-tree +/// InitData checks encode them for the fixed allocation table before GPU access. +pub(crate) struct Records { + pub(crate) root: [u8; init::ROOT_SIZE], + pub(crate) runtime: [u8; init::RUNTIME_SIZE], + pub(crate) control: KVec, +} +impl Records { + pub(crate) fn encode(region: impl Fn(usize) -> Result) -> Result { + let root = init::Root { region_a: region(REGION_A)?, runtime: region(RUNTIME_POINTERS)?, + globals: region(GLOBALS)?, new_a: region(RUNTIME_FLAGS)?, + control: region(CONTROL_REGION)?, power: region(GLOBALS_POWER)? }; + let mut channels = [init::Channel { state: region(0)?, ring: region(1)? }; 17]; + for (index, channel) in channels.iter_mut().enumerate() { + *channel = init::Channel { state: region(index * 2)?, ring: region(index * 2 + 1)? }; + } + let runtime = init::RuntimePointers { hardware: region(HARDWARE_DATA)?, + unknown_pair: region(UNKNOWN_PAIR)?, fwlog_payload: region(FWLOG_PAYLOAD)?, + unknown_c0: region(UNKNOWN_C0)?, unknown_c1: region(UNKNOWN_C1)?, + unknown_c3: region(UNKNOWN_C3)?, channels }; + let control = init::ControlRegion { channel: init::Channel { + state: region(FW_CONTROL_STATE)?, ring: region(FW_CONTROL_RING)? } }; + let mut root_bytes = [0; init::ROOT_SIZE]; + root.encode(&mut root_bytes).map_err(|_| EINVAL)?; + let mut runtime_bytes = [0; init::RUNTIME_SIZE]; + runtime.encode(&mut runtime_bytes).map_err(|_| EINVAL)?; + // The control/status region is 50 KiB; use temporary heap storage, + // never a kernel-stack array. All three owners precede firmware startup. + let mut control_bytes = KVec::new(); + control_bytes.resize(init::CONTROL_SIZE, 0u8, GFP_KERNEL)?; + control.encode(&mut control_bytes).map_err(|_| EINVAL)?; + Ok(Self { root: root_bytes, runtime: runtime_bytes, control: control_bytes }) + } +} + +pub(crate) struct Config { + objects: KVec, + #[cfg(CONFIG_DEV_COREDUMP)] + fault_reserve: Option, + _iomaps: KVec, + pub(crate) completed_events: u64, + pstates: crate::m3_adt_config::PstateWatch, + thermal: crate::m3_thermal::Governor, + request_top: u32, + request_value: u32, + request_checked: bool, + request_check_at: Option>, + request_writes: u64, + request_observed: u32, +} +impl Config { + /// Snapshot owned RAM only. Firmware can still update these buffers; + /// per-record time intervals explicitly describe sequential observations. + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { + let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; + dump.begin(0); + let mut cause = [0u8; 16]; + cause[..4].copy_from_slice(&1u32.to_le_bytes()); + cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); + cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); + dump.record("host-first-error", 0, cause.len(), |out| { + out.copy_from_slice(&cause); Ok(()) + })?; + for (name, index) in [ + ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), + ("hardware", HARDWARE_DATA), ("globals", GLOBALS), + ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), + // The power-controller block, and the host-to-firmware device-control and + // firmware-control queues: their state words show whether the firmware read a + // queued control (read index) and what it was sent. + ("power", GLOBALS_POWER), + ("devctl-state", storage::DEVICE_CONTROL * 2), + ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), + ("fwctl-state", storage::FW_CONTROL_STATE), + ("fwctl-ring", storage::FW_CONTROL_RING), + ("event-state", storage::EVENT * 2), + ("event-ring", storage::EVENT * 2 + 1), + ("fwlog-state", storage::FIRMWARE_LOG * 2), + ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), + ("fwlog-payload", FWLOG_PAYLOAD), + ("trace-state", storage::TRACE * 2), + ("trace-ring", storage::TRACE * 2 + 1), + ] { + let buffer = &mut self.objects[index]; + dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; + } + dump.publish(dev.as_ref()) + } + + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, + contents: &crate::m3_adt_config::Contents) -> Result { + contents.check_for_upload()?; + let mut objects = KVec::new(); + let mut iomaps = KVec::new(); + for index in 0..storage::COUNT { + let mut a=storage::allocation(index).map_err(|_|EINVAL)?; + let image=contents.initial(index,&a)?; + if a.initial!=storage::Initial::Zero && image.len()!=a.size {return Err(EINVAL);} + // The SoC may place the HwData object elsewhere and have it mapped cacheable + // (`Soc::hwdata_object`); the records and the upload check use the live address. + let object=if index==HARDWARE_DATA {contents.hwdata_object} else {None}; + if let Some(o)=object {a.address=o.address;} + let mut b=match object { + Some(o) if o.cached=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address),a.size, + prot::PROT_FW_PRIV_RW,prot::PROT_GPU_SHARED_RW)?, + _=>Buffer::at(dev,uat.kernel_vm(),None,Some(a.address),a.size,a.gpu_shared)?, + }; + b.write(0,image)?;objects.push(b,GFP_KERNEL)?; + } + for io in contents.iomaps { + // Read-write unless the contents say otherwise (only the T8122 start experiment). + let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} + else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; + let field=io.pointer_field().map_err(|_|EINVAL)?; + objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; + iomaps.push(map,GFP_KERNEL)?; + } + // This field is a physical address owned by the reserved GPU region, + // unlike the firmware virtual references in the IOMapping records. + objects[HARDWARE_DATA].u64(storage::GPU_REGION_PHYSICAL,firmware.resources.regions[0].base)?; + Self::initialize_records(&mut objects)?; + // The root record carries the G15 14.8.3 InitData version, which is every T6030 + // image's. Only the T8122 start experiment gives the firmware another one + // (`asahi.t8122_initdata_version`). + if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + objects[storage::ROOT].u64(0,firmware.initdata_magic)?; + } + // Vertex/tessellation loops need not retire a primitive at every + // firmware progress poll. The inherited interval of 10 falsely + // declares a valid 35.7 ms TA job stuck on J514S. Keep progress + // detection enabled, with a finite interval of 100 firmware polls. + // This is independent of the host job bound and 60-second watchdog. + // RTKit 2419 copies Globals+0x980 into its TA poll threshold at + // text+0x6514; text+0xd710 checks it before sampling engine progress. + const TA_PROGRESS_INTERVAL: usize = 0x980; + if objects[GLOBALS].read_u32(TA_PROGRESS_INTERVAL)? != 10 { return Err(EINVAL); } + objects[GLOBALS].u32(TA_PROGRESS_INTERVAL, 100)?; + dev_info!(dev.as_ref(), "M3: firmware TA progress-check interval=100\n"); + // RTKit 2419 +2de48..2dec8 accepts user timestamp stores only + // within the 64 MiB arena at HwDataB+28. Match map_timestamp(). + objects[HARDWARE_DATA].u64(0x28,agx_memory::TIMESTAMP_RANGE.start)?; + // RTKit 2419 HwDataA follows the 0x4580-byte HwDataB. Request the + // boot performance state through firmware (asahi.m3_boot_pstate) and cap + // the states it may use (asahi.m3_max_pstate), retaining the voltage + // tables, power/thermal ceilings and minimum operating state. + // The bring-up image initialized actual/target and every base to 1 + // (338 MHz), so short desktop jobs never left that startup state. + // The uploaded table is the SoC's (`Soc::pstates`): its highest state at +0xb58 and that + // state's frequency in the table at +0xb5c (8 and 1380 MHz on T6030). + let (states, top_mhz) = contents.table; + if objects[HARDWARE_DATA].read_u32(0xb58)? != states + || objects[HARDWARE_DATA].read_u32(0xb5c + states as usize * 4)? != top_mhz + { + return Err(EINVAL); + } + let (hwdata, globals) = objects.split_at_mut(GLOBALS); + crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + // Firmware ktrace is for explicit detailed diagnostics. Normal games + // still drain error/events, but need not generate trace records for + // every firmware operation on the command-processing core. + objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( + crate::debug::DebugFlags::SubmitTiming)))?; + crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + agx_memory::publish(); + let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; + #[cfg(CONFIG_DEV_COREDUMP)] + let fault_reserve = match crate::agx_fault::Dump::new_m3() { + Ok(dump) => Some(dump), + Err(error) => { + dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); + None + } + }; + Ok(Self { objects, _iomaps: iomaps, completed_events: 0, + #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, + pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + request_top:contents.table.0,request_value:0,request_checked:false, + request_check_at:None,request_writes:0,request_observed:0 }) + } + fn initialize_records(objects: &mut [Buffer]) -> Result { + let records = Records::encode(|index: usize| { + init::Region::new(objects[index].va(), objects[index].size()).map_err(|_| EINVAL) + })?; + objects[storage::ROOT].write(0, &records.root)?; + objects[RUNTIME_POINTERS].write(0, &records.runtime)?; + objects[CONTROL_REGION].write(0, &records.control) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", + u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + Ok(()) + } + pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { + let Some(value)=crate::m3_consistent_perf::request(chip, + crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + if value==0 && self.request_value==0 {return Ok(());} + let now=kernel::time::Instant::::now(); + let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); + if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + return Ok(()); + } + type P=crate::fw::initdata::raw::PowerCtlBlock; + let at=core::mem::offset_of!(P,consistent_perf_state); + if !self.request_checked { + let mut flag=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut flag)?; + if flag[0]!=0 || self.objects[GLOBALS_POWER].read_u32(at)?!=0 {return Err(EINVAL);} + self.request_checked=true; + } + let observed=self.objects[GLOBALS_POWER].read_u32(at)?; + self.request_observed=observed; + if observed!=value { + self.objects[GLOBALS_POWER].packed_request(at,value)?; + agx_memory::publish(); + self.request_writes=self.request_writes.saturating_add(1); + } + self.request_value=value; + self.request_check_at=Some(now); + Ok(()) + } + /// Move the runtime cap by temperature (the thermal limit), then check the performance + /// state the firmware reports against the ceiling and the runtime cap. Called after boot and + /// after every job, by the job thread. + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + let now = kernel::time::Instant::::now(); + if let Some(cap) = self.thermal.update(dev.as_ref(), now) { + crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + } + let allowed = self.thermal.allowed(now); + let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref(), now, over, peak)?; + self.thermal.busy(now); + Ok(()) + } + pub(crate) fn stats_region(&self) -> Result { + let owner = &self.objects[HARDWARE_DATA]; + init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) + } + + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, + device: &crate::m3_device::Device, + sequence: u32) -> Result { + let cap = self.thermal.allowed(kernel::time::Instant::::now()); + self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + device.pstate_register()?, cap, sequence)?; + type P=crate::fw::initdata::raw::PowerCtlBlock; + type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; + let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; + let power_if=core::mem::offset_of!(G,power_if_target); + let target1=self.objects[GLOBALS].read_u32(power_if+4)?; + let target2=self.objects[GLOBALS].read_u32(power_if+8)?; + let mut override_active=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; + dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", + sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + self.request_writes,self.request_observed,override_active[0]); + Ok(()) + } + + pub(crate) fn render_pb(&mut self) -> Result { + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + } + pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { + if kind>2 {return Err(EINVAL);} + let state=kind*2;let ring=state+1; + let w=self.objects[state].read_u32(0x20)?; + if w>=256 || self.objects[state].read_u32(0)?!=w { return Err(EBUSY); } + let mut message=[0u8;0x18]; + message[8..16].copy_from_slice(&queue.to_le_bytes()); + message[16..20].copy_from_slice(&(kind as u32).to_le_bytes()); + message[20..22].copy_from_slice(&head.to_le_bytes()); + message[22]=event;message[23]=u8::from(new); // New queue; caller selected the event slot. + self.objects[ring].write(w as usize*0x18,&message)?; + agx_memory::publish(); + self.objects[state].u32(0x20,(w+1)%256)?; + agx_memory::publish(); + Ok(()) + } + /// Whether the firmware has consumed every message on pipe `kind` (0 TA, 1 3D, 2 compute). + pub(crate) fn pipe_free(&mut self, kind:usize) -> Result { + if kind>2 {return Err(EINVAL);} + Ok(self.objects[kind*2].read_u32(0)? == self.objects[kind*2].read_u32(0x20)?) + } + pub(crate) fn pipes_idle(&mut self) -> Result { + for state in [0,2,4] { + if self.objects[state].read_u32(0)? != self.objects[state].read_u32(0x20)? { + return Ok(false); + } + } + Ok(true) + } + pub(crate) fn ready(&mut self) -> Result { + // NewRegionB+4 is an input initialized to 1, not an acknowledgement. + // These are firmware-written transitions present in compute-15's + // pre-submit snapshot; both start at zero in the exported schema. + Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd + && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) + } + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; + let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; + let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; + let b=self.objects[RUNTIME_FLAGS].read_u32(0x14)?; + let c=self.objects[CONTROL_REGION].read_u32(0x45c4)?; + dev_info!(dev.as_ref(),"M3 firmware readiness: magic={:#x} runtime={} newA={}/{} newB={}\n",magic,state,a,b,c); + Ok(()) + } + /// The firmware's halt counter and its halted and resume flags. + pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> { + let control=&mut self.objects[CONTROL_REGION]; + Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?)) + } + /// Let a halted firmware continue: clear halted, then set resume. These + /// words keep the halt/resume layout of earlier Apple GPU firmware. + pub(crate) fn resume_halted(&mut self) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + control.u32(0x4590,0)?; + agx_memory::publish(); + control.u32(0x45a0,1)?; + agx_memory::publish(); + Ok(()) + } + /// Read-only snapshot of the firmware's recovery words in the control + /// region (InitData+0xb0): the halt counter and the halted/resume flags + /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information + /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. + /// Nothing is acknowledged, cleared or resumed here. + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + let halt_count=control.read_u64(0x4580)?; + let halted=control.read_u32(0x4590)?; + let resume=control.read_u32(0x45a0)?; + let info=[control.read_u32(0x40a0)?,control.read_u32(0x40a4)?]; + let mut packet=[0u8;0x14]; + control.read(0x44a0,&mut packet)?; + dev_info!(dev.as_ref(),"M3 firmware recovery state: halt_count={} halted={} resume={} info={:#x}/{:#x} packet={:02x?} completed_events={} event_messages={}\n", + halt_count,halted,resume,info[0],info[1],packet,self.completed_events,events); + Ok(()) + } + /// Read-only: log the event ring entries from the host's read index to the firmware's write + /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the + /// completion count is not changed. + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + const SIZE: usize = 0x38; + const COUNT: usize = 256; + let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; + let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize; + dev_info!(dev.as_ref(), "M3 event ring: read={} write={}\n", r, w); + if r >= COUNT || w >= COUNT { return Err(EIO); } + let mut index = r; + for _ in 0..16 { + if index == w { break; } + let mut raw = [0u8; SIZE]; + self.objects[storage::EVENT*2+1].read(index * SIZE, &mut raw)?; + dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n", + index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw); + index = (index + 1) % COUNT; + } + Ok(()) + } + pub(crate) fn control(&mut self, opcode: u32) -> Result { + if !matches!(opcode, 0x13 | 9) { return Err(EINVAL); } + let w = self.objects[storage::DEVICE_CONTROL*2].read_u32(0x20)?; + let r = self.objects[storage::DEVICE_CONTROL*2].read_u32(0)?; + if w >= 256 || r != w { return Err(EBUSY); } + let mut bytes = [0; 0x38]; + bytes[..4].copy_from_slice(&opcode.to_le_bytes()); + self.objects[storage::DEVICE_CONTROL*2+1].write(w as usize * 0x38, &bytes)?; + agx_memory::publish(); + let next = (w + 1) % 256; + self.objects[storage::DEVICE_CONTROL*2].u32(0x20, next)?; + agx_memory::publish(); + Ok(next) + } + pub(crate) fn control_done(&mut self, next: u32) -> Result { + Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) + } + pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + // Event, trace, stats and six independent firmware log rings. + for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), + (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), + (storage::STATS*2,storage::STATS*2+1,0x40,256,1), + (storage::FIRMWARE_LOG*2,storage::FIRMWARE_LOG*2+1,init::fwlog::ENTRY_SIZE,init::fwlog::SLOTS,init::fwlog::CHANNELS)] { + for channel in 0..channels { + let base = channel * 0x30; + let mut r = self.objects[state].read_u32(base)? as usize; + let w = self.objects[state].read_u32(base+0x20)? as usize; + if r >= count || w >= count { return Err(EIO); } + let previous_read = r; + while r != w { + if (state==storage::TRACE*2 && self.completed_events<4) || state==storage::FIRMWARE_LOG*2 { + let mut raw=[0u8;0xd8]; + self.objects[ring].read((channel*count+r)*size,&mut raw[..size])?; + dev_info!(dev.as_ref(),"M3 fw-channel {} {:02x?}\n",state,&raw[..size]); + if state == storage::FIRMWARE_LOG*2 && self.objects[ring].read_u32((channel*count+r)*size)? == 2 { + let index = self.objects[ring].read_u64((channel*count+r)*size+8)?; + let offset = init::fwlog::payload_offset(channel,index).map_err(|_| EIO)?; + let mut payload = [0u8;init::fwlog::PAYLOAD_SIZE]; + self.objects[FWLOG_PAYLOAD].read(offset,&mut payload)?; + dev_info!(dev.as_ref(),"M3 fwlog payload {} {:02x?}\n",channel,payload); + } + } + if state == storage::EVENT*2 { + let kind = self.objects[ring].read_u32(r*size)?; + if kind == 1 { + let flags=self.objects[ring].read_u64(r*size+4)?; + self.completed_events+=u64::from((flags&7).count_ones()); + } + if kind == 0 || kind == 4 { + let mut raw=[0u8;0x38];self.objects[ring].read(r*size,&mut raw)?; + dev_err!(dev.as_ref(), "M3 firmware error event {} bytes={:02x?}\n", kind,raw); + return Err(EIO); + } + } + r = (r+1) % count; + } + // Publish actual consumption only, as RxChannel::get does. + // Empty polls need no write to firmware's shared state line. + if r != previous_read { self.objects[state].u32(base, r as u32)?; } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs new file mode 100644 index 0000000000000..1c2f0b1e8866d --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Owned M3 register apertures. The platform +//! power domain owns TVM/PMGR and stays on across probe/remove. Firmware and +//! queues must be stopped before this object is released. + +use kernel::{ + bindings, c_str, + device::Core, + devres::Devres, + error::to_result, + io::{ + mem::IoMem, + Io, + }, + platform, + prelude::*, + sync::aref::ARef, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use crate::m3_firmware::Firmware; +use crate::m3_soc::Soc; + +const ASC_CPU_CONTROL: usize = 0x44; +const ASC_CPU_RUN: u32 = 1 << 4; + +pub(crate) struct Device { + dev: ARef, + asc: Pin>>>, + sgx: Pin>>, + firmware: Firmware, + core_mask: u64, + soc: &'static Soc, + power_vote: bool, +} + +impl Device { + /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides + /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. + pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>) -> Result { + // Map ASC before taking a vote. The larger SGX aperture contains + // this control window and the separate mailbox provider, so it is + // mapped without a conflicting claim over those child resources. + // Only the 16 KiB control window is used. Some device trees describe + // the ASC block as a larger range that also covers the mailbox, which + // the mailbox driver has already claimed, so do not claim the range. + let asc = KBox::pin_init( + pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); + let sgx = KBox::pin_init( + pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); + let pmp = crate::m3_board::has_pmp_link(pdev); + let t8122 = crate::t8122_start::is_t8122(soc); + let t6031_soc = crate::t6031_start::is_t6031(soc); + if pmp && !t8122 && !t6031_soc { + dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); + return Err(ENOTSUPP); + } + if t8122 && !pmp { + dev_err!(pdev.as_ref(), "M3 G15G: current14 GPU requires its PMP supplier link\n"); + return Err(ENODEV); + } + if pmp { + // SAFETY: the consumer is live; the managed link pins its declared PMP supplier. + to_result(unsafe { bindings::apple_pmp_link_device(pdev.as_ref().as_raw()) })?; + } else { + dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); + } + if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + dev_err!( + pdev.as_ref(), + "M3 {}: ASC already running; refusing to take ownership\n", + soc.gpu_name + ); + return Err(EBUSY); + } + let mut device = Self { + dev: pdev.into(), + asc, + sgx, + firmware, + core_mask: 0, + soc, + power_vote: false, + }; + + if pmp { + // SAFETY: the linked current14 T8122 PMP owns AGX logical device 5. The bridge + // waits for firmware readiness and the power-command acknowledgement. + // An unacknowledged request may still have enabled AGX. Keep ownership so a + // failed acquisition releases it only after the ASC-stopped check in Drop. + device.power_vote = true; + to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; + if t6031_soc { + dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); + } else { + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); + } + } + + let registers = device.sgx.access(pdev.as_ref())?; + let version = registers.try_read32(0xd04000)?; + let counts = registers.try_read32(0xd04010)?; + // The core-enable words: the second only on a SoC with more than 32 core slots. + let mut core_mask = u64::from(registers.try_read32(0xe01500)?); + if crate::m3_board::core_slots(soc) > 32 { + core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; + } + dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", + soc.gpu_name, version, counts, core_mask, device.firmware.version()); + // These words identify the qualified configuration of this SoC. The + // fused core mask varies with SKU; absent cores must never be enabled. + if version & soc.id.version_mask != soc.id.version + || counts & soc.id.counts_mask != soc.id.counts + || !crate::m3_board::core_mask_valid(soc, core_mask) + { + return Err(ENODEV); + } + let setup = match experiment { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!(pdev.as_ref(), + "M3 G15G start: SGX setup write: SGX+{:#x} = {:#x} (asahi.t8122_sgx_setup=t6030)\n", offset, value), + None => dev_info!(pdev.as_ref(), + "M3 G15G start: no SGX setup write (asahi.t8122_sgx_setup=none)\n"), + } + e.sgx_setup() + } + None => match t6031 { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!( + pdev.as_ref(), + "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", + offset, + value + ), + None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), + } + e.sgx_setup() + } + None => Some(soc.sgx_setup.ok_or(ENODEV)?), + }, + }; + if let Some((setup_offset, setup_value)) = setup { + registers.try_write32(setup_value, setup_offset)?; + } + device.core_mask = core_mask; + Ok(device) + } + + pub(crate) fn core_mask(&self) -> u64 { self.core_mask } + + /// The SoC table this device was admitted with. + pub(crate) fn soc(&self) -> &'static Soc { self.soc } + + pub(crate) fn check_idle(&self)->Result { + self.check_idle_parts(true,true) + } + /// [`Self::check_idle`] with its engine-busy (`busy`) and fault-bank (`faults`) register + /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. + pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { + if !busy && !faults {return Ok(());} + let sgx=self.sgx.try_access().ok_or(ENODEV)?; + if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { + return Err(EBUSY); + } + if !faults {return Ok(());} + let selector=sgx.try_read64(0xd800)?; + let mut fault=0; + // One fault bank (selected through 0xd800) per GPU cluster: banks 0 and 1 on the + // two-cluster T6030, bank 0 alone on the single-cluster T8122. Never select a bank past + // the SoC's clusters. + for bank in 0..u64::from(self.soc.clusters) { + sgx.try_write64(bank,0xd800)?;fault|=sgx.try_read64(0xd8c0)?; + } + sgx.try_write64(selector,0xd800)?; + if fault!=0 {return Err(EIO);} Ok(()) + } + + pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { + use kernel::dma::{Device as _, DmaMask}; + // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + crate::mmu::check_handoff_guard()?; + dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); + for _ in 0..2 { + let drm: ARef = kernel::drm::Device::new( + pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; + if drm.gpu().is_ok() { return Err(EIO); } + let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; + object.vmap()?.memset(0); + drop(object); + drop(drm); + } + dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); + Ok(()) + } + + /// The raw performance-state register (SGX+0xe01000). + pub(crate) fn pstate_register(&self) -> Result { + self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + } + + /// Read-only snapshot. Do not select or acknowledge fault banks here; + /// the firmware owns those registers until recovery is ported. + pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let pstate = sgx.try_read32(0xe01000)? & 0xf; + if pstate == 0 { + dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); + return Ok(()); + } + dev_info!(self.dev.as_ref(), "M3 {}: reading engine/MMU snapshot at pstate={}\n", self.soc.gpu_name, pstate); + let selector=sgx.try_read64(0xd800)?; + for bank in [0,1,0x100,0x40000,0x40001,0x80000,0xc0000] { + sgx.try_write64(bank,0xd800)?; + let address=sgx.try_read64(0xd8c8)?; + let fault=sgx.try_read64(0xd8c0)?; + dev_info!(self.dev.as_ref(),"M3 bank {} fault={:#x} address_word={:#x} address={:#x}\n",bank,fault,address,address<<6); + if fault&1!=0 { + dev_info!(self.dev.as_ref(),"M3 fault mapping bank={} address={:#x} translation={:?}\n",bank,address<<6,vm.translate_iova(address<<6)); + } + } + sgx.try_write64(selector,0xd800)?; + // Read-only per-core service/debug snapshot, using the same + // core-selector protocol as the idle check. + let service=sgx.try_read32(0xa010)?; + let debug=sgx.try_read32(0xa000)?; + let mut cores=u64::from(sgx.try_read32(0xe01500)?); + if crate::m3_board::core_slots(self.soc) > 32 { + cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; + } + for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + if cores & (1u64< &Firmware { &self.firmware } + + pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { + if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + return Err(EBUSY); + } + Ok(()) + } + + pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + self.require_stopped(pdev)?; + let asc = self.asc.access(pdev.as_ref())?; + asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); + Ok(()) + } + + pub(crate) fn stop_asc(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); + let start = Instant::::now(); + while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if start.elapsed() >= Delta::from_millis(100) { return Err(ETIMEDOUT); } + fsleep(Delta::from_micros(10)); + } + Ok(()) + } + +} + +impl Drop for Device { + fn drop(&mut self) { + if !self.power_vote { + return; + } + // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. + match self.asc.try_access() { + Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + _ => { + dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); + return; + } + } + // SAFETY: the managed consumer link keeps the PMP bound through device removal. + if let Err(error) = to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 0) }) { + dev_err!(self.dev.as_ref(), "M3 G15G: PMP GPU power release failed: {:?}\n", error); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs new file mode 100644 index 0000000000000..25e65c6599084 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 through the common Asahi GEM, GPUVM and render-node frontend. +use core::ops::Range; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; +use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; + +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); +impl kernel::debugfs::Writer for ProgressView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); + writeln!(f, "version=1 generation_ns={} completed={} last_completion_ns={} healthy={}", + generation, completed, last_ns, u8::from(healthy)) + } +} +// Separate endpoint preserves the strict completion-v1 ABI for old guards. +struct ActivityView(Arc); +impl kernel::debugfs::Writer for ActivityView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, epoch, healthy) = self.0.activity_snapshot(); + writeln!(f, "version=1 generation_ns={} epoch={} pending={} healthy={}", + generation, epoch, epoch & 1, u8::from(healthy)) + } +} + +struct TimingView(Arc); +impl kernel::debugfs::Writer for TimingView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, _, _, healthy) = self.0.progress_snapshot(); + self.0.timing.write(f, generation, healthy) + } +} + +// Root-only writable debugfs policy; no mutable module-parameter references. +// Keep ownership independent of the runtime lock and remove the file on stop. +struct CompletionView(Arc); +impl kernel::debugfs::Writer for CompletionView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); + writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + us, irqs, sleeps, timers, preparations) + } +} +impl kernel::debugfs::Reader for CompletionView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len = reader.len(); + if len == 0 || len > 8 { return Err(EINVAL); } + let mut bytes = [0u8; 8]; + reader.read_slice(&mut bytes[..len])?; + let text = core::str::from_utf8(&bytes[..len]).map_err(|_| EINVAL)?; + let us = text.trim().parse::().map_err(|_| EINVAL)?; + self.0.set_wait_us(us) + } +} + +struct SubmitOverlapView(Arc); +impl kernel::debugfs::Writer for SubmitOverlapView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "{}", u8::from(self.0.cpu_overlap())) + } +} +impl kernel::debugfs::Reader for SubmitOverlapView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len=reader.len(); + if len==0 || len>2 {return Err(EINVAL);} + let mut bytes=[0u8;2];reader.read_slice(&mut bytes[..len])?; + let text=core::str::from_utf8(&bytes[..len]).map_err(|_|EINVAL)?.trim(); + match text { + "0" => self.0.set_cpu_overlap(false), + "1" => self.0.set_cpu_overlap(true), + _ => return Err(EINVAL), + } + Ok(()) + } +} + +pub(crate) struct Registered { + registration: Pin>>>>, + shared: Shared, + health: Arc, + _progress: Pin>>, + _activity: Pin>>, + _memory: Pin>>, + _timing: Pin>>, + _completion: Pin>>, + _overlap: Pin>>, +} +impl Registered { + pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { + let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::t8122_start::not_admitted(pdev.as_ref(), soc); + crate::t6031_start::not_admitted(pdev.as_ref(), soc); + })?; + dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); + // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, + // or an unarmed T8122 or T6031, needs a complete table. + let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; + let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() && t6031.is_none() { + soc.require_complete(pdev.as_ref())?; + } + let experiment = experiment.as_ref(); + let t6031 = t6031.as_ref(); + let refused = |stage: &'static str| { + move |e: &Error| { + crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); + crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); + } + }; + let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + .inspect_err(refused("the firmware identity"))?; + dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); + // InitData source and contents, before any GPU register access. + let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + .inspect_err(refused("the InitData contents"))?; + let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + .inspect_err(refused("the GPU registers and identity"))?; + device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; + let core_mask = device.core_mask(); + let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); + let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + runtime.boot(pdev)?; + dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", + u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), + u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); + let drm = runtime.drm(); + let health = runtime.health(); + // Health holds no device references. The read-only file owns an Arc + // and is removed with registration; teardown latches failure first. + let directory = kernel::debugfs::Dir::new(c"asahi-m3"); + let progress = KBox::pin_init(directory.read_only_file(c"progress", + ProgressView(health.clone())), GFP_KERNEL)?; + let activity = KBox::pin_init(directory.read_only_file(c"activity", + ActivityView(health.clone())), GFP_KERNEL)?; + let memory = KBox::pin_init(directory.read_only_file(c"memory", + crate::agx_memory_stats::View), GFP_KERNEL)?; + let timing = KBox::pin_init(directory.read_only_file(c"timing", + TimingView(health.clone())), GFP_KERNEL)?; + let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", + CompletionView(runtime.completion_wait())), GFP_KERNEL)?; + let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", + SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; + let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + return Err(EBUSY); + } + let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; + let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; + let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), + core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; + if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } + if crate::m3_board::expose_render_node(soc) { + *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + } else { + dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); + } + Ok(owner) + } + pub(crate) fn stop(&self) { + self.health.mark_failed(); + drop(self.registration.lock().take()); + drop(self.shared.lock().take()); + } +} +impl Drop for Registered { fn drop(&mut self) { self.stop(); } } +struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, + scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, + soc: &'static crate::m3_soc::Soc } +impl DrmGpu for Backend { + fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { !self.health.healthy() } + fn update_globals(&self) {} + + fn supports_vm_status(&self) -> bool { true } + fn supports_scheduled_queues(&self)->bool {true} + fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } + fn params(&self) -> Result { + let soc = self.soc; + let masks = crate::m3_board::core_masks(soc, self.core_mask); + Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, + gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, + chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, + num_cores_per_cluster: soc.cores_per_cluster, + core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, + max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) + } + fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } + fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + fn base_clock_hz(&self) -> u32 { + let value: u64; + // SAFETY: read-only architectural counter frequency. + unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; + value as u32 + } + fn new_vm(&self, range: Range) -> Result { + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) + } + fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, + _ualloc_priv: Arc>, priority: u32, + usc_exec_base: u64) -> Result> { + Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + self.soc.features.fragment_dependency)?,GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs new file mode 100644 index 0000000000000..9ae509ad959c4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use crate::m3_resources as agx_resources; + +/// A GPU firmware image the runtime can identify. +#[derive(Debug)] +pub(crate) struct KnownImage { + /// Human-readable identity, for logs. + pub(crate) name: &'static str, + uuid: Option<[u8; 16]>, + stkg_sha256: Option<[u8; 32]>, + /// The validated InitData magic (InitData+0), or `None` until runtime admission is supported. + pub(crate) initdata_magic: Option, +} + +impl KnownImage { + /// Runtime admission requires a validated InitData version and all recorded identities. + pub(crate) fn accepts_runtime(&self, uuid: Option<[u8; 16]>, digest: &[u8; 32]) -> bool { + self.initdata_magic.is_some() + && (self.uuid.is_some() || self.stkg_sha256.is_some()) + && self.uuid.map_or(true, |known| uuid == Some(known)) + && self.stkg_sha256.map_or(true, |known| known == *digest) + } +} + +/// T6030 GPU firmware images known to the M3 runtime. An image is accepted when every recorded +/// identity (UUID, hash) matches. +pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ + KnownImage { + name: "J514S RTKit-2419.140.12", + uuid: None, + stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, + KnownImage { + name: "g15s build b0 (firmware 14.8.3) RTKit-2419.140.12", + uuid: Some([ + 0xdb, 0xf3, 0x7c, 0x40, 0xea, 0xd5, 0x37, 0x60, 0x94, 0x41, 0x99, 0x50, + 0x18, 0x78, 0x29, 0x55, + ]), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, +]; + +/// The G15 InitData version of firmware-compat 14.8.3: the version of both T6030 images and of +/// the T8122 image. +pub(crate) const G15_V14_8_3_INITDATA: u64 = 0x0c08_e21e_8380_0490; + +/// The image-info UUID of the T8122 C0 firmware-compat 14.8.3 image. +const T8122_C0_UUID: [u8; 16] = [ + 0xdf, 0x69, 0x7f, 0x05, 0xf6, 0xb5, 0x33, 0xef, 0xa1, 0x37, 0x61, 0xc4, 0xa7, 0x3d, 0x66, 0x6a, +]; + +/// T8122 C0, firmware-compat 14.8.3: the image the macOS 14.8.3 system firmware loads on the +/// M3 MacBook Airs. It takes the G15 14.8.3 InitData version. +pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +/// The same T8122 image for the start experiment (`t8122_start`, only with +/// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version. +pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3 (start experiment)", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +const IMAGE_INFO_OFFSET: usize = 0x4200; +/// Size of the identifying part of the image-info header. +const IMAGE_INFO_SIZE: usize = 0x38; +/// First word of the header: a branch over it (`b +0x44`). +const IMAGE_INFO_BRANCH: u32 = 0x1400_0011; +/// Header magic, "uuid". +const IMAGE_INFO_MAGIC: u32 = 0x6469_7575; + +/// Parsed image-info header: UUID, patchbay offset and size, TEXT size. +pub(crate) struct ImageInfo { + pub(crate) uuid: [u8; 16], + pub(crate) patchbay: core::ops::Range, + pub(crate) text_size: usize, +} + +pub(crate) fn image_info(text: &[u8]) -> Option { + let hdr = text.get(IMAGE_INFO_OFFSET..IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE)?; + let word = |i: usize| u32::from_le_bytes([hdr[i], hdr[i + 1], hdr[i + 2], hdr[i + 3]]); + if word(0) != IMAGE_INFO_BRANCH || word(4) != IMAGE_INFO_MAGIC || word(8) != 5 { + return None; + } + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&hdr[0x14..0x24]); + let start = word(0x2c) as usize; + let end = start.checked_add(word(0x30) as usize)?; + if word(0x34) as usize != text.len() { + return None; + } + text.get(start..end)?; + Some(ImageInfo { + uuid, + patchbay: start..end, + text_size: word(0x34) as usize, + }) +} + +/// The layout of a loaded GPU firmware the runtime accepts: segment sizes and VAs, and the +/// per-boot words of the text segment that identification ignores. +#[derive(Debug)] +pub(crate) struct Layout { + pub(crate) text_size: u64, + pub(crate) data_size: u64, + pub(crate) vas: [u64; 2], + /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing. + pub(crate) entropy_tag_offset: usize, + pub(crate) entropy_tag: [u8; 8], + /// The firmware version, for logs. + pub(crate) version: &'static str, +} + +pub(crate) static T6030_LAYOUT: Layout = Layout { + text_size: 0x5c000, + data_size: 0x114000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000], + entropy_tag_offset: 0x58e88, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// T8122 firmware-compat 14.8.3 segment sizes, virtual addresses and entropy-tag bounds. +pub(crate) static T8122_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0xb0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0x5d54a, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 +/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither +/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); +/// an armed start searches the loaded text for the tag. The loaded image's version string is +/// not known ("unknown" stands for it). +pub(crate) const T6031_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0x14_0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "unknown", +}; + +pub(crate) const TEXT_SHA256: [u8; 32] = [ + 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; + +fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool { + let tag = layout.entropy_tag_offset; + if text.len() != layout.text_size as usize { + return false; + } + let Some(end) = tag.checked_add(16) else { return false; }; + let Some(entropy) = text.get_mut(tag..end) else { return false; }; + if entropy[..8] != layout.entropy_tag { + return false; + } + entropy[8..].fill(0); + true +} + +#[derive(Debug)] +pub(crate) struct Firmware { + pub(crate) resources: agx_resources::Resources, + /// The InitData version validated for the identified image. + pub(crate) initdata_magic: u64, + /// The accepted layout. + pub(crate) layout: &'static Layout, +} + +impl Firmware { + pub(crate) const fn version(&self) -> &'static str { + self.layout.version + } + + fn matching_layout(layout: &Layout, resources: &agx_resources::Resources) -> bool { + resources.regions[4].size == layout.text_size + && resources.regions[5].size == layout.data_size + && resources.firmware_vas == layout.vas + } +} + +#[cfg(not(test))] +pub(crate) fn identify_loaded( + pdev: &kernel::platform::Device, + soc: &'static crate::m3_soc::Soc, + resources: agx_resources::Resources, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> kernel::error::Result { + use kernel::{ + bindings, c_str, + io::mem::{Mem, MemFlag}, + prelude::*, + }; + + let layout = soc.firmware.ok_or(ENODEV)?; + if !Firmware::matching_layout(layout, &resources) { + dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); + return Err(ENODEV); + } + let node = pdev.as_ref().of_node().ok_or(ENODEV)?; + let text = agx_resources::reserved_resource(&node, c_str!("fw-text"))?; + if text.start() != resources.regions[4].base || text.size() != layout.text_size { + return Err(EINVAL); + } + let mapping = unsafe { Mem::try_new(text, MemFlag::WB.into()) }?; + let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; + let mut canonical = KVec::new(); + canonical.extend_from_slice(bytes, GFP_KERNEL)?; + // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is + // a placeholder, so the T8122 image table must not see this image. + if let Some(experiment) = t6031 { + experiment.normalize_text(&mut canonical)?; + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the synchronous + // SHA-256 call. Normalization never writes the loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + experiment.admit_image(pdev.as_ref(), bytes, &digest)?; + return Ok(Firmware { + resources, + initdata_magic: experiment.initdata_version(), + layout, + }); + } + if !normalize_boot_entropy(layout, &mut canonical) { + return Err(ENODEV); + } + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the + // synchronous SHA-256 call. Normalization never writes loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + // The T8122 start experiment identifies the same image, with its own InitData version. + let images = match experiment { + Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None => soc.images, + }; + let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let initdata_magic = match experiment { + Some(experiment) => experiment.initdata_version(), + None => image.initdata_magic.ok_or(ENODEV)?, + }; + Ok(Firmware { resources, initdata_magic, layout }) +} + +#[cfg(test)] +mod tests { + use super::*; + use agx_resources::{Region, Resources}; + + fn image_header(uuid: [u8; 16]) -> Vec { + let mut text = vec![0; T6030_LAYOUT.text_size as usize]; + let size = text.len() as u32; + for (offset, value) in [ + (0, IMAGE_INFO_BRANCH), (4, IMAGE_INFO_MAGIC), (8, 5), + (0x2c, size - 32), (0x30, 32), (0x34, size), + ] { + header_word(&mut text, offset, value); + } + text[IMAGE_INFO_OFFSET + 0x14..IMAGE_INFO_OFFSET + 0x24].copy_from_slice(&uuid); + text + } + + fn header_word(text: &mut [u8], offset: usize, value: u32) { + let start = IMAGE_INFO_OFFSET + offset; + text[start..start + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn admits_text(image: &KnownImage, text: &[u8], digest: &[u8; 32]) -> bool { + image.accepts_runtime(image_info(text).map(|info| info.uuid), digest) + } + + #[test] + fn uuid_admission_requires_header_bounds_and_exact_text_size() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + let size = text.len() as u32; + for (offset, value) in [ + (0x34, 0), (0x34, size - 1), (0x34, size + 1), + (0x2c, size), (0x2c, u32::MAX), (0x30, u32::MAX), + ] { + let mut bad = text.clone(); + header_word(&mut bad, offset, value); + assert!(!admits_text(image, &bad, &[0; 32]), "offset {offset:#x}, {value:#x}"); + } + let info = image_info(&text).unwrap(); + assert_eq!(info.patchbay, text.len() - 32..text.len()); + assert_eq!(info.text_size, text.len()); + } + + #[test] + fn uuid_admission_rejects_bad_header_words_and_truncation() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + for offset in [0, 4, 8] { + let mut bad = text.clone(); + header_word(&mut bad, offset, 0); + assert!(!admits_text(image, &bad, &[0; 32])); + } + for len in [0, IMAGE_INFO_OFFSET, IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE - 1, text.len() - 1] { + assert!(!admits_text(image, &text[..len], &[0; 32])); + } + } + + #[test] + fn runtime_admission_requires_every_recorded_identity() { + let image = KnownImage { + name: "test", uuid: Some([1; 16]), stkg_sha256: Some([2; 32]), + initdata_magic: Some(3), + }; + let text = image_header([1; 16]); + assert!(admits_text(&image, &text, &[2; 32])); + assert!(!admits_text(&image, &text, &[0; 32])); + assert!(!admits_text(&image, &image_header([0; 16]), &[2; 32])); + assert!(!image.accepts_runtime(None, &[2; 32])); + let anonymous = KnownImage { uuid: None, stkg_sha256: None, ..image }; + assert!(!admits_text(&anonymous, &text, &[2; 32])); + } + + #[test] + fn digest_only_admission_preserves_exact_digest_requirement() { + let image = &KNOWN_IMAGES[0]; + assert!(admits_text(image, &[], &TEXT_SHA256)); + let mut wrong = TEXT_SHA256; + wrong[0] ^= 1; + assert!(!admits_text(image, &[], &wrong)); + } + + #[test] + fn t8122_admission_requires_qualified_initdata_and_matching_image_identity() { + let image = &KNOWN_IMAGES_T8122[0]; + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + let text = image_header(image.uuid.unwrap()); + assert_eq!(image_info(&text).unwrap().uuid, image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + + // A qualified version does not admit an unknown or another chip's image. + for wrong_uuid in [[0; 16], KNOWN_IMAGES[1].uuid.unwrap()] { + assert!(!admits_text(image, &image_header(wrong_uuid), &[0; 32])); + } + assert!(!image.accepts_runtime(None, &[0; 32])); + + // Matching identity still cannot admit an unqualified InitData layout. + let unqualified = KnownImage { + name: image.name, uuid: image.uuid, stkg_sha256: image.stkg_sha256, + initdata_magic: None, + }; + assert!(!admits_text(&unqualified, &text, &[0; 32])); + let unidentified = KnownImage { + name: "unknown", uuid: None, stkg_sha256: None, + initdata_magic: image.initdata_magic, + }; + assert!(!admits_text(&unidentified, &text, &[0; 32])); + } + + #[test] + fn t8122_experiment_record_identifies_the_same_image_with_the_g15_version() { + let image = &KNOWN_IMAGES_T8122_EXPERIMENT[0]; + assert_eq!(image.uuid, KNOWN_IMAGES_T8122[0].uuid); + assert_eq!(image.stkg_sha256, None); + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + assert!(KNOWN_IMAGES.iter().all(|i| i.initdata_magic == Some(G15_V14_8_3_INITDATA))); + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + assert!(!admits_text(image, &image_header(KNOWN_IMAGES[1].uuid.unwrap()), &[0; 32])); + assert!(!admits_text(&KNOWN_IMAGES[1], &text, &[0; 32])); + } + + #[test] + fn each_layout_rejects_the_other_chips_segments() { + for (layout, other) in [(&T6030_LAYOUT, &T8122_LAYOUT), (&T8122_LAYOUT, &T6030_LAYOUT)] { + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let mut resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(other, &resources)); + resources.firmware_vas[1] += 0x4000; + assert!(!Firmware::matching_layout(layout, &resources)); + } + } + + #[test] + fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { + let layout = &T6031_LAYOUT; + assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); + assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); + assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); + assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); + } + + #[test] + fn entropy_normalization_changes_only_the_eight_boot_bytes() { + for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { + let mut text = vec![0xa5; layout.text_size as usize]; + let tag = layout.entropy_tag_offset; + text[tag..tag + 8].copy_from_slice(&layout.entropy_tag); + let before = text.clone(); + assert!(normalize_boot_entropy(layout, &mut text)); + assert_eq!(&text[..tag + 8], &before[..tag + 8]); + assert_eq!(&text[tag + 8..tag + 16], &[0; 8]); + assert_eq!(&text[tag + 16..], &before[tag + 16..]); + } + } + + #[test] + fn bad_entropy_bounds_tags_and_lengths_do_not_mutate_text() { + for offset in [8, 16, usize::MAX] { + let layout = Layout { text_size: 16, entropy_tag_offset: offset, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + } + let layout = Layout { text_size: 16, entropy_tag_offset: 0, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + text.pop(); + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 15]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs new file mode 100644 index 0000000000000..cbfb771fd0873 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned firmware/engine buffers with the qualified M3 page offsets. +use kernel::prelude::*; +use crate::{driver, gem, mmu, pgtable::prot}; + +enum Backing { + Coherent(kernel::dma::Coherent<[u8]>), + Paged(gem::ObjectRef), +} + +pub(crate) struct Buffer { + mapping: mmu::KernelMapping, + gpu_mapping: Option, + // Every client view must preserve the owner's cache and access policy. + // A VM switch changes the address space, not the backing's attributes. + gpu_protection: prot::Prot, + object: Backing, + offset: usize, + size: usize, + _coherent_allocation: Option, +} + +impl Buffer { + pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + gpu_shared: bool) -> Result { + Self::at_prot(dev,fw,gpu,address,size, + if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, + prot::PROT_GPU_SHARED_RW) + } + pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + protection: prot::Prot, gpu_protection: prot::Prot) -> Result { + let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; + if size==0 || gpu.is_some_and(|(_,a)| a&0x3fff!=offset as u64) {return Err(EINVAL);} + let allocated=(size.checked_add(offset).and_then(|n|n.checked_add(0x3fff)).ok_or(EOVERFLOW)?)&!0x3fff; + // The DMA core tries the default CMA area first and falls back to the + // page allocator. The graphs outgrow that area, so a CMA miss is the + // normal case here and is not reported; only a failed fallback is. + // SAFETY: M3 runtime owns the bound platform device for the entire + // diagnostic; all mappings retire before normal DMA allocation Drop. + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; + let pa:usize=object.dma_handle().try_into()?; + let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, + _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + } + /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the + /// old allocation mapped until the replacement has been fully allocated. + pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} + // Large render targets can require 128 MiB of tiler storage. Engines + // and firmware address it through UAT, so physically contiguous DMA + // backing is unnecessary and can exceed the entire CMA reservation. + // GEM provides zeroed pages and the same WC CPU/device coherency used + // by M4. Keep both mappings ahead of their backing in drop order. + let mut object=gem::new_kernel_object_wc(dev,size)?; + let mapping=object.map_into_range(fw,0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000, + mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; + // Growth retains the cached GPU-only policy of the original TPC and + // tilemap allocations; the firmware alias remains shared/uncached. + let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, + mmu::UAT_PGSZ as u64,gpu_protection,true)?); + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) + } + pub(crate) fn va(&self)->u64 {self.mapping.iova()+self.offset as u64} + pub(crate) fn gpu_va(&self)->Result {Ok(self.gpu_mapping.as_ref().ok_or(EINVAL)?.iova()+self.offset as u64)} + pub(crate) fn size(&self)->usize {self.size} + pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { + let address=self.gpu_va()?&!0x3fff; + match &mut self.object { + Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), + } + } + /// The backing retains its CPU mapping until Buffer is dropped. Callers + /// check the range and do not let this pointer escape the current access. + fn cpu_ptr(&mut self)->Result<*mut u8> { + match &mut self.object { + Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), + Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + } + } + + pub(crate) fn swap_gpu_view(&mut self, mapping: mmu::KernelMapping)->Result { + self.gpu_mapping.replace(mapping).ok_or(EINVAL) + } + pub(crate) fn write(&mut self,offset:usize,data:&[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + unsafe {core::ptr::copy_nonoverlapping(data.as_ptr(),ptr.add(self.offset+offset),data.len())}; Ok(()) + } + /// Initialize the complete owned span through its retained CPU mapping. + /// The caller must own the storage or have retired all device consumers. + pub(crate) fn fill(&mut self, value: u8) -> Result { + self.fill_range(0,self.size,value) + } + /// Reset one unpublished slot without touching other queued device owners. + pub(crate) fn fill_range(&mut self,offset:usize,size:usize,value:u8)->Result { + if offset.checked_add(size).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr = self.cpu_ptr()?; + // SAFETY: allocation validates offset + size against its backing, and + // cpu_ptr retains that entire coherent/WC mapping for this access. + // Resolve the mapping once, as the G16 buffer clear path does, instead + // of repeating cached_vmap and memcpy for every 256-byte chunk. + unsafe { core::ptr::write_bytes(ptr.add(self.offset+offset), value, size) }; + Ok(()) + } + pub(crate) fn u32(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, bounded WC GEM storage retained by this owner. + unsafe {ptr.add(start).cast::().write_volatile(value.to_le())};Ok(()) + } + pub(crate) fn u64(&mut self,offset:usize,value:u64)->Result {self.write(offset,&value.to_le_bytes())} + /// Publish the unaligned performance-request word with bounded byte stores. + pub(crate) fn packed_request(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked span in retained CPU storage; no aligned access is required. + unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + Ok(()) + } + pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked WC GEM span; firmware may change the shared bytes. + unsafe { + core::arch::asm!("dmb osh",options(nostack,preserves_flags)); + for (i,b) in data.iter_mut().enumerate() {*b=ptr.cast_const().add(self.offset+offset+i).read_volatile();} + } Ok(()) + } + pub(crate) fn read_u32(&mut self,offset:usize)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {let mut b=[0;4];self.read(offset,&mut b)?;return Ok(u32::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, checked atomic producer/consumer word in WC RAM. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u32::from_le(value)) + } + pub(crate) fn read_u64(&mut self,offset:usize)->Result { + if offset.checked_add(8).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + // Packed firmware records also contain unaligned u64 fields. Retain + // their byte-wise access rather than issuing an unaligned word load. + if start&7!=0 {let mut b=[0;8];self.read(offset,&mut b)?;return Ok(u64::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: retained page-aligned coherent/WC backing, checked aligned + // eight-byte span. One volatile word load avoids eight separate reads + // and byte-wise tearing of an aligned firmware timestamp/counter. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u64::from_le(value)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs new file mode 100644 index 0000000000000..d34e1f23840eb --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters of the T6030 (M3) backends. +//! +//! The `module!` parameter support only parses plain integers. The parameters here are declared +//! with their own `kernel_param` entries (the same layout the `module!` macro emits), so they can +//! take symbolic values on the kernel command line, e.g. `asahi.m3_backend=manager`, and hex or +//! decimal masks. + +use core::ffi::{ + c_char, + c_int, +}; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + device::Core, + platform, + prelude::*, // +}; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. +fn parse_u64(text: &str) -> Option { + let text = text.trim(); + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) => u64::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +/// Store the parsed value of a parameter string into the `AtomicU64` behind `kp`. +/// +/// # Safety +/// +/// `val` must be NULL or a NUL-terminated string, and `kp` must be a `kernel_param` declared by +/// [`m3_param!`], whose `arg` points at a static `AtomicU64`. +unsafe fn set_param( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + parse: fn(&str) -> Option, +) -> c_int { + if val.is_null() { + return EINVAL.to_errno(); + } + // SAFETY: `val` is a NUL-terminated string per the function contract. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(val) }.to_str() else { + return EINVAL.to_errno(); + }; + let Some(value) = parse(text) else { + return EINVAL.to_errno(); + }; + // SAFETY: `arg` points at the static `AtomicU64` this entry was declared with. + let slot = unsafe { &*(*kp).__bindgen_anon_1.arg.cast::() }; + slot.store(value, Ordering::Relaxed); + 0 +} + +/// Declare a module parameter `asahi.` stored in the static `AtomicU64` `$storage` and +/// parsed by `$parse: fn(&str) -> Option`. Not visible in sysfs. +macro_rules! m3_param { + ($name:literal, $storage:ident, $parse:path) => { + m3_param!($name, $storage, $parse, 0, None); + }; + ($name:literal, $storage:ident, $parse:path, $perm:expr, $get:expr) => { + const _: () = { + unsafe extern "C" fn set( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + ) -> c_int { + // SAFETY: Called by the parameter parser with a value string (or NULL) and the + // `kernel_param` entry declared below. + unsafe { set_param(val, kp, $parse) } + } + + static OPS: kernel::bindings::kernel_param_ops = kernel::bindings::kernel_param_ops { + flags: 0, + set: Some(set), + get: $get, + free: None, + }; + + #[link_section = "__param"] + #[used(compiler)] + static PARAM: kernel::module_param::KernelParam = + kernel::module_param::KernelParam::new(kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!($name) + } else { + kernel::c_str!(concat!("asahi.", $name)) + }), + // `__this_module` is constructed by the kernel at load time and is not + // freed until the module is unloaded. Take its address without creating a + // reference to the mutable static. + #[cfg(MODULE)] + mod_: (&raw const kernel::bindings::__this_module).cast_mut(), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&OPS), + perm: $perm, + level: -1, + flags: 0, + __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&$storage).cast_mut().cast(), + }, + }); + }; + }; +} + +/// Read an atomic parameter without borrowing storage that a sysfs write may +/// change concurrently. The module-parameter core supplies a PAGE_SIZE buffer. +/// +/// # Safety +/// +/// `buffer` must be the module-parameter core's writable output buffer and +/// `kp` must be an entry declared by `m3_param!` with AtomicU64 storage. +unsafe extern "C" fn get_atomic_param( + buffer: *mut c_char, + kp: *const kernel::bindings::kernel_param, +) -> c_int { + use core::fmt::Write; + + // SAFETY: The entry points to static AtomicU64 storage by contract. Loads + // and stores are atomic; module-parameter serialization alone would not + // protect the scheduler's concurrent reads. + let value = unsafe { &*((*kp).__bindgen_anon_1.arg.cast::()) } + .load(Ordering::Relaxed); + // SAFETY: 32 bytes fit in the core's PAGE_SIZE output buffer and hold any + // decimal u64 plus its NUL. The core appends the sysfs newline itself. + let bytes = unsafe { core::slice::from_raw_parts_mut(buffer.cast::(), 32) }; + let mut output = kernel::str::Formatter::new(bytes); + if write!(output, "{}\0", value).is_err() { + return EINVAL.to_errno(); + } + (output.bytes_written() - 1) as c_int +} + +fn parse_render_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) +} + +/// Runtime tuning is opt-in: zero keeps the existing boot parameter and its +/// clamping semantics. Only root may write a bounded 1..=SLOTS override; an +/// invalid write leaves the previous value intact. Compute batching is separate. +static M3_RENDER_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_batch_override", M3_RENDER_BATCH_OVERRIDE, + parse_render_batch_override, 0o644, Some(get_atomic_param)); + +fn parse_render_stamp_coalesce(text:&str)->Option { + parse_u64(text).filter(|value| *value<=1) +} + +/// Root-only T6030 render stamp-flush coalescing; zero retains per-pass flushes. +/// Each publication snapshots the setting before encoding any pass. +static M3_RENDER_STAMP_COALESCE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_stamp_coalesce", M3_RENDER_STAMP_COALESCE, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_stamp_coalesce()->bool { + M3_RENDER_STAMP_COALESCE.load(Ordering::Relaxed)!=0 +} + +/// T6030 cross-engine publications retain GPU ordering with firmware barriers. +/// Zero preserves the existing host retirement between engine groups. +static M3_MIXED_BATCH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_mixed_batch", M3_MIXED_BATCH, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn mixed_batch()->bool { + M3_MIXED_BATCH.load(Ordering::Relaxed)!=0 +} + +/// Boot-only T6030 power-state sampling while GPU stamps are outstanding. +/// Zero preserves the completion path without additional register reads. +static M3_ACTIVE_POWER: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_active_power", M3_ACTIVE_POWER, parse_render_stamp_coalesce); +pub(crate) fn active_power()->bool { + M3_ACTIVE_POWER.load(Ordering::Relaxed)==1 +} + +/// Boot-only T6030 Globals debug-word experiment. Zero preserves the runtime word. +static M3_GLOBALS_DEBUG_OFF: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_globals_debug_off",M3_GLOBALS_DEBUG_OFF,parse_render_stamp_coalesce); +pub(crate) fn globals_debug_off()->bool {M3_GLOBALS_DEBUG_OFF.load(Ordering::Relaxed)==1} + +/// T6030-only request while work is pending or active. Zero leaves firmware +/// performance selection unchanged; three requests the top published state. +static M3_CONSISTENT_PERF: AtomicU64 = AtomicU64::new(0); +fn parse_consistent_perf(text: &str)->Option { + parse_u64(text).filter(|v| matches!(*v,0|3)) +} +m3_param!("m3_consistent_perf",M3_CONSISTENT_PERF,parse_consistent_perf,0o644,Some(get_atomic_param)); +pub(crate) fn consistent_perf()->u64 {M3_CONSISTENT_PERF.load(Ordering::Relaxed)} + +/// One preserves internal render measurements. Application timestamp requests +/// remain recorded when zero omits measurements on T6030. +static M3_RENDER_INTERNAL_TIMESTAMPS: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_internal_timestamps", M3_RENDER_INTERNAL_TIMESTAMPS, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_internal_timestamps()->bool { + M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 +} + +/// Snapshot once per batch. A live change affects later publications, never the +/// storage, ordering or retirement checks of work already on the firmware queues. +pub(crate) fn render_batch_size() -> usize { + let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_render_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_pass_layout::SLOTS) +} + +fn parse_compute_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) +} + +/// Independently tune the existing adjacent-compute path. Zero preserves the +/// boot parameter; invalid writes do not replace the current value. Storage and +/// engine/VM retirement boundaries are unchanged. +static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, + parse_compute_batch_override, 0o644, Some(get_atomic_param)); + +/// Snapshot once per packet, so a live write never changes an in-flight batch. +pub(crate) fn compute_batch_size() -> usize { + let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_compute_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_compute_storage::SLOTS) +} + +/// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. +static G15_DEBUG: AtomicU64 = AtomicU64::new(0); +m3_param!("g15_debug", G15_DEBUG, parse_u64); + +/// Bits of `asahi.g15_debug`, the bring-up switches of the G15 manager backend. +/// +/// The bit numbers are unchanged from their earlier `asahi.debug_flags` positions. Bits 49 +/// and 50 are no longer switches: the HwDataB power-management flags they enabled are always +/// set on G15. +#[derive(Copy, Clone)] +#[repr(u32)] +pub(crate) enum G15Debug { + /// Debug switch: issue the Fender kick (0x11, then 0x10) before starting the ASC. The + /// G15 firmware boots and runs jobs without it. + FenderKick = 41, + /// Register the DRM device (card/render nodes) for the manager backend, like + /// `asahi.m3_expose=1`. Off by default so compositors and system Mesa never see a GPU that + /// cannot run userspace jobs yet; the firmware bring-up runs either way. + ExposeDrm = 42, + /// Turn the firmware log on (RuntimePointers+0x230 = 1) before MSG_INIT. The firmware + /// writes no log records while it is 0. + FwLog = 43, + /// A/B: ring a DEVCTRL (0x83|0x11) and a KICKFW (0x83|0x10) doorbell immediately after + /// MSG_INIT, before the firmware has registered its EP 0x21 handler. + LegacyDoorbells = 44, + /// Skip the post-init DeviceControl liveness probe (no doorbell at all after MSG_INIT). + NoDevctlProbe = 45, + /// Map the gfx-data carveout (firmware data) read-only and log the init progress words, + /// thread handles and crashlog fill at every health sample. + GfxDataPeek = 46, + /// After firmware init, submit one empty compute job and poll its stamp for 2 seconds. + /// Never fails probe and never blocks boot past that poll. + SelfTest = 47, + /// With [`G15Debug::SelfTest`]: first run a firmware-only copy of the job (cmd+0x878 = 1, + /// StartCL skip path, no hardware kick). The real job runs only if that one completes. + SelfTestSkipFirst = 51, + /// Runtime backend: stop the probe right before the GPU coprocessor would be started. The + /// firmware never runs; everything before that point (admission, firmware identity, + /// InitData generation, register identity checks) still does. + StopBeforeAsc = 53, + /// A/B for the manager backend: publish the performance states as the two voltage-sorted + /// tables (with their device-tree index maps) instead of one table in device-tree order. + ManagerSplitPstates = 54, + /// A/B for the manager backend: write the power-controller block's PPM words at +0xc8..+0xd8 + /// in the order the runtime backend's InitData uses (0, enable, target power, kp, ki*dt), + /// and the Globals power-interface targets and performance-state cap, instead of the + /// manager's validated values. + ManagerReferencePpm = 55, + /// Debug switch for the runtime backend: when a job fails and the firmware reports that it has + /// halted, clear the halted flag and set resume, then log for about half a second what the + /// firmware, the engines and the failed job do. The GPU stays marked failed either way. + M3ResumeAfterFault = 56, +} + +/// Returns whether a `asahi.g15_debug` bit is set. +pub(crate) fn g15_debug(bit: G15Debug) -> bool { + G15_DEBUG.load(Ordering::Relaxed) & (1u64 << (bit as u32)) != 0 +} + +/// The raw `asahi.g15_debug` value, for logging. +pub(crate) fn g15_debug_mask() -> u64 { + G15_DEBUG.load(Ordering::Relaxed) +} + +/// `asahi.m3_backend` values. +const BACKEND_AUTO: u64 = 0; +const BACKEND_RUNTIME: u64 = 1; +const BACKEND_MANAGER: u64 = 2; +const BACKEND_OFF: u64 = 3; + +fn parse_backend(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(BACKEND_AUTO), + "runtime" | "1" => Some(BACKEND_RUNTIME), + "manager" | "2" => Some(BACKEND_MANAGER), + "off" | "3" => Some(BACKEND_OFF), + _ => None, + } +} + +/// `asahi.m3_backend=auto|runtime|manager|off`: which T6030 GPU backend probe starts. +static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); +m3_param!("m3_backend", M3_BACKEND, parse_backend); + +/// The T6030 GPU backend to start. +pub(crate) enum T6030Backend { + /// The serialized M3 runtime (m3_drm / m3_runtime). + Runtime, + /// The G15 GpuManager backend (g15_probe). + Manager, + /// No backend: the probe stops without touching the GPU. + Off, +} + +/// `asahi.m3_initdata` values. +const INITDATA_AUTO: u64 = 0; +const INITDATA_ADT: u64 = 2; + +fn parse_initdata(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(INITDATA_AUTO), + "adt" | "2" => Some(INITDATA_ADT), + _ => None, + } +} + +/// `asahi.m3_initdata=auto|adt`: the runtime backend's InitData contents are generated from the +/// device tree either way; the value is logged. +static M3_INITDATA: AtomicU64 = AtomicU64::new(INITDATA_AUTO); +m3_param!("m3_initdata", M3_INITDATA, parse_initdata); + +/// Requested source of the runtime backend's InitData contents. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitDataParam { + /// Not given: contents generated from the device tree. + Auto, + /// Contents generated from the device tree. + Adt, +} + +/// The `asahi.m3_initdata` setting. +pub(crate) fn initdata_param() -> InitDataParam { + match M3_INITDATA.load(Ordering::Relaxed) { + INITDATA_ADT => InitDataParam::Adt, + _ => InitDataParam::Auto, + } +} + +/// Default of `asahi.m3_max_pstate` for device-tree InitData and the G15 manager backend: the +/// fifth performance state (1056 MHz on J516S). +pub(crate) const MAX_PSTATE_DEFAULT: u64 = 5; +/// A performance-state parameter that is not given. +const PSTATE_UNSET: u64 = u64::MAX; + +/// `asahi.m3_max_pstate`: the highest GPU performance state the driver lets the firmware use, +/// as an index into the voltage-sorted performance-state table (1 = lowest, 8 = highest on +/// T6030; the G15 manager backend translates it to its device-tree ordered table). It is +/// clamped to the device tree's highest performance state. Unset means +/// [`MAX_PSTATE_DEFAULT`], except with the runtime backend while its thermal limit is on or +/// holding, which keeps the whole table. +static M3_MAX_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_max_pstate", M3_MAX_PSTATE, parse_u64); + +/// `asahi.m3_boot_pstate`: the performance state the runtime backend requests at boot, clamped +/// to the cap. Unset means the cap. +static M3_BOOT_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_boot_pstate", M3_BOOT_PSTATE, parse_u64); + +/// The `asahi.m3_max_pstate` setting, if given, before clamping to the performance-state table. +pub(crate) fn max_pstate_param() -> Option { + Some(M3_MAX_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// The `asahi.m3_boot_pstate` setting, if given, before clamping. +pub(crate) fn boot_pstate_param() -> Option { + Some(M3_BOOT_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// `asahi.m3_timeout_nohang`: 1 (default) makes a scheduler timeout of a runtime-backend job +/// report "no hang", because the runtime completes every packet itself and enforces its own +/// per-batch completion limit. 0 keeps the previous handling: mark the GPU failed, fail the +/// packet again and report the device as gone. +static M3_TIMEOUT_NOHANG: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_timeout_nohang", M3_TIMEOUT_NOHANG, parse_u64); + +/// Whether `asahi.m3_timeout_nohang` is set (the default). +pub(crate) fn timeout_nohang() -> bool { + M3_TIMEOUT_NOHANG.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_unlocked_wait`: 1 (default) releases the runtime lock while the runtime backend +/// waits for a batch to complete, so VM creation, timestamp mapping and the event worker are +/// not held up by a running job. 0 holds the lock from preparation to retirement, as before. +static M3_UNLOCKED_WAIT: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_unlocked_wait", M3_UNLOCKED_WAIT, parse_u64); + +/// Whether `asahi.m3_unlocked_wait` is set (the default). +pub(crate) fn unlocked_wait() -> bool { + M3_UNLOCKED_WAIT.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback +/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); +const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; +fn parse_ut_engagement(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 1) +} +m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); + +/// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. +pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { + match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { + UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), + v => (v as u32, true), + } +} + +/// `asahi.m3_pipeline_depth`: writable at runtime. How many render batches may be in flight on +/// the firmware's render queues. Zero selects the SoC default: 2 on the explicitly enabled +/// experimental T8122 runtime, 1 on T6030. Depth 1 publishes only after the previous batch has +/// fully retired. Larger values publish the next render batch of the same VM while earlier +/// ones still run, in disjoint pass slots, ordered by the existing cross-pass barriers. +static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); + +fn parse_pipeline_depth(text: &str) -> Option { + parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) +} + +/// Effective depth; changing it never releases already published pass slots. +pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { + match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { + 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + } +} + +/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets +/// of its VM only while the VM's measured GPU time per pass, times the passes already in the +/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro). +/// Coalescing a client's heavy frames +/// otherwise makes every other client's work, the compositor's included, wait for all of them. +/// 0 disables the budget; u64::MAX restores the SoC default. +static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); +m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); + +/// The render batch GPU-time budget in ns, or `None` without one. +pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { + let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); + let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + match us { + 0 => None, + us => Some(us.saturating_mul(1000)), + } +} + +/// Bits of `asahi.m3_retire_mmio`. +pub(crate) const RETIRE_MMIO_BUSY: u64 = 1 << 0; +pub(crate) const RETIRE_MMIO_FAULTS: u64 = 1 << 1; +pub(crate) const RETIRE_MMIO_PSTATE: u64 = 1 << 2; +const RETIRE_MMIO_UNSET: u64 = u64::MAX; + +fn parse_retire_mmio(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 7) +} + +/// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every +/// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 +/// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance +/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); +m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); + +/// The `asahi.m3_retire_mmio` mask for `soc`. +pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { + match M3_RETIRE_MMIO.load(Ordering::Relaxed) { + RETIRE_MMIO_UNSET => soc.retire_mmio, + mask => mask, + } +} + +/// `asahi.m3_thermal` values. +const THERMAL_OFF: u64 = 0; +const THERMAL_HOLD: u64 = 1; +const THERMAL_ON: u64 = 2; +const THERMAL_UNSET: u64 = u64::MAX; + +fn parse_thermal(text: &str) -> Option { + match text.trim() { + "off" | "0" => Some(THERMAL_OFF), + "hold" => Some(THERMAL_HOLD), + "on" | "1" => Some(THERMAL_ON), + _ => None, + } +} + +static M3_THERMAL: AtomicU64 = AtomicU64::new(THERMAL_UNSET); +m3_param!("m3_thermal", M3_THERMAL, parse_thermal); + +/// How the runtime backend limits the GPU performance state by temperature. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum ThermalMode { + /// Today's fixed cap: the firmware is given the cap as its highest state, and nothing + /// changes it at runtime. + Off, + /// The firmware is given the whole table as its highest state, and the runtime cap words + /// hold the fixed cap for the whole boot. The same performance as `Off`, if the firmware + /// follows the runtime cap: this checks that it does. + Hold, + /// As `Hold`, and the runtime cap follows the SoC die temperature, up to the whole table + /// while the die is cool. + On, +} + +/// The `asahi.m3_thermal` setting, if given. +pub(crate) fn thermal_param() -> Option { + match M3_THERMAL.load(Ordering::Relaxed) { + THERMAL_OFF => Some(ThermalMode::Off), + THERMAL_HOLD => Some(ThermalMode::Hold), + THERMAL_ON => Some(ThermalMode::On), + _ => None, + } +} + +/// Default and bounds of `asahi.m3_thermal_hot`, in degrees Celsius. +pub(crate) const THERMAL_HOT_DEFAULT: u64 = 80; +pub(crate) const THERMAL_HOT_MIN: u64 = 40; +pub(crate) const THERMAL_HOT_MAX: u64 = 85; + +/// `asahi.m3_thermal_hot`: the SoC die temperature (degrees Celsius) at or above which the thermal +/// limit lowers the runtime cap, clamped to [`THERMAL_HOT_MIN`]..=[`THERMAL_HOT_MAX`]. +static M3_THERMAL_HOT: AtomicU64 = AtomicU64::new(THERMAL_HOT_DEFAULT); +m3_param!("m3_thermal_hot", M3_THERMAL_HOT, parse_u64); + +/// The `asahi.m3_thermal_hot` setting and whether it was clamped. +pub(crate) fn thermal_hot_param() -> (u64, bool) { + let value = M3_THERMAL_HOT.load(Ordering::Relaxed); + let clamped = value.clamp(THERMAL_HOT_MIN, THERMAL_HOT_MAX); + (clamped, clamped != value) +} + +/// Whether to start the runtime backend on a T8122 (`asahi.m3_backend`), and say why not. +/// +/// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with +/// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another +/// board. The G15 manager backend is T6030-only. +pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + false + } + BACKEND_MANAGER => { + dev_info!( + dev, + "M3: asahi.m3_backend=manager: the G15 manager backend is T6030-only; no GPU backend started\n" + ); + false + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ => { + dev_info!( + dev, + "M3: no GPU backend on this T8122 board: the runtime backend may start only on the M3 MacBook Airs (J613, J615)\n" + ); + false + } + } +} + +/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. +/// +/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook +/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; +/// `asahi.m3_backend=runtime` starts it anyway. +pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_RUNTIME => { + dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); + T6030Backend::Runtime + } + BACKEND_MANAGER => { + dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); + T6030Backend::Manager + } + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + T6030Backend::Off + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, + _ => { + dev_info!( + dev, + "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" + ); + T6030Backend::Off + } + } +} + +fn parse_t8122_number(text: &str) -> Option { + Some(crate::t8122_knobs::parse_number(text)) +} + +fn parse_t8122_fender(text: &str) -> Option { + Some(crate::t8122_knobs::parse_fender(text)) +} + +fn parse_t8122_clkgen(text: &str) -> Option { + Some(crate::t8122_knobs::parse_clkgen(text)) +} + +fn parse_t8122_sgx_setup(text: &str) -> Option { + Some(crate::t8122_knobs::parse_sgx_setup(text)) +} + +// The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a +// value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather +// than fall back to a default. +use crate::t8122_knobs::UNSET as T8122_UNSET; + +/// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. +/// Only with a boot loader that handed the GPU over. +static T8122_START: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_start", T8122_START, parse_t8122_number); +/// `asahi.t8122_initdata_version=`: the InitData version given to the T8122 firmware. +static T8122_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_initdata_version", T8122_INITDATA_VERSION, parse_t8122_number); +/// `asahi.t8122_fender=0x104000|0x12c000` (`rule`, `adt`): the Fender window size. +static T8122_FENDER: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_fender", T8122_FENDER, parse_t8122_fender); +/// `asahi.t8122_clkgen=e1c|e5c|none`: the GPU clock-generator IO mapping. +static T8122_CLKGEN: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_clkgen", T8122_CLKGEN, parse_t8122_clkgen); +/// `asahi.t8122_sgx_setup=none|t6030`: the SGX write made before the firmware starts. +static T8122_SGX_SETUP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_sgx_setup", T8122_SGX_SETUP, parse_t8122_sgx_setup); +/// `asahi.t8122_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T8122_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_a", T8122_UNIT_MASK_A, parse_t8122_number); +/// `asahi.t8122_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T8122_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number); +/// `asahi.t8122_pstate_cap=N`: the highest performance state the T8122 firmware may use. +static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); + +/// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) +/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// range zero, for an A/B test. Readable in sysfs. +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, + Some(get_atomic_param)); + +/// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). +pub(crate) fn t8122_hwdata_418c() -> bool { + crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) +} + +/// The T8122 start-experiment parameters of this boot, as given. +pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t8122_knobs::Raw { + start: get(&T8122_START), + initdata_version: get(&T8122_INITDATA_VERSION), + fender: get(&T8122_FENDER), + clkgen: get(&T8122_CLKGEN), + sgx_setup: get(&T8122_SGX_SETUP), + unit_mask_a: get(&T8122_UNIT_MASK_A), + unit_mask_b: get(&T8122_UNIT_MASK_B), + pstate_cap: get(&T8122_PSTATE_CAP), + } +} + +fn parse_t6031_number(text: &str) -> Option { + Some(crate::t6031_knobs::parse_number(text)) +} + +fn parse_t6031_fender(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fender(text)) +} + +fn parse_t6031_clkgen(text: &str) -> Option { + Some(crate::t6031_knobs::parse_clkgen(text)) +} + +fn parse_t6031_sgx_setup(text: &str) -> Option { + Some(crate::t6031_knobs::parse_sgx_setup(text)) +} + +fn parse_t6031_mcache(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mcache(text)) +} + +fn parse_t6031_aic_swint(text: &str) -> Option { + Some(crate::t6031_knobs::parse_aic_swint(text)) +} + +fn parse_t6031_slot7(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot7(text)) +} + +fn parse_t6031_gifaf(text: &str) -> Option { + Some(crate::t6031_knobs::parse_gifaf(text)) +} + +fn parse_t6031_slot(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot(text)) +} + +fn parse_t6031_ane(text: &str) -> Option { + Some(crate::t6031_knobs::parse_ane(text)) +} + +fn parse_t6031_mtr(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mtr(text)) +} + +fn parse_t6031_fw_words(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fw_words(text)) +} + +fn parse_t6031_hwdata_object(text: &str) -> Option { + Some(crate::t6031_knobs::parse_hwdata_object(text)) +} + +fn parse_t6031_rev_id(text: &str) -> Option { + Some(crate::t6031_knobs::parse_rev_id(text)) +} + +fn parse_t6031_tristate(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tristate(text)) +} + +fn parse_t6031_tag(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tag(text)) +} + +// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its +// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. +use crate::t6031_knobs::UNSET as T6031_UNSET; + +/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe +/// is the same refusal as before. +static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_start", T6031_START, parse_t6031_number); +/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. +static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); +/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. +static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); +/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. +static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); +/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. +static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); +/// `asahi.t6031_fender=rule|adt`: the Fender window size. +static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); +/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. +static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); +/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. +static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); +/// `asahi.t6031_mcache=none|x2|x4|x8`. +static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); +/// `asahi.t6031_aic_swint=page|reg`. +static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); +/// `asahi.t6031_slot7=none|rule`. +static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); +/// `asahi.t6031_gifaf=none|fender`. +static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); +/// `asahi.t6031_slot21=none` or a physical address. +static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); +/// `asahi.t6031_slot24=none` or a physical address. +static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); +/// `asahi.t6031_slot28=none` or a physical address. +static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); +/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. +static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); +/// `asahi.t6031_io_drop`: a mask of default slots to leave out. +static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); +/// `asahi.t6031_mtr=t6030|t8122|adt|none`. +static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); +/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. +static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); +/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. +static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); +/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); +/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); +/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. +static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); +/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. +static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); +/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. +static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); +/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. +static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); +/// `asahi.t6031_fw_words=none|t8122`. +static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); +/// `asahi.t6031_hwdata_object=aligned|fixed`. +static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); +/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. +static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); +/// `asahi.t6031_rev_id=auto` or 1 to 6. +static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); +/// `asahi.t6031_csafr=auto|on|off`. +static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); +/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. +static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); + +/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. +pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t6031_knobs::Raw { + start: get(&T6031_START), + image_uuid: crate::t6031_knobs::UuidParam::Unset, + image_hash: get(&T6031_IMAGE_HASH), + tag: get(&T6031_TAG), + initdata_version: get(&T6031_INITDATA_VERSION), + pstate_cap: get(&T6031_PSTATE_CAP), + fender: get(&T6031_FENDER), + clkgen: get(&T6031_CLKGEN), + sgx_setup: get(&T6031_SGX_SETUP), + mcache: get(&T6031_MCACHE), + aic_swint: get(&T6031_AIC_SWINT), + slot7: get(&T6031_SLOT7), + gifaf: get(&T6031_GIFAF), + slot21: get(&T6031_SLOT21), + slot24: get(&T6031_SLOT24), + slot28: get(&T6031_SLOT28), + ane: get(&T6031_ANE), + io_drop: get(&T6031_IO_DROP), + mtr: get(&T6031_MTR), + mtr_fast_die: get(&T6031_MTR_FAST_DIE), + mtr_alarm: get(&T6031_MTR_ALARM), + unit_mask_a: get(&T6031_UNIT_MASK_A), + unit_mask_b: get(&T6031_UNIT_MASK_B), + hwb_454: get(&T6031_HWB_454), + hwb_b20: get(&T6031_HWB_B20), + hwb_17b8: get(&T6031_HWB_17B8), + hwb_1818: get(&T6031_HWB_1818), + fw_words: get(&T6031_FW_WORDS), + hwdata_object: get(&T6031_HWDATA_OBJECT), + gpu_core: get(&T6031_GPU_CORE), + rev_id: get(&T6031_REV_ID), + csafr: get(&T6031_CSAFR), + power_cap_mw: get(&T6031_POWER_CAP_MW), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs new file mode 100644 index 0000000000000..273d2701c3f14 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Private storage for one retired/unpublished G15 render pass. A Pass retains +//! all its buffers and mappings until Render retires the shared queues. +use kernel::prelude::*; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use layout::{Field,Space}; + +pub(crate) struct Pass {buffers:KVec} +impl Pass { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + slot:usize,clusters:u32,aliases:&mut KVec)->Result { + let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; + for field in layout::FIELDS { + let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let mut b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), + a.size,prot::Prot::from_pte(a.access.pte()),prot::PROT_GPU_SHARED_RW)?, + }; + if a.gpu_alias.is_some() {aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?;} + buffers.push(b,GFP_KERNEL)?; + } + Ok(Self{buffers}) + } + pub(crate) fn get(&self,field:Field)->&Buffer {&self.buffers[field as usize]} + pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} + /// Caller must prove retirement, and must not clear a prepared sibling. + /// Dependency commands are rewritten completely by the sync encoder. + pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + // Tiler arenas grow to retain the largest prior target. The firmware + // descriptors use the current geometry's strides and layer count, so + // only that prefix is addressable by this pass. Clear it on every use, + // including when growing again after a smaller pass; the unused tail + // may still contain retired data and is never assumed to be zero. + for field in &layout::FIELDS[..layout::RESET_COUNT] { + let bytes=match field { + Field::Tpc=>usize::try_from(geometry.tpc_bytes).map_err(|_|EOVERFLOW)?, + Field::Tilemap=>usize::try_from(geometry.tilemap_bytes).map_err(|_|EOVERFLOW)?, + _=>self.get(*field).size(), + }; + self.get_mut(*field).fill_range(0,bytes,0)?; + } + Ok(()) + } +} +/// Cached client views retain their resource identity across switches. A pass +/// field cannot accidentally name a shared allocator or another pass's field. +#[derive(Clone,Copy)] +pub(crate) enum ViewOwner {Shared(usize),Pass{slot:usize,field:Field}} diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs new file mode 100644 index 0000000000000..979c1a37bb202 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Construct a firmware-owned UMA allocator once from its graph's allocations. +//! GPU mappings and counters outlive this initialization and remain owned by +//! Render/Compute. Callers must never invoke this on a live or replayed pool. +use kernel::prelude::*; +use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +pub(crate) struct Owners { + pub(crate) backing: core::ops::Range, + pub(crate) pages: usize, + pub(crate) table: usize, + pub(crate) manager: usize, + pub(crate) counter: usize, + pub(crate) table_index: u8, +} +impl Owners { + pub(crate) fn initialize(&self, objects: &mut [Buffer]) -> Result { + let buffers = objects.get(self.backing.clone()).ok_or(EINVAL)?; + let mut backings = KVec::with_capacity(buffers.len(), GFP_KERNEL)?; + for buffer in buffers { + backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?; + } + let pages = objects.get(self.pages).ok_or(EINVAL)?; + let pages = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?; + let table = objects.get(self.table).ok_or(EINVAL)?; + let table = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?; + let counter = FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?; + if objects.get(self.manager).ok_or(EINVAL)?.size() < wire::MANAGER_SIZE { return Err(EINVAL); } + let pool = wire::Pool::new(&backings, pages, table, counter, self.table_index).map_err(|_| EINVAL)?; + let mut manager = [0; wire::MANAGER_SIZE]; + pool.encode(&mut manager).map_err(|_| EINVAL)?; + // Unpopulated entries remain explicitly zero from the fresh DMA owner. + for page in 0..wire::INITIAL_PAGES { + objects[self.pages].u64(page * 8, pool.page(page).map_err(|_| EINVAL)?)?; + } + for entry in 0..backings.len() { + objects[self.table].write(entry * wire::POOL_ENTRY_SIZE, + &pool.table_entry(entry).map_err(|_| EINVAL)?)?; + } + objects[self.manager].write(0, &manager) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs new file mode 100644 index 0000000000000..c2c67f5c3aa5a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_sync_layout as sync; +use crate::m3_state_layout as state; +use crate::m3_scene_layout as scene; +use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; +use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; +use crate::m3_timeline as timeline; +use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; +const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; +pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; +// Serial execution permits retaining each client's views of the owned arenas. +// Bound the cache to four VMs including the active one; cache entries retain +// their ASID lease and all aliases until eviction after exact retirement. +struct CachedViews { + aliases:KVec, + views:KVec<(ViewOwner,mmu::KernelMapping)>, + // Release mappings before returning their ASID lease (Rust field order). + binding:mmu::VmBind, +} +pub(crate) struct Render { + _aliases:KVec, objects:KVec, + passes:KVec, init_bm:Buffer, + // All active mappings above must drop before their ASID binding. + _binding:mmu::VmBind, + stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + last_progress_ns:i64, + draw:u64, heads:[u16;2], cached_views:KVec, + slot:usize, batch_count:usize, early_count:u32, + /// Measurement packets in the pass most recently appended to its queues. + stage_timestamps:[bool;2], + /// First pass slot of the current batch; its passes use base..base+batch_count mod SLOTS. + base:usize, + checkpoint:(u64,[u16;2],usize,usize,u32,usize), + /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. + clusters:u32, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers:crate::m3_compute_layout::RegisterSet, +} +// Queue allocations belong to Render, not to a pass. Keep their fixed arenas +// and firmware/shared mapping permissions from the qualified allocation map. +// Scratch is GPU-visible firmware storage; it is not the application's VM. +struct QueueObjects { + info: usize, state: usize, ring: usize, scratch: usize, + tiler: bool, uuid: u32, +} +use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, + PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; +const MANAGER_SCENE_COUNT: u32 = 48; +const TA_POOL_COUNTER: usize = 11; +const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ + crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +]; +const FRAGMENT_POOL_COUNTER: usize = 23; +// Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. +const DEPENDENCY_UUID: u32 = 250912; +const RENDER_QUEUES: [QueueObjects; 2] = [ + QueueObjects { info: s::TA_QUEUE, state: s::TA_POINTERS, ring: s::TA_RING, + scratch: s::TA_SCRATCH, tiler: true, uuid: 0x7a0000 }, + QueueObjects { info: s::FRAGMENT_QUEUE, state: s::FRAGMENT_POINTERS, ring: s::FRAGMENT_RING, + scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, +]; +impl Render { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3_compute_layout::RegisterSet)->Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new();let mut aliases=KVec::new(); + for index in 0..s::COUNT { + let a=s::allocation(index).map_err(|_|EINVAL)?; + let protection=prot::Prot::from_pte(a.access.pte()); + let b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + a.size,protection,prot::PROT_GPU_SHARED_RW)?, + }; + objects.push(b,GFP_KERNEL)?; + } + // InitBM is shared, not part of a pass. Preserve its qualified mapping. + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; + let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + let mut sequence_bytes=KVec::new(); + sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; + let mut tiler_bytes=KVec::new(); + tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + let mut fragment_bytes=KVec::new(); + fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + // Lists and the matching manager are built before InitBM publication. + job.initialize_parameter_buffer()?; + for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + job.initialize_state()?; + job.initialize_queues()?; + dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(job) + } + fn initialize_parameter_buffer(&mut self) -> Result { + let region = |owner: usize| GpuRegion::new(self.objects[owner].gpu_va()?, self.objects[owner].size()) + .map_err(|_| EINVAL); + let pages = region(PARAMETER_PAGE_LIST)?; + let blocks = region(PARAMETER_BLOCK_LIST)?; + let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; + let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; + let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; + let mut bytes = [0; parameter::MANAGER_SIZE]; + manager.encode(&mut bytes).map_err(|_| EINVAL)?; + for group in 0..PB_GROUPS { + let buffer = &self.objects[PB_FIRST + group]; + let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) + .map_err(|_| EINVAL)?; + for block in 0..PB_BLOCKS_PER_GROUP { + self.objects[PARAMETER_BLOCK_LIST].write((group * PB_BLOCKS_PER_GROUP + block) * 8, + &backing.block(block).map_err(|_| EINVAL)?)?; + } + for page in 0..PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK { + self.objects[PARAMETER_PAGE_LIST].u32((group * PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK + page) * 4, + backing.page(page).map_err(|_| EINVAL)?)?; + } + } + self.objects[BUFFER_MANAGER].write(0, &bytes) + } + fn initialize_state(&mut self) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + self.objects[JOB_LIST].write(0, &list)?; + let mut manager = [0; state::MANAGER_STATE_SIZE]; + state::block_control(&mut manager, PB_BLOCKS, PB_BLOCKS).map_err(|_| EINVAL)?; + self.objects[BLOCK_CONTROL].write(0, &manager)?; + state::manager_misc(&mut manager).map_err(|_| EINVAL)?; + self.objects[MANAGER_MISC].write(0, &manager)?; + for (owner, value) in [(TA_POOL_COUNTER, 1), (FRAGMENT_POOL_COUNTER, 1), + (s::EVENT_COUNT, 2), (s::BM_COUNTER, 1), + (s::TA_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), (s::TA_FW_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), + (s::FRAGMENT_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0)), (s::FRAGMENT_FW_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..self.objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + self.objects[owner].write(0, bytes)?; + } + Ok(()) + } + fn initialize_queues(&mut self) -> Result { + for owner in &RENDER_QUEUES { + let address = |index: usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { + state: address(owner.state)?, ring: address(owner.ring)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(owner.scratch)?, + context: address(GPU_CONTEXT)?, uuid: owner.uuid, + }; + let seed_addresses=if owner.tiler { + [self.init_bm.va(),self.passes[0].get(P::TilerCommand).va()] + }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; + let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + self.objects[owner.info].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } + .encode(&mut state).map_err(|_| EINVAL)?; + self.objects[owner.state].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &seeds).map_err(|_| EINVAL)?; + self.objects[owner.ring].write(index as usize * 8, &entry)?; + } + } + Ok(()) + } + fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { + let mut bytes = [0; sync::BARRIER_SIZE]; + command.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(owner).write(0, &bytes) + } + // Construct synchronization from the actual owners after resetting a pass, + // before publishing its ring entries. InitBM is shared and dispatched only + // for draw one; rewriting its bytes after retirement preserves the old path. + fn prepare_sync(&mut self, early: bool, draw: u64, + predecessor:Option) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let manager = address(BUFFER_MANAGER)?; + let ta_stamp = address(s::TA_FW_STAMP)?; + let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; + let mut init = [0; sync::INIT_BM_SIZE]; + sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } + .encode(&mut init).map_err(|_| EINVAL)?; + self.init_bm.write(0, &init)?; + let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); + let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); + let previous=predecessor.unwrap_or(sync::CompletionDependency { + stamp:if early {ta_stamp} else {fs_stamp}, + value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), + event:u32::from(!early), + }); + self.write_barrier(P::TilerToFragment, sync::Barrier { + stamp: ta_stamp, wait_value: ta_value, event: 0, self_value: fs_value, + uuid: 0x3d0000, dependency: sync::Dependency::TilerToFragment })?; + self.write_barrier(P::TilerDependency, sync::Barrier { + stamp: previous.stamp, wait_value: previous.value, + event: previous.event, self_value: ta_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?; + self.write_barrier(P::FragmentDependency, sync::Barrier { + stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass }) + } + // Prepare each private command after its retired slot is reset. The + // reusable host staging allocation is copied before the next slot is built. + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + use crate::m3_compute_layout::GpuVa; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::TilerSequence); + let command=pass.get(P::TilerCommand); + let g=Self::geometry(r,self.clusters)?; + if g.tpc_bytes>pass.get(P::Tpc).size() as u64 + || g.tilemap_bytes>pass.get(P::Tilemap).size() as u64 {return Err(E2BIG);} + // The old per-slot register keeps its seed entry, independent of the + // draw-dependent RenderScene reference. Do not change this policy here. + let initial_scene_entry=queue::FirmwareVa::new(self.objects[s::BM_SCENES].va() + +(self.slot as u64+1)*4).map_err(|_|EINVAL)?; + let value=tiler::Command { + context:self._binding.slot(),register_set:self.registers, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, + scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, + scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, + stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], + user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], + tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, + scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, + vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, + }; + value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) + } + fn prepare_sequences(&mut self,draw:u64)->Result { + for fs in [false,true] { + let pass=&self.passes[self.slot]; + let region=|field:P| {let b=pass.get(field);Region::new(b.va(),b.size()).map_err(|_|EINVAL)}; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let target=if fs{P::FragmentSequence}else{P::TilerSequence}; + let notifier=&self.objects[NOTIFIER]; + let owners=sequence::Owners { + command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, + stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, + manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, + fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, + context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), + }; + let timed=self.stage_timestamps[usize::from(fs)]; + let length=if fs { + let b=pass.get(P::FragmentScratch); + let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; + sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? + }else{sequence::Tiler{owners,scratch:fw(P::TilerScratch)?}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)?}; + self.passes[self.slot].get_mut(target).write(0,&self.sequence_bytes)?; + self.passes[self.slot].get_mut(if fs{P::FragmentCommand}else{P::TilerCommand}).u32(sequence::command_length_offset(fs),length as u32)?; + } + Ok(()) + } + fn view_owner(&mut self,owner:ViewOwner)->&mut Buffer { + match owner {ViewOwner::Shared(index)=>&mut self.objects[index], + ViewOwner::Pass{slot,field}=>self.passes[slot].get_mut(field)} + } + fn patch_context(&mut self)->Result { + let context=self._binding.slot(); + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + Ok(()) + } + /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::agx_render::{Geometry,Utile}; + let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; + let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; + let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; + g.set_samples(r.samples).map_err(|_|EINVAL)?; + Ok(g) + } + fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { + use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::FragmentSequence); + let command=pass.get(P::FragmentCommand); + let geometry=Self::geometry(r,self.clusters)?; + let mask=if r.flags&16!=0 {u64::MAX} else {u32::MAX as u64}; + let program=|p:agx_uapi::UapiProgram|->Result {Ok(Program { + address:usc.checked_add(u64::from(p.usc&!63)).ok_or(EOVERFLOW)?,resources:p.resource_spec&mask, + })}; + let tilebuffer_control=(u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)).div_ceil(2048) + |(u32::from(r.samples.trailing_zeros() as u8)<<17); + let tile_mode=0x280|u64::from(r.layers>1)|if r.flags&2!=0 {1<<16}else{0}; + let value=fragment::Command { + context:self._binding.slot(),register_set:self.registers, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, + heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, + scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, + fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw), + timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?], + state:fragment::State { + geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y], + eot:program(r.end_of_tile)?,background:program(r.background)?, + partial_eot:program(r.partial_end_of_tile)?,partial_background:program(r.partial_background)?, + scissor:r.scissor_base,depth_bias:r.depth_bias_base,query:r.occlusion_query_base, + isp_control:0xc000|(r.flags&(1<<18)),tilebuffer_control,mirror_tilebuffer_control:tilebuffer_control, + tile_mode,mirror_tile_mode:tile_mode as u32, + depth:DepthStencil{base:r.depth.base,stride:u64::from(r.depth.stride)}, + stencil:DepthStencil{base:r.stencil.base,stride:u64::from(r.stencil.stride)}, + depth_compression:DepthStencil{base:r.depth.compression_base,stride:u64::from(r.depth.compression_stride)}, + stencil_compression:DepthStencil{base:r.stencil.compression_base,stride:u64::from(r.stencil.compression_stride)}, + zls_control:r.zls_control,depth_dimensions:r.depth_dimensions,depth_clear:r.depth_clear, + stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, + }, + }; + value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { + if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} + for (stage,(command,sequence,offset,start,end)) in [ + (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), + (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) + ].into_iter().enumerate() { + let pair=addresses[stage]; + if !self.stage_timestamps[stage] {continue;} + self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; + self.passes[self.slot].get_mut(command).u64(offset+8,pair[1])?; + let pointer=if pair==[0,0] {0} else {self.passes[self.slot].get_mut(command).va()+offset as u64}; + self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; + self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; + } + crate::agx_memory::publish();Ok(()) + } + fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} + /// GPU time of the batch of `count` passes starting at pass slot `base`. + pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { + let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; + let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. + pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { + Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + } + /// Read retained timestamps only after exact batch retirement. The first + /// TA and final fragment delimit the batch in the shared 24 MHz counter. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + } + pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { + let mut total=[0;3]; + for index in 0..count { + let stages=self.slot_stage_ns(base,count,index)?; + for i in 0..3 {total[i]+=stages[i];} + } + Ok(total) + } + pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { + if index>=count {return Err(EINVAL);} + let slot=Self::at(base,index); + let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; + let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; + let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; + let ta_end=self.passes[slot].get_mut(P::TilerEnd).read_u64(0)?; + Ok([duration(ta_start,ta_end),duration(fs_start,fs_end),duration(ta_end,fs_start)]) + } + pub(crate) fn early_count(&self)->u32 {self.early_count} + pub(crate) fn heads(&self)->[u16;2] {self.heads} + pub(crate) fn ordinal(&self)->u64 {self.draw} + /// The current batch: first pass slot, pass count and last draw number. + pub(crate) fn batch(&self)->(usize,usize,u64) {(self.base,self.batch_count,self.draw)} + /// Whether draws 1..=`draw` have completed both stages (TA and fragment stamps, host and + /// firmware copies, have reached `draw`'s values). Use signed modular ordering + /// across stamp wrap; the runtime bounds outstanding passes to SLOTS. + pub(crate) fn complete_through(&mut self,draw:u64)->Result { + if draw==0 {return Ok(true);} + for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] { + if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);} + } + Ok(true) + } + /// Whether any draw has been published (the first one initializes the buffer manager). + pub(crate) fn started(&self)->bool {self.draw>0} + pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then + /// the VM must already be bound, no storage may grow and the slots must not be in flight + /// (the caller keeps the in-flight passes plus this batch within SLOTS). + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} + if overlap { + if !self._binding.matches(vm) {return Err(EBUSY);} + for (index,&command) in commands.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + let slot=Self::at(base,index); + if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 + || g.tilemap_bytes>self.passes[slot].get(P::Tilemap).size() as u64 {return Err(EBUSY);} + } + } + if commands.is_empty() || commands.len()>pass_layout::SLOTS {return Err(E2BIG);} + self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; + // Validate every pass before changing any queue or completion state. + for &command in commands { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + Self::geometry(r,self.clusters)?; + crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + } + if !self._binding.matches(vm) { + let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_|EIO)? + } else { + let binding=uat.bind(vm)?; + let mut views=KVec::new();let mut aliases=KVec::new(); + for i in 0..s::COUNT { + if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + for (slot,pass) in self.passes.iter_mut().enumerate() { + for field in pass_layout::FIELDS { + let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} + if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} + } + } + CachedViews{binding,aliases,views} + }; + // Finish fallible allocations before replacing the current owner. + let old_views=KVec::with_capacity(next.views.len(),GFP_KERNEL)?; + self.cached_views.reserve(1,GFP_KERNEL)?; + if self.cached_views.len()==3 {drop(self.cached_views.remove(0).map_err(|_|EIO)?);} + let mut old=CachedViews { + binding:core::mem::replace(&mut self._binding,next.binding), + aliases:core::mem::replace(&mut self._aliases,next.aliases), + views:old_views, + }; + for (i,view) in next.views { + old.views.push((i,self.view_owner(i).swap_gpu_view(view)?),GFP_KERNEL)?; + } + self.cached_views.push(old,GFP_KERNEL)?; + } + // Growth is legal only after exact retirement, and before append. + // Cached VM mappings must be dropped before freeing the old backing. + // Partial allocation failure leaves retired queue/stamp state intact; + // already-grown buffers are reusable on the next submission. + for (index,&command) in commands.iter().enumerate() { + let slot=Self::at(base,index); + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { + let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; + if size>self.passes[slot].get(field).size() { + let buffer=Buffer::tiler(dev,uat.kernel_vm(),vm,size)?; + self.cached_views.clear(); + dev_info!(dev.as_ref(),"M3 render arena growth: slot={} object={} {} -> {} bytes GPU={:#x}\n", + slot,base,self.passes[slot].get(field).size(),size,buffer.gpu_va()?); + *self.passes[slot].get_mut(field)=buffer; + } + } + } + self.checkpoint=(self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base); + self.batch_count=0; + self.base=base; + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can + /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. + pub(crate) fn abort_batch(&mut self)->Result { + (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; + self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish(); + Ok(()) + } + /// Append only into caller-reserved free pass slots. With overlap, WRITE updates + /// are immediately visible to running firmware, even before a new doorbell. + /// Each append selects disjoint host-mutated pass storage. A TA dependency + /// retains full fragment-to-next-TA ordering for arbitrary resource hazards. + pub(crate) fn append(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,coalesce_stamps:bool,timestamps:[bool;2])->Result { + self.append_after(r,usc,coalesce_stamps,None,timestamps) + } + pub(crate) fn completion_dependency(&self)->Result { + Ok(sync::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?, + value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1, + }) + } + pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64, + coalesce_stamps:bool,predecessor:Option,timestamps:[bool;2])->Result { + if predecessor.is_some_and(|p|p.event!=2) {return Err(EINVAL);} + if self.batch_count>=pass_layout::SLOTS { + pr_err!("M3 render batch limit: batch={} slots={}\n", + self.batch_count,pass_layout::SLOTS); + return Err(E2BIG); + } + self.slot=Self::at(self.base,self.batch_count); + self.stage_timestamps=timestamps; + // Retain the conservative stage-scope permission. Only a pass after the + // first of its batch may overlap its predecessor's stages. Explicit + // start timestamps keep the original full-order execution boundary. + let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 + && r.fragment_timestamps.start.handle==0; + if early {self.early_count+=1;} + // Only reset per-draw objects. Pool allocators, PB lists, queues, + // notification state and previous stamps retain firmware ownership. + self.passes[self.slot].reset(&Self::geometry(r,self.clusters)?)?; + self.init_bm.fill(0)?; // retain the prior shared InitBM reset before encoding + self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; + self.batch_count+=1; + let draw=self.draw; + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; + let scene_list = &mut self.objects[s::BM_SCENES]; + if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } + let manager_entry = queue::FirmwareVa::new(scene_list.va().checked_add(entry_offset as u64) + .ok_or(EOVERFLOW)?).map_err(|_| EINVAL)?; + let manager_misc = queue::FirmwareVa::new(self.objects[MANAGER_MISC].va()).map_err(|_| EINVAL)?; + let user_buffer = self.passes[self.slot].get_mut(P::SceneUser); + let user_buffer = scene::UserBuffer::new(user_buffer.gpu_va()?, user_buffer.size()).map_err(|_| EINVAL)?; + let mut bytes = [0; scene::SIZE]; + scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; + self.prepare_tiler(r,draw)?; + self.patch_context()?; + self.prepare_fragment(r,usc,draw)?; + self.prepare_sequences(draw)?; + self.prepare_sync(early, draw, predecessor)?; + // Encode before either ring WRITE exposes the command to firmware. + // The caller retains every pass owner until the final flushed stamps, + // queues, events and engine-idle checks retire this entire batch. + if coalesce_stamps { + self.passes[self.slot].get_mut(P::TilerCommand).u32(tiler::FLUSH_STAMPS,0)?; + self.passes[self.slot].get_mut(P::FragmentCommand).u32(fragment::FLUSH_STAMPS,0)?; + } + let dep=P::TilerDependency; + let fs_dep=P::FragmentDependency; + let ta=self.passes[self.slot].get_mut(P::TilerCommand).va(); + let fragment=self.passes[self.slot].get_mut(P::FragmentCommand).va(); + let barrier=self.passes[self.slot].get_mut(P::TilerToFragment).va(); + let dependency=self.passes[self.slot].get(dep).va(); + let init=self.init_bm.va(); + let ta_commands=[init,dependency,ta]; + let fs_commands=[self.passes[self.slot].get(fs_dep).va(),barrier,fragment]; + for (stage,ring,pointers,addresses) in [ + (0,s::TA_RING,s::TA_POINTERS,&ta_commands[usize::from(draw!=1)..]), + (1,s::FRAGMENT_RING,s::FRAGMENT_POINTERS,&fs_commands[usize::from(!early)..])] { + for &address in addresses { + self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; + self.heads[stage]=(self.heads[stage]+1)%0x500; + } + crate::agx_memory::publish(); + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + } + /// Emitted only after both events, stamp/queue checks, clear faults, + /// idle engines and consumed firmware pipes have been checked by Runtime. + /// The existing TA/fragment descriptors request completion stamp flushes. + pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + // Keep summaries sparse at high throughput, but emit at least once a + // second while completed batches advance. Batched draw counts can skip + // multiples of 128; a count-only gate can starve the watchdog observer. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.draw != 1 && self.draw % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(),"M3_RETIRED draw={} ta={:x}/{:x} fragment={:x}/{:x} queues={}/{} read={}/{} idle=1 cache_flush=1 events=2 faults=0 ordered_barrier=1 batch={} early={}\n", + self.draw,self.objects[s::TA_STAMP].read_u32(0)?,self.objects[s::TA_FW_STAMP].read_u32(0)?, + self.objects[s::FRAGMENT_STAMP].read_u32(0)?,self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?, + self.objects[s::TA_POINTERS].read_u32(queue::DONE)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::DONE)?, + self.objects[s::TA_POINTERS].read_u32(queue::READ)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::READ)?,self.batch_count,self.early_count); + Ok(()) + } + pub(crate) fn queues(&self)->[u64;2] {[self.objects[s::TA_QUEUE].va(),self.objects[s::FRAGMENT_QUEUE].va()]} + /// Read retained command bytes only after exact batch retirement. + pub(crate) fn stamp_flush_masks(&mut self,base:usize,count:usize)->Result<[u16;2]> { + if base>=pass_layout::SLOTS || count==0 || count>pass_layout::SLOTS {return Err(EINVAL);} + let mut masks=[0;2]; + for slot in 0..count { + let pass=&mut self.passes[Self::at(base,slot)]; + for (stage,field,offset) in [(0,P::TilerCommand,tiler::FLUSH_STAMPS), + (1,P::FragmentCommand,fragment::FLUSH_STAMPS)] { + if pass.get_mut(field).read_u32(offset)?!=0 {masks[stage]|=1u16<Result<[bool;2]> { + let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw); + Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value, + self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value]) + } + /// Observe each completion condition across both render queues independently. + pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> { + let mut done=[true;4]; + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + done[0]&=self.objects[stamp].read_u32(0)?==value; + done[1]&=self.objects[fw].read_u32(0)?==value; + done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head); + done[3]&=self.objects[pointers].read_u32(queue::READ)?==u32::from(head); + } + Ok(done) + } + pub(crate) fn complete(&mut self)->Result { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value + || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) + || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} + } Ok(true) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + // Retain allocator state as well as payloads: an exhausted parameter + // pool can stall TA without producing an MMU fault. + for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { + for offset in (0..bytes).step_by(4) { + let value=self.objects[index].read_u32(offset)?; + dev_info!(dev.as_ref(),"M3_PB object={} offset={:#x} value={:#x}\n",index,offset,value); + } + } + dev_info!(dev.as_ref(),"M3 fault render VM={} slot={} root={:#x} draw={} batch={}\n",self._binding.vm_id(),self._binding.slot(),self._binding.root(),self.draw,self.batch_count); + for index in 0..self.batch_count { + let slot=Self::at(self.base,index); + let command=self.passes[slot].get_mut(P::TilerCommand); + for index in 0..tiler::REGISTER_COUNT { + let offset=tiler::REGISTERS+index*tiler::REGISTER_STRIDE; + if command.read_u32(offset)?==0x1c880 { + dev_info!(dev.as_ref(),"M3 fault pass slot={} compact_vdm={:#x}\n",slot,command.read_u64(offset+4)?); + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + // Preserve historic diagnostic IDs while selecting named owners. + for (index,bytes) in s::fault_buffers() { + let b=match index {s::TILEMAP_LABEL=>self.passes[0].get_mut(P::Tilemap), + s::HEAP_METADATA_LABEL=>self.passes[0].get_mut(P::HeapMetadata),_=>&mut self.objects[index]}; + for offset in (0..bytes).step_by(16) { + let lo=b.read_u64(offset)?; + let hi=b.read_u64(offset+8)?; + if lo!=0 || hi!=0 {dev_info!(dev.as_ref(),"M3_BUFFER object={} offset={:#x} {:016x} {:016x}\n",index,offset,lo,hi);} + } + } + } + for (name,stamp,fw,pointers) in [("TA",s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS), + ("fragment",s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS)] { + dev_info!(dev.as_ref(),"M3 {} stamps={:#x}/{:#x} done={} read={}\n",name, + self.objects[stamp].read_u32(0)?,self.objects[fw].read_u32(0)?, + self.objects[pointers].read_u32(queue::DONE)?,self.objects[pointers].read_u32(queue::READ)?); + } Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs new file mode 100644 index 0000000000000..bfc94fdfa0ca8 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 GPU resources admitted against the measured boot contract (J514S on T6030). +pub(crate) use crate::agx_resources::{Region, Resources}; + +use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; + +/// Live overlays are not inserted in OF's boot-time reserved_mem registry. +/// Keep the ordinary lookup for every static handoff. A no-map node is +/// accepted only when its range is outside all Linux System RAM. +pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result { + if let Ok(r) = node.reserved_mem_region_to_resource_byname(name) { + return Ok(r); + } + if let Some(found) = crate::m3_board::static_region(node, name) { + return found.map(|(r, _)| r); + } + // Only for a GPU node whose compatible is exactly that of an M3 SoC table. + let compatible: KVec = node.get_property(c_str!("compatible"))?; + let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; + if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + return Err(EINVAL); + } + let names: KVec = node.get_property(c_str!("memory-region-names"))?; + let index = names.split(|b| *b == 0).position(|n| n == name.to_bytes()).ok_or(EINVAL)?; + let region = node.parse_phandle(c_str!("memory-region"), index).ok_or(EINVAL)?; + let nomap: KVec = region.get_property(c_str!("no-map"))?; + let reg: KVec = region.get_property(c_str!("reg"))?; + if !nomap.is_empty() || reg.len()!=2 { return Err(EINVAL); } + let (base,size)=(reg[0],reg[1]); + if base < 1<<40 || size == 0 || (base|size)&0x3fff != 0 + || base.checked_add(size).ok_or(EOVERFLOW)? > 1<<42 { return Err(EINVAL); } + // SAFETY: read-only resource-tree query with checked numeric bounds. + if unsafe { bindings::region_intersects(base, size.try_into()?, bindings::IORESOURCE_SYSTEM_RAM as _, 0) } != bindings::REGION_DISJOINT as i32 { + return Err(EBUSY); + } + let raw = bindings::resource { + start: base, end: base+size-1, flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque, which + // preserves layout. The initialized descriptor is owned, with no pointers + // into temporary data; its range is checked above. + Ok(unsafe { core::mem::transmute::(raw) }) +} + +pub(crate) fn from_device( + pdev: &kernel::platform::Device, + soc: &crate::m3_soc::Soc, +) -> kernel::error::Result { + crate::m3_board::admit(pdev, soc) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs new file mode 100644 index 0000000000000..3e8cbbfa93781 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! RTKit callbacks for the identified J514S firmware. The firmware advertises +//! its crash storage by physical address inside the reserved data segment. + +use crate::agx_resources::Region; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use kernel::{ + c_str, impl_has_hr_timer, + device::Core, + sync::Completion, + io::mem::{Mem, MemFlag}, + iosys_map::IoSysMapRef, + platform, + prelude::*, + soc::apple::rtkit, + sync::{aref::ARef, Arc, ArcBorrow}, + time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, + HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, +}; + +/// IRQ-safe notification tokens for the single GPU completion worker. +#[pin_data] +pub(crate) struct EventWait { + count: AtomicU64, + wait_us: AtomicU64, + overlap_cpu: AtomicBool, + cpu_preparations: AtomicU64, + sleeps: AtomicU64, + timer_wakes: AtomicU64, + #[pin] + arrived: Completion, + #[pin] + timer: HrTimer, +} + +impl EventWait { + fn new() -> Result> { + Arc::pin_init( + pin_init!(EventWait { + count: AtomicU64::new(0), + wait_us: AtomicU64::new(100), + overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + cpu_preparations: AtomicU64::new(0), + sleeps: AtomicU64::new(0), + timer_wakes: AtomicU64::new(0), + arrived <- Completion::new(), + timer <- HrTimer::new(), + }), + GFP_KERNEL, + ) + } + + fn record(&self) { + self.count.fetch_add(1, Ordering::Release); + self.arrived.complete(); + } + + /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// stamps, pipe retirement, event counts, fences or the two-second timeout. + /// Atomic publication permits root to compare policies in the same boot. + pub(crate) fn set_wait_us(&self, us: u64) -> Result { + if !(25..=1000).contains(&us) { return Err(EINVAL); } + self.wait_us.store(us, Ordering::Relaxed); + Ok(()) + } + + pub(crate) fn cpu_overlap(&self) -> bool { + self.overlap_cpu.load(Ordering::Relaxed) + } + pub(crate) fn set_cpu_overlap(&self, enabled: bool) { + self.overlap_cpu.store(enabled, Ordering::Relaxed); + } + + pub(crate) fn record_cpu_preparation(&self) { + self.cpu_preparations.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn snapshot(&self) -> (u64, u64, u64, u64, u64) { + (self.wait_us.load(Ordering::Relaxed), self.count.load(Ordering::Acquire), + self.sleeps.load(Ordering::Relaxed), self.timer_wakes.load(Ordering::Relaxed), + self.cpu_preparations.load(Ordering::Relaxed)) + } + + /// Sleep until a notification later than `seen` has been recorded or + /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// even when the GPU stamp arrives just after its notification. The timer + /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. + /// Only the single completion worker may call this method. + pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + // Events can arrive while the worker is polling. Discard stale wakeup + // tokens before checking the sequence. A concurrent event either moves + // the sequence or leaves a token, including between the check and wait. + // Bound draining so a busy interrupt source cannot hold this worker. + for _ in 0..64 { + if !self.arrived.try_wait_for_completion() { + if self.count.load(Ordering::Acquire) != seen { return true; } + // The handle keeps EventWait alive and cancels synchronously + // on every return, before a subsequent wait can arm the timer. + self.sleeps.fetch_add(1, Ordering::Relaxed); + let timer = self.clone().start(Delta::from_micros( + self.wait_us.load(Ordering::Relaxed) as i64)); + // Retain one tick as a backup bound; normally either the real + // interrupt or our high-resolution timer supplies the token. + self.arrived.wait_for_completion_timeout(1); + drop(timer); + return self.count.load(Ordering::Acquire) != seen; + } + } + // Resnapshot GPU progress before draining another batch of tokens. + true + } +} + +impl HrTimerCallback for EventWait { + type Pointer<'a> = Arc; + + fn run(this: ArcBorrow<'_, Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { + // Wake only: a timer expiry is not a firmware completion notification. + this.timer_wakes.fetch_add(1, Ordering::Relaxed); + this.arrived.complete(); + HrTimerRestart::NoRestart + } +} + +impl_has_hr_timer! { + impl HasHrTimer for EventWait { + mode: RelativeMode, field: self.timer + } +} + +pub(crate) struct State { + dev: ARef, + drm: crate::driver::AsahiDevRef, + data: Region, + pub(crate) health: Arc, + claimed: AtomicBool, + /// Firmware event notifications received on the application endpoint. + /// The firmware sends one after a command's completion processing. + pub(crate) event_messages: AtomicU64, + pub(crate) events: Arc, +} + +/// Shared status contains no device references, so DRM files may retain it +/// after runtime teardown without retaining MMIO or creating a DRM cycle. +pub(crate) struct Health { + crashed: AtomicBool, + mapped: AtomicBool, + failed: AtomicBool, + progress: crate::agx_host_progress::Progress, + pub(crate) timing: crate::agx_timing_stats::Stats, +} + +impl Health { + pub(crate) fn healthy(&self) -> bool { + self.mapped.load(Ordering::Acquire) + && !self.crashed.load(Ordering::Acquire) + && !self.failed() + } + + pub(crate) fn set_gpu_pending(&self, pending: bool) { self.progress.set_pending(pending); } + + pub(crate) fn activity_snapshot(&self) -> (u64, u64, bool) { + let (generation, epoch) = self.progress.activity_snapshot(); + (generation, epoch, self.healthy()) + } + + pub(crate) fn record_completion(&self) { self.progress.record_completion(); } + + pub(crate) fn progress_snapshot(&self) -> (u64, u64, u64, bool) { + let (generation, count, timestamp) = self.progress.snapshot(); + (generation, count, timestamp, self.healthy()) + } + + pub(crate) fn failed(&self) -> bool { self.failed.load(Ordering::Acquire) } + + /// Whether RTKit reported a firmware crash. + pub(crate) fn crashed(&self) -> bool { self.crashed.load(Ordering::Acquire) } + + pub(crate) fn mark_failed(&self) { self.failed.store(true, Ordering::Release); } +} + +impl State { + pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + Ok(Arc::new( + Self { + dev: dev.into(), + drm, + data, + health: Arc::new(Health { + crashed: AtomicBool::new(false), + mapped: AtomicBool::new(false), + failed: AtomicBool::new(false), + progress: crate::agx_host_progress::Progress::new(), + timing: crate::agx_timing_stats::Stats::new(), + }, GFP_KERNEL)?, + claimed: AtomicBool::new(false), + event_messages: AtomicU64::new(0), + events: EventWait::new()?, + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn healthy(&self) -> bool { + self.health.healthy() + } +} + +pub(crate) struct FirmwareBuffer { + state: Arc, + mapping: Mem, + physical: usize, + offset: usize, + size: usize, +} + +impl Drop for FirmwareBuffer { + fn drop(&mut self) { + // Includes allocation failure in RTKit's wrapper after shmem_map + // returned: readiness must not survive loss of the actual mapping. + self.state.health.mapped.store(false, Ordering::Release); + } +} + +impl rtkit::Buffer for FirmwareBuffer { + fn iova(&self) -> Result { + Ok(self.physical) + } + fn buf(&mut self) -> Result> { + self.mapping.iosys_map(self.offset, self.size) + } +} + +pub(crate) struct Operations; + +#[kernel::macros::vtable] +impl rtkit::Operations for Operations { + type Data = Arc; + type Buffer = FirmwareBuffer; + + fn crashed(state: ArcBorrow<'_, State>, crashlog: Option<&[u8]>) { + state.health.crashed.store(true, Ordering::Release); + // Wake the serialized worker even if the firmware can no longer send + // its usual completion notification. Never take the runtime lock here. + state.events.record(); + crate::m3_completion::queue(&state.drm); + dev_err!( + state.dev.as_ref(), + "M3 G15S: firmware crashed, retained crashlog bytes={}\n", + crashlog.map_or(0, |b| b.len()) + ); + if let Some(bytes) = crashlog { + crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); + } + } + + fn shmem_map( + state: ArcBorrow<'_, State>, + physical: usize, + size: usize, + ) -> Result { + // The only system buffer advertised by this admitted firmware is its + // preallocated crash log. Never reinterpret an arbitrary DVA as a PA. + let offset = physical + .checked_sub(state.data.base as usize) + .ok_or(EINVAL)?; + if size == 0 + || (physical | size) & 0xfff != 0 + || offset.checked_add(size).ok_or(EOVERFLOW)? > state.data.size as usize + { + return Err(EINVAL); + } + if state.claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; + let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + if resource.start() != state.data.base || resource.size() != state.data.size { + return Err(EINVAL); + } + // SAFETY: The validated no-map data reservation belongs to this + // firmware session. This CPU mapping is retained by RTKit until ASC + // is stopped and callbacks drained; physical pages are never freed. + // An uncached CPU alias lets RTKit snapshot newly written crash data. + let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?; + dev_info!( + state.dev.as_ref(), + "M3 G15S: mapped firmware crash storage PA={:#x} size={:#x}\n", + physical, + size + ); + state.health.mapped.store(true, Ordering::Release); + Ok(FirmwareBuffer { + state: state.into(), + mapping, + physical, + offset, + size, + }) + } + + fn recv_message(state: ArcBorrow<'_, State>, endpoint: u8, message: u64) { + // Firmware retries its event notification until the shared rings are + // drained, including after the final userspace job. Never take the + // runtime mutex in the RTKit callback: enqueue the serialized worker. + if endpoint == 0x20 && message == 0x0042_0000_0000_0000 { + // Record the IRQ-safe token before publishing the external count. + // A concurrent waiter sees either the moved sequence or a token. + state.events.record(); + state.event_messages.fetch_add(1, Ordering::Release); + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(state.dev.as_ref(), "M3 G15S_TIMING event monotonic_ns={} messages={}\n", + ::ktime_get(), + state.event_messages.load(Ordering::Acquire)); + } + crate::m3_completion::queue(&state.drm); + return; + } + dev_warn!( + state.dev.as_ref(), + "M3 G15S: RTKit application ep={:#x} message={:#018x}\n", + endpoint, + message + ); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs new file mode 100644 index 0000000000000..08885f591c140 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -0,0 +1,1246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use core::{mem::ManuallyDrop, sync::atomic::Ordering}; +use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, + time::{Instant, Monotonic, Delta, delay::fsleep}}; +use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; + +#[path = "m3_batch_policy.rs"] +pub(crate) mod policy; +const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); + +enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +/// Completion-worker outcome. +pub(crate) enum Service {Idle,Progress,Waiting(u64)} +/// The published batch: its commands in order, as (packet, command index). +struct Batch { + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3_submit::Command, + preparation_ns: i64, previous_events: u64, expected_events: u64, + start: Instant, measure: bool, doorbell_counter_ns: i64, + /// Render: first pass slot and the batch's last draw number. + base: usize, last_draw: u64, publication_tick:u64, + polling_ns: i64, polls: u64, stamp_ns: i64, first_event_ns: i64, events_ns: i64, + completion_ns: [i64;4], + /// First successful stamp observations and the preceding incomplete reads, in 24 MHz ticks. + fragment_host_tick:u64, fragment_fw_tick:u64, + fragment_host_pending_tick:u64, fragment_fw_pending_tick:u64, + pipes_idle_tick:u64, + coalesced_stamps: u64, + mixed: Option, +} +struct MixedBatch {compute_index:usize, renders:usize, computes:usize, dependencies:usize} +#[derive(Clone, Copy, Default)] +struct GeometryTiming { + width: u16, height: u16, samples: u8, count: u64, + prepare_ns: i64, active_ns: i64, gpu_ns: u64, + tiling_ns: u64, fragment_ns: u64, max_fragment_ns: u64, + polling_ns: i64, retirement_ns: i64, sleeping_ns: i64, polls: u64, + stamp_ns:i64, first_event_ns:i64, events_ns:i64, pickup_ns:i64, tail_ns:i64, + completion_ns: [i64;4], +} +impl NativeJob { + fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} + fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} +} +struct Inner { + transport: rtkit::RtKit, + state: Arc, + jobs: KVec, + packets: KVec>, + /// Packets not yet (fully) published: (packet, next command index), most urgent first. + pending: KVec<(Arc,usize)>, + /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. + active: KVec, + gpu_pending: bool, + active_power_last: Option>, + active_power_samples: u32, + fault_captured: bool, + timing: [[i64;7];2], + render_batches: [u64;crate::m3_pass_layout::SLOTS], + geometry: [GeometryTiming; 32], + geometry_overflow: u64, + /// Whether a T8122 job verdict line was already emitted for the current job, so the + /// execute-path fallback does not add a second one. T8122 only. + t8122_verdict: bool, + /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. + boot_step: crate::t8122_start::BootStep, + /// Packets that joined a batch started by an earlier packet (debug summary). + coalesced_packets: u64, + coalesced_render_stamps: u64, + mixed_batches:u64, + /// Recent measured render GPU time per pass by VM. Smoothed over batches; + /// replacing the least recently measured VM does not retain its mappings. + pass_cost: policy::PassCosts, + /// Latched only after the isolated InitBM draw fully retired. + render_initialized: bool, + config: Config, + uat: mmu::Uat, + drm: driver::AsahiDevRef, + device: Device, +} +impl Inner { + /// Latch before allocating diagnostics, once per runtime. Called under + /// its mutex even for errors outside the active-job polling loop. + fn capture_fault(&mut self, primary: Error) { + self.state.health.mark_failed(); + if self.fault_captured { return; } + self.fault_captured = true; + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { + dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); + } + } + + /// Record a batch that failed or never retired. Mark the GPU failed first, + /// so that a failing diagnostic read cannot leave a job that never retired + /// behind a device that still reports itself healthy. + fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { + self.capture_fault(primary); + if crate::t8122_start::is_t8122(self.device.soc()) { + let pstate=self.device.pstate_register().ok(); + // The job's first GPU start timestamp: nonzero means the firmware dispatched it, + // whatever the power state reads at the timeout. + let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); + crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + self.t8122_verdict=true; + } + let events=self.state.event_messages.load(Ordering::Acquire); + if let Err(e) = self.jobs[index].log(&self.drm) { + dev_err!(self.drm.as_ref(), "M3 job diagnostics failed: {:?}\n", e); + } + if let Err(e) = self.device.log_engine_state(vm) { + dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); + } + let _=self.config.log_recovery_state(&self.drm,events); + if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if let Err(e)=self.resume_experiment(index, vm) { + dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); + } + } + } + /// asahi.g15_debug bit 56. Wait up to 100 ms for the firmware to report + /// that it halted, resume it, and log every change of the halt words, + /// the engine state, the failed job's retirement, whether every queue + /// message was consumed and the event count for about half a second, + /// then the event ring entries the host has not consumed. The GPU stays + /// marked failed; the event ring's read index is not advanced. + fn resume_experiment(&mut self, index: usize, vm: &mmu::Vm) -> Result { + let start=Instant::::now(); + let mut halt=self.config.halt_state()?; + while halt.1==0 && start.elapsed()::now(); + let mut last=None; + for _ in 0..500 { + let halt=self.config.halt_state()?; + // 0: idle and no fault bits, 1: an engine is busy, 2: fault bits set. + let engines=match self.device.check_idle() {Ok(())=>0u8,Err(e) if e==EBUSY=>1,Err(_)=>2}; + let complete=self.jobs[index].complete().unwrap_or(false); + // 0: queues not drained, 1: every submitted queue message consumed, 2: read failed. + let pipes=match self.config.pipes_idle() {Ok(false)=>0u8,Ok(true)=>1,Err(_)=>2}; + let events=self.state.event_messages.load(Ordering::Acquire); + let crashed=self.state.health.crashed(); + let now=(halt,engines,complete,pipes,events,crashed); + if last!=Some(now) { + dev_info!(self.drm.as_ref(),"M3: resume-after-fault: +{}us: halt_count={} halted={} resume={} engines={} job_complete={} pipes_idle={} event_messages={} crashed={}\n", + resumed.elapsed().as_nanos()/1000,halt.0,halt.1,halt.2,engines,complete,pipes,events,crashed); + last=Some(now); + } + if crashed {break;} + fsleep(Delta::from_millis(1)); + } + self.jobs[index].log(&self.drm)?; + self.device.log_engine_state(vm)?; + self.config.log_event_ring(&self.drm)?; + let events=self.state.event_messages.load(Ordering::Acquire); + self.config.log_recovery_state(&self.drm,events) + } +} +impl Inner { + /// Fail every packet that was never published. Their fences signal with the error. + fn fail_pending(&mut self, error: Error) { + for (packet,_) in self.pending.drain_all() {packet.finish(Err(error));} + } + /// A published batch failed or never retired: latch the fault, log it, and fail its + /// packets. The packets stay retained (with their VM job guards) until teardown, because + /// the firmware has not provably stopped using their memory. + fn fail_active(&mut self, error: Error) { + let _=self.config.performance_request(self.device.soc().chip_id,false); + if let Some((index,vm))=self.active.first().map(|b|(b.index,b.entries[0].0.vm.clone())) { + self.log_owners(error); + self.fail(index,&vm,error); + } else {self.capture_fault(error);} + for batch in self.active.drain_all() {Self::finish_all(&batch.entries,Err(error));} + self.fail_pending(error); + } + /// Name the processes whose published commands were in flight when the GPU failed. Batches + /// in flight share one VM, so this is normally one process; a VM shared between processes + /// names each submitter once (the first four). + fn log_owners(&self, error: Error) { + let commands:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let mut logged=[0i32;4]; + let mut count=0; + for (packet,_) in self.active.iter().flat_map(|b|b.entries.iter()) { + let owner=&packet.owner; + if logged[..count].contains(&owner.pid) {continue;} + if count==logged.len() {break;} + dev_err!(self.drm.as_ref(),"M3 failed work owner: VM={} process {} pid {} thread {} (error {:?}, {} commands in flight)\n", + packet.vm.id(),owner.process(),owner.pid,owner.thread(),error,commands); + logged[count]=owner.pid;count+=1; + } + } + /// Hardware completion is separate from fence signaling: cancellation can win the + /// signal race while the DMA guard must survive until its final published batch retires. + fn packet_active(&self, packet:&Arc)->bool { + self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) + } + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + for (packet,_) in entries { + let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); + if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} + } + // Keep exactly active DMA owners. Successful retirement may precede a VM switch. + self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q)))); + } + fn fail_front(&mut self,error:Error) { + if let Ok((packet,_))=self.pending.remove(0) { + if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();} + packet.finish(Err(error)); + } + } + /// Signal every packet of `entries` (packets that continue past the batch included). + fn finish_all(entries:&[(Arc,usize)],result:Result) { + for (packet,_) in entries {packet.finish(result);} + } + /// Choose the next batch from the front of `pending`: the front packet's next commands of + /// one engine, and for render also the leading render commands of following packets of the + /// same VM, up to the ordered-batch limit. + fn next_batch(&self,max:usize)->Result,usize)>> { + let mut entries=KVec::new(); + let Some((packet,next))=self.pending.first() else {return Ok(entries);}; + let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + if mixed>0 { + for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} + return Ok(entries); + } + let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let limit=if render { + // The first render draw initializes the shared buffer manager (InitBM) and opens + // both queues. Publish it alone: appending later draws first advances the shared + // manager and event counters it starts from, and on J613 the second pass's + // fragment then fails with firmware error event 4 (draw=2 batch=2). + let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); + if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + } else { + crate::m3_params::compute_batch_size() + .clamp(1,crate::m3_compute_storage::SLOTS) + }; + let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let per_pass=self.pass_cost.get(packet.vm.id()); + let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + for (position,(candidate,start)) in self.pending.iter().enumerate() { + // Later packets join only render batches, only from their first command, and only + // for the VM the batch is bound to, while the batch's expected GPU time is within + // the budget. + if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + let mut index=*start; + while index=limit || indexResult> { + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + if !self.render_initialized {return Ok(None);} + let Some((packet,next))=self.pending.first() else {return Ok(None);}; + if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; + // Yield at the next retirement when more urgent work is already waiting. + if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), + last.entries[0].0.urgency) {return Ok(None);} + if !policy::within_budget(self.pass_cost.get(packet.vm.id()), + crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + if !policy::overlap_ready(self.render_initialized,true,true,true, + self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} + Ok(Some(room)) + } + /// Publish pending batches while allowed: one when none is active, more behind it with + /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed. + fn publish_next(&mut self)->bool { + let mut progress=false; + while !self.pending.is_empty() && self.state.healthy() { + let overlap=!self.active.is_empty(); + let (base,max)=match self.pipeline_slot() { + Ok(Some(slot))=>slot, + Ok(None)=>break, + Err(error)=>{self.fail_active(error);return true;} + }; + let entries=match self.next_batch(max) { + Ok(entries) if !entries.is_empty()=>entries, + _=>{self.fail_front(ENOMEM);progress=true;continue;}, + }; + match self.publish(&entries,base,overlap) { + Ok(batch)=>{ + // Consume the published commands from `pending`. + let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;}; + while let Some((packet,_))=self.pending.first() { + if Arc::ptr_eq(packet,&last) { + if last_index+1==last.commands.len() {let _=self.pending.remove(0);} + else {self.pending[0].1=last_index+1;} + break; + } + let _=self.pending.remove(0); + } + if self.active.push(batch,GFP_KERNEL).is_err() { + // Published but untracked: it can never be retired. + self.capture_fault(ENOMEM);Self::finish_all(&entries,Err(ENOMEM)); + self.fail_active(ENOMEM);return true; + } + progress=true; + } + Err((error,published))=>{ + if published { + if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { + crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + self.t8122_verdict=true; + } + // A live ring WRITE or doorbell may have been published: keep packets + // retained (self.packets) and fail them and everything in flight. + self.capture_fault(error); + Self::finish_all(&entries,Err(error)); + self.fail_active(error); + self.fail_pending(error); + return true; + } else if overlap && error==EBUSY { + // Not publishable behind the running batch (new VM binding or storage + // growth): wait for it to retire. + break; + } else { + // Nothing reached the firmware: fail only the front packet, as a failed + // execute() did before publication. + self.fail_front(error); + progress=true; + } + } + } + } + progress + } + /// Prepare and publish one batch. On error, reports whether firmware may see its writes. + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + if render && compute {return self.publish_mixed(entries,base,overlap);} + let early=|e:Error|(e,false); + let (packet,first_index)=(&entries[0].0,entries[0].1); + let control=packet.commands[first_index]; + let batch_count=entries.len(); + let prepare=Instant::::now(); + // All host bookkeeping is reserved before mutating shared producer state. + // No allocation may fail after publication and lose a DMA owner. + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,i) in entries {owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + + // Retain the batch's packets (and their VM job guards) until it retires; with batches + // in flight, also those of the earlier ones. + if !overlap {self.packets.clear();} + for (p,_) in entries { + if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + } + let found=self.jobs.iter().position(|job| matches!((job,control), + (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + let index=found.unwrap_or(self.jobs.len()); + if found.is_none() { + let job=match control { + crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + }; + self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + } else { + match (&mut self.jobs[index],control) { + (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + _=>return Err(early(ENOTSUPP)), + } + } + if let NativeJob::Compute(j)=&mut self.jobs[index] { + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + if slot!=0 { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + j.append(c)?; + } + let timestamps=p.timestamp_addresses(*next); + let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + p.wide_visibility[*next],timestamps,slot+1==batch_count); + j.prepare_completion(coalesce)?; + j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.attachments[*next])?; + } + Ok(()) + })(); + if let Err(error)=prepared { + j.abort_batch().map_err(|e|(e,true))?; + if found.is_none() {drop(self.jobs.remove(index).map_err(|_|early(EIO))?);} + return Err(early(error)); + } + } + let mut coalesced_stamps=0; + let coalesce_render=crate::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let chip=self.device.soc().chip_id; + if let NativeJob::Render(j)=&mut self.jobs[index] { + let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,next) in entries {commands.push(p.commands[*next],GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*next); + let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, + overlap,slot,batch_count,timestamps); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); + j.append(command,usc,coalesce,timed)?; + j.set_user_timestamps(timestamps)?; + coalesced_stamps+=u64::from(coalesce); + } + Ok(()) + })(); + if let Err(error)=prepared { + // Running firmware can consume append's WRITE updates without a new doorbell. + // Even an append that returns an error may have changed shared counters or one + // pipe. Do not rewind live producer state or let the caller release owners and + // reuse these slots. Its published-error path stops scheduling and retains them. + if overlap {return Err((error,true));} + j.abort_batch().map_err(|e|(e,true))?; + return Err(early(error)); + } + } + let preparation_ns=prepare.elapsed().as_nanos(); + self.t8122_verdict=false; + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true); + self.gpu_pending = true; + let late=|e:Error|(e,true); + let expected_events=match &self.jobs[index] { + NativeJob::Compute(j)=>{ + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?;1 + }, + NativeJob::Render(j)=>{ + if j.first() {self.config.render_pb().map_err(late)?;} + let queues=j.queues(); + self.config.submit_queue(1,queues[1],j.heads()[1],1,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],j.heads()[0],0,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?;2 + }, + }; + let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); + let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + first:control, preparation_ns, previous_events, expected_events, + start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, + polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) + } + /// Prepare both idle queues from one packet. Cross-engine changes wait on + /// the preceding command's firmware stamp; all commands remain GPU ordered. + /// Producer storage and DMA guards survive every partial publication error. + fn publish_mixed(&mut self,entries:&[(Arc,usize)], + base:usize,overlap:bool)->core::result::Result { + let early=|e:Error|(e,false); + if overlap || base!=0 || self.device.soc().chip_id!=0x6030 + || !self.render_initialized || entries.len()>policy::SLOTS {return Err(early(EINVAL));} + let packet=&entries[0].0; + if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} + let prepare=Instant::::now(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_commands=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut first_compute=None; + let mut computes=0; + for (p,i) in entries { + owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + match p.commands[*i] { + c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + } + } + let renders=render_commands.len(); + if renders==0 || computes==0 {return Err(early(EINVAL));} + let index=self.jobs.iter().position(|j|matches!(j,NativeJob::Render(r) if r.started())).ok_or(early(EINVAL))?; + let existing=self.jobs.iter().position(|j|matches!(j,NativeJob::Compute(_))); + let compute_index=existing.unwrap_or(self.jobs.len()); + // Keep the packet before any producer mutation. No other batch is live. + self.packets.clear(); + self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_prepared=false; + let mut compute_prepared=false; + let prepared=(||->Result { + let control=first_compute.ok_or(EINVAL)?; + if existing.is_none() { + let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + self.config.stats_region()?,control,self.device.soc().registers)?; + self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; + } else { + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + j.replay(&self.uat,&packet.vm,control)?; + } + compute_prepared=true; + if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.reserve_barriers(&self.drm,&self.uat)?; + } + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; + render_prepared=true; + let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut compute_slot=0; + let mut dependencies=0; + for (p,i) in entries { + match p.commands[*i] { + crate::m3_submit::Command::Render{command,usc}=>{ + let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*i); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); + j.append_after(command,usc,false,predecessor,timed)?; + j.set_user_timestamps(timestamps)?; + dependencies+=usize::from(predecessor.is_some()); + previous=Some((true,j.completion_dependency()?)); + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + if compute_slot!=0 {j.append(c)?;} + if let Some((true,dependency))=previous { + j.insert_dependency(dependency)?; + dependencies+=1; + } + // Every cross-engine predecessor flushes its stamp. + // Final queue, timestamp and cache visibility are unchanged. + j.prepare_completion(false)?; + j.set_user_timestamps(p.timestamp_addresses(*i))?; + j.set_attachments(&p.attachments[*i])?; + previous=Some((false,j.completion_dependency()?)); + compute_slot+=1; + }, + } + } + Ok(dependencies) + })(); + let dependencies=match prepared { + Ok(n)=>n, + Err(error)=>{ + if render_prepared { + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err((EIO,true));}; + j.abort_batch().map_err(|e|(e,true))?; + } + if compute_prepared { + if existing.is_none() {drop(self.jobs.remove(compute_index).map_err(|_|(EIO,true))?);} + else if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.abort_batch().map_err(|e|(e,true))?; + } + } + return Err(early(error)); + }, + }; + let preparation_ns=prepare.elapsed().as_nanos(); + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true);self.gpu_pending=true; + let late=|e:Error|(e,true); + let NativeJob::Render(j)=&self.jobs[index] else {return Err(late(EIO));}; + let queues=j.queues();let heads=j.heads();let (_,_,last_draw)=j.batch(); + self.config.submit_queue(1,queues[1],heads[1],1,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],heads[0],0,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?; + let NativeJob::Compute(j)=&self.jobs[compute_index] else {return Err(late(EIO));}; + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?; + Ok(Batch {entries:owned,index,kind:2,first:packet.commands[entries[0].1], + preparation_ns,previous_events,expected_events:3,start:Instant::::now(), + measure:crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary), + doorbell_counter_ns:0,polling_ns:0,polls:0,stamp_ns:-1,first_event_ns:-1, + events_ns:-1,completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0,coalesced_stamps:0,base:0,last_draw,publication_tick, + mixed:Some(MixedBatch {compute_index,renders,computes,dependencies})}) + } + /// A mixed publication retires only after both jobs, all three queue + /// messages, firmware health and engine idle passed the ordinary checks. + /// Its wall time is reported once rather than charged to both engines. + fn retire_mixed(&mut self,batch:&Batch,mixed:&MixedBatch)->Result { + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_ns=j.gpu_ns(0,mixed.renders)?; + let stages=j.stage_ns(0,mixed.renders)?; + j.progress(&self.drm)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_ns=j.gpu_ns()?; + j.progress(&self.drm)?; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let vm=batch.entries[0].0.vm.id(); + let observed_tick=physical_counter(); + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_span=j.gpu_ticks(0,mixed.renders)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_span=j.batch_gpu_span()?; + let start=render_span.0.min(compute_span[0]); + let end=render_span.1.max(compute_span[1]); + let valid=batch.publication_tick!=0 && start>=batch.publication_tick && end>=start && observed_tick>=end; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} mixed=1 batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + vm,batch.entries.len(),batch.publication_tick,start,end,observed_tick,u32::from(valid)); + self.log_batch_milestones(batch,render_span.0,render_span.1,render_ns,stages[1]); + // Render and compute have separate slot sequences in one mixed batch. + let mut render_slot=0; + let mut compute_slot=0; + for (packet,next) in batch.entries.iter() { + match packet.commands[*next] { + crate::m3_submit::Command::Render{command:r,..}=>{ + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", + vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + render_slot+=1; + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(compute_slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", + vm,compute_slot,mixed.computes,c.base,c.end,c.usc_base,ns); + compute_slot+=1; + }, + } + } + } + self.state.health.record_completion(); + self.render_batches[mixed.renders-1]+=1; + self.pass_cost.record(batch.entries[0].0.vm.id(),render_ns/mixed.renders as u64); + self.mixed_batches+=1; + if (self.mixed_batches==1 || self.mixed_batches%128==0) && batch.measure { + dev_info!(self.drm.as_ref(),"M3_MIXED_BATCH_RETIRED count={} commands={} render={} compute={} barriers={} prepare_ns={} active_ns={} render_gpu_ns={} compute_gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} stamps=1 queues=1 events=3 idle=1\n", + self.mixed_batches,batch.entries.len(),mixed.renders,mixed.computes,mixed.dependencies, + batch.preparation_ns,batch.start.elapsed().as_nanos(),render_ns,compute_ns,stages[0],stages[1],stages[2]); + } + if self.active.is_empty() { + self.gpu_pending=false;self.state.health.set_gpu_pending(false); + } + Ok(()) + } + /// Check the active batch once. Ok(true): retired (its packets are signalled). + fn poll_active(&mut self)->Result { + let Some((index,measure,last_draw))=self.active.first().map(|b|(b.index,b.measure,b.last_draw)) else {return Ok(false);}; + let poll_start=measure.then(Instant::::now); + if !self.state.healthy() || self.config.drain(&self.drm).is_err() { + self.fail_active(EIO);return Err(EIO); + } + // With later batches in flight, the oldest retires once its own last draw's stamps + // have arrived; the last one is also checked for exact queue, event and engine idle. + if self.active.len()>1 { + let done=match &mut self.jobs[index] {NativeJob::Render(j)=>j.complete_through(last_draw),_=>Err(EINVAL)}; + match done { + Ok(true)=>{ + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e);Self::finish_all(&batch.entries,Err(e)); + self.fail_active(e);return Err(e); + } + self.retire_packets(&batch.entries); + return Ok(true); + } + Ok(false)=>{ + let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?; + if expired { + dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + return Ok(false); + } + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + let secondary=self.active.first().and_then(|b|b.mixed.as_ref().map(|m|m.compute_index)); + let completion=(||->Result { + let primary=self.jobs[index].complete()?; + let other=match secondary {Some(i)=>self.jobs[i].complete()?,None=>true}; + Ok(primary && other) + })(); + let stamped=match completion {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}}; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let before=physical_counter(); + let parts=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.fragment_stamp_parts().ok(), + _=>None, + }; + let after=physical_counter(); + if let Some(parts)=parts { + if let Some(batch)=self.active.first_mut() { + if batch.fragment_host_tick==0 { + if parts[0] {batch.fragment_host_tick=after;} + else {batch.fragment_host_pending_tick=before;} + } + if batch.fragment_fw_tick==0 { + if parts[1] {batch.fragment_fw_tick=after;} + else {batch.fragment_fw_pending_tick=before;} + } + } + } + } + let power_enabled=crate::m3_params::active_power(); + if power_enabled { + let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); + if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + stamped,self.active_power_samples,since_last) { + self.active_power_last=Some(Instant::::now()); + self.active_power_samples+=1; + // Diagnostic read errors do not change job completion or recovery. + if let Err(e)=self.config.log_active_power(&self.drm,&self.device, + self.active_power_samples) { + dev_info!(self.drm.as_ref(),"M3_ACTIVE_POWER sample={} unavailable={:?}\n", + self.active_power_samples,e); + } + } + } + let completion_parts=if measure && secondary.is_none() { + match &mut self.jobs[index] { + NativeJob::Render(j)=>Some(if stamped {[true;4]} else { + match j.completion_parts() {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}} + }), + _=>None, + } + } else {None}; + let completed=self.config.completed_events; + let (evented,expired,previous)={ + let batch=self.active.first_mut().ok_or(EIO)?; + let evented=completed>=batch.previous_events+batch.expected_events; + if batch.measure { + let at=batch.start.elapsed().as_nanos(); + if stamped && batch.stamp_ns<0 {batch.stamp_ns=at;} + if let Some(parts)=completion_parts { + for (part,done) in parts.iter().enumerate() { + if *done && batch.completion_ns[part]<0 {batch.completion_ns[part]=at;} + } + } + if completed>batch.previous_events && batch.first_event_ns<0 {batch.first_event_ns=at;} + if evented && batch.events_ns<0 {batch.events_ns=at;} + } + if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;} + (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) + }; + if stamped && evented { + let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + match self.device.check_idle_parts( + retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + Ok(())=>match self.config.pipes_idle() { + Ok(true)=>{ + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let tick=physical_counter(); + if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} + } + if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { + if crate::t8122_start::is_t8122(self.device.soc()) { + crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + self.t8122_verdict=true; + } + self.fail_active(e);return Err(e); + } + } + // Stamps, both queue indices, required events, + // firmware health, engines and pipes are verified. + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e); + Self::finish_all(&batch.entries,Err(e)); + self.fail_pending(e); + return Err(e); + } + // Signal every packet whose last command was in this batch. A packet + // that continues with other commands is still at the front of `pending`. + self.retire_packets(&batch.entries); + return Ok(true); + }, + Ok(false)=>{}, // Firmware must consume every submitted queue message. + Err(e)=>{self.fail_active(e);return Err(e);} + }, + Err(e) if e==EBUSY=>{}, // Retirement can trail the event. + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + if expired { + dev_err!(self.drm.as_ref(),"M3 completion events={} previous={}\n",completed,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + Ok(false) + } + fn log_batch_milestones(&self,batch:&Batch,render_start:u64,render_end:u64, + render_gpu_ns:u64,fragment_sum_ns:u64) { + dev_info!(self.drm.as_ref(),"M3_BATCH_MILESTONE vm={} publication_tick={} draw={} mixed={} render_start_tick={} render_end_tick={} render_gpu_ns={} fragment_sum_ns={} fragment_host_tick={} fragment_fw_tick={} fragment_host_pending_tick={} fragment_fw_pending_tick={} pipes_idle_tick={} counter_hz=24000000\n", + batch.entries[0].0.vm.id(),batch.publication_tick,batch.last_draw,u32::from(batch.mixed.is_some()), + render_start,render_end,render_gpu_ns,fragment_sum_ns,batch.fragment_host_tick,batch.fragment_fw_tick, + batch.fragment_host_pending_tick,batch.fragment_fw_pending_tick,batch.pipes_idle_tick); + for (packet,command) in batch.entries.iter() { + dev_info!(self.drm.as_ref(),"M3_BATCH_PACKET vm={} publication_tick={} packet={:#x} command={}\n", + packet.vm.id(),batch.publication_tick,packet.profile_id(),command); + } + } + /// Bookkeeping for a retired batch, then signal its completed packets. + fn retire(&mut self,batch:&Batch)->Result { + if let Some(mixed)=&batch.mixed {return self.retire_mixed(batch,mixed);} + let retire_start=batch.measure.then(Instant::::now); + let index=batch.index;let batch_count=batch.entries.len();let control=batch.first; + self.state.health.record_completion(); + if self.active.is_empty() { + match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.progress(&self.drm)?, + NativeJob::Render(j)=>j.progress(&self.drm)?, + } + } + let retired_counter_ns=if batch.measure {physical_counter() as i64 * 1000 / 24} else {0}; + let ticks=match &mut self.jobs[index] { + NativeJob::Render(j) if batch.measure=>Some(j.gpu_ticks(batch.base,batch_count)?), + _=>None, + }; + let stages=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.stage_ns(batch.base,batch_count)?, + _=>[0;3], + }; + let (kind,gpu_ns)=match &mut self.jobs[index] { + NativeJob::Compute(j)=>(1,j.gpu_ns()?), + NativeJob::Render(j)=>(0,j.gpu_ns(batch.base,batch_count)?), + }; + if kind==0 { + self.render_batches[batch_count-1]+=1; + if batch.measure && batch.coalesced_stamps>0 && self.coalesced_render_stamps==0 { + if let NativeJob::Render(j)=&mut self.jobs[index] { + let masks=j.stamp_flush_masks(batch.base,batch_count)?; + dev_info!(self.drm.as_ref(),"M3_RENDER_STAMP_FLUSH vm={} batch={} ta_mask={:#x} fs_mask={:#x} coalesced={}\n", + batch.entries[0].0.vm.id(),batch_count,masks[0],masks[1],batch.coalesced_stamps); + } + } + self.coalesced_render_stamps+=batch.coalesced_stamps; + let vm=batch.entries[0].0.vm.id(); + self.pass_cost.record(vm,gpu_ns/batch_count as u64); + self.render_initialized=true; + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let observed_tick=physical_counter(); + let (cs,span)=match (&mut self.jobs[index],control) { + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + _=>return Err(EINVAL), + }; + let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} cs={:#x} batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + batch.entries[0].0.vm.id(),cs,batch_count,batch.publication_tick,span[0],span[1],observed_tick,u32::from(valid)); + self.log_batch_milestones(batch, + if kind==0 {span[0]} else {0},if kind==0 {span[1]} else {0}, + if kind==0 {gpu_ns} else {0},stages[1]); + if let NativeJob::Compute(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); + } + } + if let NativeJob::Render(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(batch.base,batch_count,slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + } + } + } + let preparation_ns=batch.preparation_ns; + let active_ns=batch.start.elapsed().as_nanos(); + self.state.health.timing.record(kind==0, batch_count as u64, + preparation_ns, active_ns, gpu_ns, stages); + let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); + if batch.measure { + if let crate::m3_submit::Command::Render {command:r,..}=control { + if batch_count>1 {self.geometry_overflow+=1;} else { + if let Some(g)=self.geometry.iter_mut().find(|g| + g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { + g.width=r.width;g.height=r.height;g.samples=r.samples; + g.count+=1;g.prepare_ns+=preparation_ns;g.active_ns+=active_ns; + g.gpu_ns+=gpu_ns;g.tiling_ns+=stages[0];g.fragment_ns+=stages[1]; + g.max_fragment_ns=g.max_fragment_ns.max(stages[1]); + g.polling_ns+=batch.polling_ns;g.retirement_ns+=retirement_ns; + g.polls+=batch.polls; + g.stamp_ns+=batch.stamp_ns;g.first_event_ns+=batch.first_event_ns;g.events_ns+=batch.events_ns; + for (total,observed) in g.completion_ns.iter_mut().zip(batch.completion_ns) {*total+=observed;} + if let Some((ta,fe))=ticks { + g.pickup_ns+=(ta as i64)*1000/24-batch.doorbell_counter_ns; + g.tail_ns+=retired_counter_ns-(fe as i64)*1000/24; + } + } else { + self.geometry_overflow+=1; + } + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + match control { + crate::m3_submit::Command::Render {command:r,..}=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), + crate::m3_submit::Command::Compute(_)=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), + } + } + let t=&mut self.timing[kind]; + t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; + for i in 0..3 {t[4+i]+=stages[i] as i64;} + // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. + if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + let span=match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.batch_gpu_span().ok(), + NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), + }; + crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); + } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + } + if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], + match &self.jobs[index] {NativeJob::Render(j)=>j.ordinal(),NativeJob::Compute(j)=>j.ordinal()}); + } + if kind==0 && t[0]%512==0 && + crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary) { + let early=match &self.jobs[index] {NativeJob::Render(j)=>j.early_count(),_=>0}; + dev_info!(self.drm.as_ref(),"M3_ORDERED_BATCHES depths={:?} early={} packets={} coalesced_stamps={}\n", self.render_batches,early,self.coalesced_packets,self.coalesced_render_stamps); + let now=::ktime_get(); + dev_info!(self.drm.as_ref(),"M3_GEOMETRY_BATCH ns={} render_count={} overflow={}\n", + now,t[0],self.geometry_overflow); + self.geometry_overflow=0; + for g in &mut self.geometry { + if g.count==0 {continue;} + dev_info!(self.drm.as_ref(),"M3_GEOMETRY ns={} width={} height={} samples={} count={} prepare_ns={} active_ns={} gpu_ns={} tiling_ns={} fragment_ns={} max_fragment_ns={} polling_ns={} retirement_ns={} sleeping_ns={} polls={} stamp_ns={} first_event_ns={} events_ns={} pickup_ns={} tail_ns={} gpu_stamp_ns={} fw_stamp_ns={} queue_done_ns={} queue_read_ns={}\n", + now,g.width,g.height,g.samples,g.count,g.prepare_ns,g.active_ns,g.gpu_ns,g.tiling_ns,g.fragment_ns,g.max_fragment_ns,g.polling_ns,g.retirement_ns,g.sleeping_ns,g.polls,g.stamp_ns,g.first_event_ns,g.events_ns,g.pickup_ns,g.tail_ns,g.completion_ns[0],g.completion_ns[1],g.completion_ns[2],g.completion_ns[3]); + *g=GeometryTiming::default(); + } + } + if self.active.is_empty() { + self.gpu_pending = false; + self.state.health.set_gpu_pending(false); + } + for pair in batch.entries.windows(2) { + if !Arc::ptr_eq(&pair[0].0,&pair[1].0) {self.coalesced_packets+=1;} + } + Ok(()) + } +} +// Keep the large graph on the heap: returning and moving it through probe, +// Registration and Mutex initialization otherwise duplicates it on the bounded +// kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop. +pub(crate) struct Runtime { inner: ManuallyDrop> } +impl Runtime { + #[inline(never)] + pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit + // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does + // not borrow `device`, which is moved into Inner below. + let t8122 = crate::t8122_start::is_t8122(device.soc()); + let t6031 = crate::t6031_start::is_t6031(device.soc()); + let coproc_refused = |e: &Error| { + if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + }; + device.require_stopped(pdev).inspect_err(coproc_refused)?; + // Reserve before starting ASC. ENOMEM must not drop firmware owners + // while the coprocessor may still access them. Writing the completed + // graph into this allocation is infallible. + let owner = KBox::::new_uninit(GFP_KERNEL)?; + let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; + let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; + if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + return Err(ENODEV); + } + // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. + let mut started=false; + let prepared=(|| -> Result<_> { + device.start_asc(pdev)?; + Pin::new(&mut transport).wake()?; + if !state.healthy() || !Pin::new(&transport).is_running() {return Err(EIO);} + for ep in [0x20,0x21] { + if !Pin::new(&mut transport).has_endpoint(ep) {return Err(ENODEV);} + Pin::new(&mut transport).start_endpoint(ep)?; + } + started=true; + // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). + // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU + // job has been published. Its running PPL handoff owns the peer lock. + let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + Ok((uat,config)) + })(); + let (uat,config)=match prepared { + Ok(v)=>v, + Err(e)=>{ + crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); + if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + state.health.mark_failed(); + if device.stop_asc().is_err() { + unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; + core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(drm);core::mem::forget(device); + } + return Err(e); + } + }; + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, + boot_step:crate::t8122_start::BootStep::Publish })) }) + } + pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } + pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { + use crate::util::RangeExt; + let reserved_range=0x70_0000_0000..0x80_0000_0000; + if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + let vm=self.inner.uat.new_vm(id, range)?; + let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; + vm.install_driver_mappings(KVec::new(),reserved)?; + Ok(vm) + } + pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + } + /// Queue a packet and return its fence at once. The scheduler can hand over further + /// ready jobs up to its credit limit while the GPU runs. Only the completion worker + /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded + /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full + /// retirement. A burst can therefore share a firmware round trip across packets. + pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + crate::debug::update_debug_flags(); + let mut guard=shared.lock(); + let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; + if !inner.state.healthy() {return Err(EIO);} + if packet.commands.is_empty() || packet.commands.len()>256 { + pr_err!("M3 submit limit: commands={} maximum=256\n",packet.commands.len()); + return Err(E2BIG); + } + // Higher urgency first, FIFO within one urgency, including unfinished packets. + // More urgent work can overtake a continuation between retired batches. + // Packets of one queue share an urgency, so their order is kept. + let urgency=packet.urgency; + let at=policy::pending_position(inner.pending.iter().map(|(p,_)|p.urgency),urgency); + if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} + inner.pending.reserve(1,GFP_KERNEL)?; + inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; + let drm=inner.drm.clone(); + drop(guard); + crate::m3_completion::queue(&drm); + Ok(()) + } + /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active + /// commands remain owned until retirement; error signaling cannot release a DMA guard. + pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + let mut guard=shared.lock(); + let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; + let inner: &mut Inner=&mut runtime.inner; + let removed=inner.pending.iter().position(|(p,_)|Arc::ptr_eq(p,packet)) + .is_some_and(|i|inner.pending.remove(i).is_ok()); + if !inner.packet_active(packet) && !inner.fault_captured {packet.release_guard();} + removed + } + pub(crate) fn completion_wait(&self) -> Arc { self.inner.state.events.clone() } + pub(crate) fn events(&self)->Arc {self.inner.state.events.clone()} + /// One completion-worker step: retire the active batch if it has completed, then publish + /// the next one. The worker drops the runtime lock between steps. + pub(crate) fn service_job(&mut self)->Service { + let chip=self.inner.device.soc().chip_id; + let busy=self.inner.state.healthy() && + (!self.inner.active.is_empty() || !self.inner.pending.is_empty()); + if let Err(error)=self.inner.config.performance_request(chip,busy) { + self.inner.fail_active(error); + return Service::Idle; + } + let result=self.service_job_inner(); + if matches!(result,Service::Idle) || !self.inner.state.healthy() || + (self.inner.active.is_empty() && self.inner.pending.is_empty()) { + let _=self.inner.config.performance_request(chip,false); + } + result + } + fn service_job_inner(&mut self)->Service { + let inner: &mut Inner=&mut *self.inner; + if inner.active.is_empty() { + if inner.fault_captured || !inner.state.healthy() {inner.fail_pending(EIO);return Service::Idle;} + if let Err(error)=inner.config.drain(&inner.drm) { + inner.capture_fault(error);inner.fail_pending(EIO);return Service::Idle; + } + return if inner.publish_next() {Service::Progress} else {Service::Idle}; + } + let messages=inner.state.event_messages.load(Ordering::Acquire); + match inner.poll_active() { + Ok(true)=>{inner.publish_next();Service::Progress}, + Ok(false)=>if inner.publish_next() {Service::Progress} else {Service::Waiting(messages)}, + Err(_)=>{inner.fail_pending(EIO);Service::Idle}, + } + } + pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { + let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); + let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); + self.inner.boot_step = crate::t8122_start::BootStep::Publish; + let result = self.boot_inner(pdev); + if let Err(error) = result { + // On T8122, log the firmware readiness words on every boot failure, so the verdict's + // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in + // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. + let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + && error == ETIMEDOUT; + if (is_t8122 || is_t6031) && !logged { + let inner: &mut Inner = &mut *self.inner; + let _ = inner.config.log_ready(&inner.drm); + } + self.inner.capture_fault(error); + } + if is_t8122 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + if is_t6031 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + result + } + fn boot_inner(&mut self, pdev: &platform::Device) -> Result { + use crate::t8122_start::BootStep; + let root = self.inner.config.root(); + dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); + self.inner.boot_step = BootStep::Publish; + Pin::new(&mut self.inner.transport).send_message(0x20, 0x0081000000000000 | (root & ((1u64<<44)-1)))?; + self.inner.boot_step = BootStep::DeviceControl; + self.send_control(0x13)?; + self.send_control(9)?; + self.inner.boot_step = BootStep::AwaitReady; + let start = Instant::::now(); + while !self.inner.config.ready()? { + if start.elapsed() >= Delta::from_secs(2) || !self.inner.state.healthy() { + let inner: &mut Inner=&mut *self.inner;let _=inner.config.log_ready(&inner.drm); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + self.inner.boot_step = BootStep::PostReady; + // Import the configured idle policy again after initial power-up. + self.send_control(0x13)?; + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + inner.config.log_ready(&inner.drm)?; + if inner.device.soc().chip_id==0x6030 && + (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + inner.config.log_globals_debug(&inner.drm)?; + } + inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; + inner.state.health.set_gpu_pending(false); + dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n"); + Ok(()) + } + fn send_control(&mut self, opcode: u32) -> Result { + let next = match self.inner.config.control(opcode) { + Ok(next) => next, + Err(error) => { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} queue failed ({:?})\n", opcode, error); + return Err(error); + } + }; + let send_start = Instant::::now(); + if let Err(error) = Pin::new(&mut self.inner.transport).send_message(0x21, 0x0083000000000011) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} mailbox send failed ({:?}), elapsed_ns={}\n", + opcode, error, send_start.elapsed().as_nanos()); + return Err(error); + } + let start = Instant::::now(); + loop { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if inner.config.control_done(next)? { return Ok(()); } + if start.elapsed() >= Delta::from_secs(2) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} acknowledgement timed out, elapsed_ns={}\n", + opcode, start.elapsed().as_nanos()); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + } +} +impl Drop for Runtime { + fn drop(&mut self) { + let chip=self.inner.device.soc().chip_id; + let _=self.inner.config.performance_request(chip,false); + self.inner.state.health.mark_failed(); + // Unpublished packets never reach the firmware; a published batch's packets stay + // retained below if the GPU may still use them, but their fences must not hang. + self.inner.fail_pending(ENODEV); + for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} + if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { + // SAFETY: a live module reference pins callbacks with retained DMA. + unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + return; + } + // SAFETY: no GPU command is pending and ASC is stopped. Field order + // drains callbacks before dropping retained commands, UAT and power. + unsafe { ManuallyDrop::drop(&mut self.inner) }; + } +} + +/// Same scaling contract as G16's qualified publication profiler. AP counter +/// rates may differ between SoCs; firmware command timestamps use 24 MHz. +pub(crate) fn physical_counter() -> u64 { + let tick: u64; + let frequency: u64; + // SAFETY: Read architectural counters only; no device state is modified. + unsafe { core::arch::asm!("mrs {f}, cntfrq_el0", "mrs {t}, cntpct_el0", + f = out(reg) frequency, t = out(reg) tick, + options(nomem, nostack, preserves_flags)) }; + let frequency = frequency & 0xffff_ffff; + if frequency == 0 { return 0; } + (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs new file mode 100644 index 0000000000000..9bd08d0257b0c --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. +use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; +use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use core::sync::atomic::{AtomicBool, Ordering}; +#[derive(Default)] +pub(crate) struct Completion; +#[vtable] +impl FenceOps for Completion { + fn get_driver_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("asahi")} + fn get_timeline_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("m3-native")} +} +pub(crate) struct Destination { mapping: Arc, offset: usize } +impl Destination { + fn resolve(objects: Pin<&xarray::XArray>>, + timestamp: agx_uapi::UapiTimestamp) -> Result> { + if timestamp.handle == 0 { return Ok(None); } + let guard = objects.lock(); + let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; + match &*object { + file::Object::TimestampBuffer(mapping) => { + let offset = timestamp.offset as usize; + if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { + return Err(EINVAL); + } + Ok(Some(Self { mapping: mapping.clone(), offset })) + } + } + } + fn firmware_address(&self) -> u64 { self.mapping.iova() + self.offset as u64 } +} + +#[derive(Copy,Clone)] +pub(crate) enum Command { + Compute(crate::agx_compute::Control), + Render{command:agx_uapi::UapiRenderCommand,usc:u64}, +} +pub(crate) struct Packet { + pub(crate) vm:mmu::Vm, + pub(crate) commands:KVec, + timestamps:KVec<[Option;4]>, + pub(crate) attachments:KVec, + pub(crate) wide_visibility:KVec, + completion:UserFence, + finish_claimed: AtomicBool, + /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. + pub(crate) urgency:u8, + /// Who submitted the packet, for the failure log only. + pub(crate) owner:Owner, + vm_job:Pin>>>, +} +/// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: +/// a renamed task leaves a stale name, which is harmless. +#[derive(Copy,Clone)] +pub(crate) struct Owner {pub(crate) pid:i32,process:[u8;16],thread:[u8;16]} +impl Owner { + fn current()->Self { + let task=kernel::current!(); + let leader=task.group_leader(); + Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), + thread:crate::m3_client::task_comm(task)} + } + fn name(comm:&[u8;16])->&str { + let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); + core::str::from_utf8(&comm[..len]).unwrap_or("?") + } + pub(crate) fn process(&self)->&str {Self::name(&self.process)} + pub(crate) fn thread(&self)->&str {Self::name(&self.thread)} +} +impl Packet { + pub(crate) fn profile_id(&self)->usize {self as *const Self as usize} + pub(crate) fn timestamp_addresses(&self,index:usize)->[u64;2] { + core::array::from_fn(|i|self.timestamps[index][i].as_ref().map_or(0,Destination::firmware_address)) + } + pub(crate) fn render_timestamp_addresses(&self,index:usize)->[[u64;2];2] { + core::array::from_fn(|stage| core::array::from_fn(|i| + self.timestamps[index][stage*2+i].as_ref().map_or(0,Destination::firmware_address))) + } + /// Called only after all published commands retired or before any publication. + pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} + pub(crate) fn finish(&self,result:Result) { + // Retirement, cancellation and a queued scheduler timeout can race. + // Claim error/signal ownership once; this flag is NOT proof that the + // fence is signaled or that hardware DMA has stopped. + if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } + if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + // A scheduler job/fence and the runtime's last packet can outlive + // hardware retirement. They must not keep the VM's active-job count + // nonzero: GEM closes otherwise accumulate across completed frames. + // On failure the runtime quarantines this guard with the packet until + // firmware has provably stopped; signalling an error is not retirement. + crate::agx_memory::publish(); + let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); + let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + self.completion.signal(); + if profile { + let after=crate::m3_runtime::physical_counter(); + let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; + pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", + self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); + } + } +} +pub(crate) struct Job {shared:Shared,packet:Arc} +/// Timeout state of a [`Job`], owned by the scheduler's timeout worker. +pub(crate) struct JobTimeout {shared:Shared,packet:Arc} +impl sched::JobImpl for Job { + type TimeoutData = JobTimeout; + fn timeout_data(&self)->JobTimeout { + JobTimeout {shared:self.shared.clone(),packet:self.packet.clone()} + } + fn run(job:&mut Self)->Result> { + // Queue the packet and return its fence; the completion worker publishes and retires + // it, so the scheduler can hand over further ready jobs meanwhile. + if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + pr_err!("M3 scheduler: submission failed {:?}\n",e); + // submit() failed before this packet could be published. + job.packet.release_guard();job.packet.finish(Err(e)); + } + Ok(Some(Fence::from_fence(&job.packet.completion))) + } + fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { + // Retain the publishing branch's explicit diagnostic override. + if !crate::m3_params::timeout_nohang() { + if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; + } + let guard=job.shared.lock(); + let Some(runtime)=guard.as_ref() else { + job.packet.finish(Err(ENODEV)); + return sched::Status::NoDevice; + }; + // The scheduler's deadline covers the whole packet, including time queued behind + // other batches. The completion worker owns each batch's bounded hardware-completion + // checks. Reinsert a healthy packet instead of turning a late timeout into a global + // device loss or corrupting an already signaled fence. + if runtime.health().healthy() { return sched::Status::NoHang; } + drop(guard); + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(EIO)); + sched::Status::NoDevice + } + fn cancel(job:&mut Self) { + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ECANCELED)); + } +} +struct AddressSpace<'a>(&'a mmu::Vm); +impl agx_uapi::GpuAddressSpace for AddressSpace<'_> { + fn covers(&self,address:u64,size:u64,access:agx_uapi::GpuAccess)->bool { + let Some(end)=address.checked_add(size) else {return false;}; + let (r,w)=match access {agx_uapi::GpuAccess::Read=>(true,false), + agx_uapi::GpuAccess::Write=>(false,true),agx_uapi::GpuAccess::ReadWrite=>(true,true)}; + address>=0x4000 && end<=(1u64<<42)-0x8000 + && !self.0.driver_range_overlaps(address..end) && self.0.covers_range(address,size,r,w) + } +} +pub(crate) struct Queue { + entity:sched::Entity,_scheduler:Arc>, + shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts, + /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`). + early_tiling:bool, + urgency:u8, +} +impl Queue { + pub(crate) fn new(shared:Shared,scheduler:Arc>,vm:mmu::Vm,priority:u32,usc:u64, + early_tiling:bool)->Result { + agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?; + // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME. + let urgency=3u8.saturating_sub(priority.min(3) as u8); + let priority=match priority {3=>sched::Priority::Low,2=>sched::Priority::Normal, + 1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)}; + Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler, + shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?, + early_tiling,urgency}) + } +} +impl queue::Queue for Queue { + fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { + let vm_job=self.vm.retain_m3_job()?; + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { + if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { + if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { + pr_err!("M3 render rejected {:?}: flags={:#x} size={}x{} layers={} samples={} utile={}x{} sample_size={} depth={:#x} stencil={:#x} zls={:#x} samplers={} heap={:#x}\n",e,payload.flags,payload.width,payload.height,payload.layers,payload.samples,payload.utile_width,payload.utile_height,payload.sample_size,payload.depth.base,payload.stencil.base,payload.zls_control,payload.sampler_count,payload.sampler_heap); + return Err(e); + } + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&vertex_attachments).map_err(|_|EINVAL)?; + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&fragment_attachments).map_err(|_|EINVAL)?; + commands.push(Command::Render{command:payload,usc:self.usc},GFP_KERNEL)?; + timestamps.push([Destination::resolve(objects,payload.vertex_timestamps.start)?, + Destination::resolve(objects,payload.vertex_timestamps.end)?, + Destination::resolve(objects,payload.fragment_timestamps.start)?, + Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; + flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + visibility.push(false,GFP_KERNEL)?; + continue; + } + let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);}; + // Other engines and helpers need their own M3 firmware translation. + let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32; + if payload.flags & !wide !=0 || payload.sampler_count!=0 || payload.sampler_heap!=0 { + pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle); + return Err(ENOTSUPP); + } + visibility.push(payload.flags & wide !=0,GFP_KERNEL)?; + // The common translator validates the original command/attachment + // addresses; this M3 scheduling hint changes none of them. + payload.flags &= !wide; + flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; + let c=agx_uapi::translate_compute_command(payload,attachments, + agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, + &AddressSpace(&self.vm)).map_err(|e| { + pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); + EINVAL + })?; + crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, + Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; + commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + } + parser.finish().map_err(|_|EINVAL)?; + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + completion:self.fences.new_fence(0,Completion)?.into(), + finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; + let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; + for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} + let mut job=job.arm();let finished=job.fences().finished();job.push(); + for mut sync in syncs {if let Some(chain)=sync.chain_fence.take() { + sync.syncobj.add_point(chain,&finished,sync.timeline_value); + } else {sync.syncobj.replace_fence(Some(&finished));}} + Ok(()) + } +} + +fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) + -> Result { + use crate::agx_attachments::{Attachments, CAPACITY}; + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if list.as_slice().len()>CAPACITY {return Err(E2BIG);} + let mut ranges = [(0, 0); CAPACITY]; + for (range, entry) in ranges.iter_mut().zip(list.as_slice()) { + *range = Attachments::range(entry.address, entry.size).map_err(|_| EINVAL)?; + if !AddressSpace(vm).covers(range.0, range.1, GpuAccess::Write) { + return Err(EINVAL); + } + } + let mut out=Attachments::new(&ranges[..list.as_slice().len()]).map_err(|_| EINVAL)?; + // G15S RTKit-2419 uses the legacy 0x17 cache attribute; retain the + // M4 range normalization and ownership checks, not its 0x18 attribute. + for i in 0..list.as_slice().len() {out.0[i*16+12..i*16+14].copy_from_slice(&0x17u16.to_le_bytes());} + Ok(out) +} + +fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpace<'_>, + early_tiling: bool) -> Result { + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} + let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; + // Early tiling (render flag bit 5) only where it is advertised. + let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) + || r.sampler_count != 0 || r.sampler_heap != 0 + || r.vertex_helper != empty || r.fragment_helper != empty + || r.stencil.compression_base != 0 || r.stencil.compression_stride != 0 + || r.ppp_control & !0x203 != 0 { return Err(ENOTSUPP); } + // Native 26A428 descriptors witness the ZLS metadata registers and mirrors + // for single-sample, single-layer depth. Keep other layouts unqualified. + if (r.depth.compression_base != 0 || r.stencil.compression_base != 0) + && (r.samples != 1 || r.layers != 1) {return Err(ENOTSUPP);} + // sample_size describes color tilebuffer storage. Depth/stencil-only + // passes legitimately require zero bytes of color storage. + if r.layers == 0 || r.layers > 2048 || r.width == 0 || r.height == 0 || r.width > 16384 || r.height > 16384 + || !matches!(r.utile_width,16|32) || !matches!(r.utile_height,16|32) + || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { + return Err(EINVAL); + } + crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) + || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { + return Err(EINVAL); + } + for (p,access) in [(r.depth_bias_base,GpuAccess::Read),(r.occlusion_query_base,GpuAccess::Write)] { + if p != 0 && (p & 7 != 0 || !space.covers(p,8,access)) { return Err(EINVAL); } + } + for zls in [r.depth,r.stencil] { agx_uapi::validate_zls(space,zls,r.layers).map_err(|_| EINVAL)?; } + let window = agx_uapi::QueueUscWindow { base:usc,user_start:0x4000,user_end:((1u64 << 42) - 2 * mmu::UAT_PGSZ as u64) }; + for p in [r.background,r.end_of_tile,r.partial_background,r.partial_end_of_tile] { + window.full_program(space,p.usc).map_err(|_| EINVAL)?; + if !space.covers(usc+u64::from(p.usc & !63),64,GpuAccess::Read) { return Err(EINVAL); } + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs new file mode 100644 index 0000000000000..59a72187c9699 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Thermal limit of the M3 runtime backend's GPU performance cap (`asahi.m3_thermal`). +//! +//! Without it (`off`, the default with the built-in InitData image) the firmware is given a +//! fixed highest performance state (5 of 8 with device-tree InitData: 1056 of 1380 MHz on +//! J516S) and nothing watches temperatures. The firmware's own die temperature controller is +//! off in the InitData this driver builds. With device-tree InitData the default is `on`; +//! `asahi.m3_thermal=off` restores the fixed cap exactly. +//! +//! With it, the InitData gives the firmware the whole table as its highest state (the +//! "ceiling"), and what bounds the firmware is the runtime cap: the Globals performance-state +//! cap and power-interface state targets, which the firmware keeps reading while it runs (a +//! host moves the cap with a single store, with no message). The runtime cap starts at the +//! fixed cap of `off` (the "safe cap") and never goes below it, so at worst the GPU runs as it +//! does without the limit. +//! +//! - `hold`: the runtime cap stays at the safe cap for the whole boot. The GPU then runs exactly +//! as with `off`, provided the firmware follows the runtime cap; the per-job check below +//! proves or disproves that before `on` is used. +//! - `on`: the runtime cap follows the hottest SoC die temperature, read by a work item every +//! 500 ms while the GPU is busy (every 2 s when idle) from the thermal zones in [`ZONES`] +//! that exist: the SMC's die sensors and the PMP's CPU cluster sensors (see [`ZONES`]): +//! * at or above `critical` (`hot` + 8 C): the safe cap at once; +//! * at or above `hot` (`asahi.m3_thermal_hot`, default 80 C): one state lower, at most every +//! 500 ms and once per reading; +//! * at or below `cool` (`hot` - 10 C): one state higher, at most every 2 s and once per +//! reading, up to the ceiling; +//! * no zone, a zone that cannot be read, a reading outside 10..=125 C, one older than 3 s +//! or one that has not changed at all for 60 s: the safe cap at once. +//! +//! The defaults leave a wide margin below the limits the boot firmware's device tree gives this +//! machine (a 94 C GPU die target and a 111 C SoC-hot temperature), because the sensors may +//! read cooler than the GPU's hottest spot (the PMP's are CPU cluster sensors), and this limit +//! reacts in steps of hundreds of milliseconds while the firmware changes state within the cap +//! much faster. +//! +//! The cap is changed only by the job thread, after a job has retired, right before the state +//! the firmware reports is checked. After a lowering the previous cap is tolerated for 1 s. A +//! reported state above the runtime cap in 20 consecutive checks spanning at least 3 s means +//! the firmware does not follow the runtime cap, so this limit cannot work: the GPU is then +//! marked failed, as for a state above the ceiling. +//! +//! The decisions are in `m3_thermal_policy`, which host tests drive; this module reads the +//! sensors, publishes the cap and logs. + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, + error::to_result, + new_spinlock, + prelude::*, + str::CStr, + sync::{Arc, SpinLock}, + time::{msecs_to_jiffies, Instant, Monotonic}, + workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}, +}; + +use crate::{ + m3_adt_config::PstatePolicy, + m3_params::{self, ThermalMode}, + m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, +}; + +/// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors +/// (registered by the SMC hwmon driver) and the PMP's CPU cluster sensors (registered by the +/// PMP report driver once the PMP runs). Zones that do not exist are skipped; a zone that +/// exists but cannot be read fails the reading. The M3 Pro boards (J514S and J516S) need +/// both zones: on a J516S the SMC die sensors alone stayed below 67 C while the PMP hotspot +/// exceeded 85 C and the cap stepped down. Without the PMP running, the cap stays safe. +/// Every T8122 board requires the SMC die zone; T8122 has no PMP hotspot zone. +const ZONES: [&CStr; 2] = [c_str!("macsmc_soc_die"), c_str!("apple_pmp_hotspot")]; +// Do not raise its cap on a partial reconstruction of that sensor set. +const J516_REQUIRED_ZONES: u32 = 3; +fn required_zones() -> u32 { + let Some(root) = kernel::of::root() else { return J516_REQUIRED_ZONES; }; + let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { + return J516_REQUIRED_ZONES; + }; + crate::m3_thermal_policy::required_zones(&compatible) +} + +/// Sensor polling period while the GPU is busy and while it is idle, in ms. +const POLL_BUSY_MS: u32 = 500; +const POLL_IDLE_MS: u32 = 2000; +/// The GPU counts as busy for this long after a job has retired, in ns. +const BUSY_WINDOW_NS: i64 = 5 * S; +/// Lines logging cap changes per boot, and the period of the summary line while busy. +const LOG_LINES: u32 = 256; +const SUMMARY_EVERY_NS: i64 = 30 * S; + +type Time = Instant; + +/// Nanoseconds from `base` to `t`. +fn ns(base: Time, t: Time) -> i64 { + (t - base).as_nanos() +} + +struct SensorState { + reading: Reading, + /// When the GPU last retired a job, in ns from the sensor's base. + busy: i64, +} + +/// Reads [`ZONES`] from a self-rearming delayed work item, so the job thread never waits for the +/// SMC. The work item holds a reference, so it stays valid after the governor is dropped; it +/// then stops at its next run. +#[pin_data] +struct Sensor { + required_zones: u32, + /// The origin of the sensor's and the governor's times. + base: Time, + stopped: AtomicBool, + #[pin] + state: SpinLock, + #[pin] + work: DelayedWork, +} + +impl_has_delayed_work! { + impl HasDelayedWork for Sensor { self.work } +} + +/// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. +fn read_zone(name: &CStr) -> Result { + let mut temp: i32 = 0; + // SAFETY: name is NUL-terminated and temp is a valid output pointer. The + // thermal core acquires a device reference under its registry lock and + // keeps it through sampling, so zone unregistration cannot free it here. + to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + Ok(temp) +} + +/// The hottest reading of the zones in [`ZONES`] that exist, and which ones were read. +fn read_zones(required: u32) -> Result<(i32, u32)> { + let mut hottest: Option = None; + let mut zones = 0; + for (i, name) in ZONES.iter().enumerate() { + match read_zone(name) { + Ok(temp) => { + hottest = Some(hottest.map_or(temp, |h| h.max(temp))); + zones |= 1 << i; + } + Err(e) if e == ENODEV => {} + Err(e) => return Err(e), + } + } + if zones & required != required { + return Err(ENODEV); + } + hottest.map(|t| (t, zones)).ok_or(ENODEV) +} + +impl WorkItem for Sensor { + type Pointer = Arc; + + fn run(this: Arc) { + if this.stopped.load(Ordering::Acquire) { + return; + } + let result = read_zones(this.required_zones); + let at = ns(this.base, Time::now()); + let reading = match result { + Ok((mc, zones)) => Reading::Temp { mc, at, zones }, + Err(e) => Reading::Failed { + errno: e.to_errno(), + at, + }, + }; + let busy = { + let mut state = this.state.lock(); + state.reading = reading; + at - state.busy < BUSY_WINDOW_NS + }; + let delay = if busy { POLL_BUSY_MS } else { POLL_IDLE_MS }; + let _ = workqueue::system().enqueue_delayed(this, msecs_to_jiffies(delay)); + } +} + +fn describe(source: Source) -> &'static str { + match source { + Source::Unknown | Source::NoReading => "no reading yet", + Source::Ok(_) => "reading", + Source::Failed(e) if e == ENODEV.to_errno() => "no SoC die temperature zone", + Source::Failed(_) => "a SoC die temperature zone cannot be read", + Source::Stale => "the reading is older than 3 s", + Source::Implausible => "the reading is outside 10..=125 C", + Source::Stuck => "the reading has not changed for 60 s", + } +} + +fn describe_why(why: Why) -> &'static str { + match why { + Why::Critical => "critical", + Why::Hot => "hot", + Why::Cool => "cool", + Why::NoTemperature(source) => describe(source), + } +} + +/// Degrees Celsius with one decimal, for the log. +struct Celsius(i32); + +impl kernel::fmt::Display for Celsius { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let sign = if self.0 < 0 { "-" } else { "" }; + let v = self.0.unsigned_abs(); + write!(f, "{}{}.{} C", sign, v / 1000, (v % 1000) / 100) + } +} + +/// The runtime cap and the thermal limit that moves it. +pub(crate) struct Governor { + mode: ThermalMode, + /// Without the limit (`off`): the cap, which never changes. + fixed: u32, + policy: Policy, + freqs: [u32; 16], + base: Time, + sensor: Option>, + source: Source, + last_summary: Option, + /// The highest state the firmware reported since the last summary. + peak: u32, + lines: u32, +} + +impl Governor { + /// The governor for `policy`. Starts the sensor work item unless the limit is off. + pub(crate) fn new(dev: &device::Device, pstates: &PstatePolicy) -> Result { + let (hot_c, clamped) = m3_params::thermal_hot_param(); + let base = Time::now(); + let mut governor = Governor { + mode: pstates.thermal, + fixed: pstates.max, + policy: Policy::new( + pstates.thermal == ThermalMode::On, + pstates.safe, + pstates.max, + hot_c as i32 * 1000, + ), + freqs: pstates.freqs, + base, + sensor: None, + source: Source::Unknown, + last_summary: None, + peak: 0, + lines: 0, + }; + if governor.mode == ThermalMode::Off { + return Ok(governor); + } + if clamped { + dev_info!( + dev, + "M3: asahi.m3_thermal_hot clamped to {} C ({}..={} C)\n", + hot_c, + m3_params::THERMAL_HOT_MIN, + m3_params::THERMAL_HOT_MAX + ); + } + let sensor = Arc::pin_init( + pin_init!(Sensor { + required_zones: required_zones(), + base, + stopped: AtomicBool::new(false), + state <- new_spinlock!(SensorState { + reading: Reading::None, + busy: 0, + }), + work <- new_delayed_work!("asahi::M3ThermalSensor"), + }), + GFP_KERNEL, + )?; + let _ = workqueue::system().enqueue_delayed(sensor.clone(), msecs_to_jiffies(POLL_BUSY_MS)); + governor.sensor = Some(sensor); + let p = &governor.policy; + let (cool, hot, critical) = p.thresholds(); + match governor.mode { + ThermalMode::Hold => dev_info!( + dev, + "M3: thermal limit holding (asahi.m3_thermal=hold): the firmware may use states 1..={} ({} MHz) and the runtime cap stays at {} ({} MHz) for this boot; the SoC die temperature is logged only\n", + p.ceiling(), + governor.mhz(p.ceiling()), + p.safe(), + governor.mhz(p.safe()) + ), + _ => dev_info!( + dev, + "M3: thermal limit on (asahi.m3_thermal=on): runtime cap {}..={} ({}..={} MHz) from the hottest SoC die temperature of zones macsmc_soc_die and apple_pmp_hotspot: up while at or below {}, down at or above {}, cap {} at or above {} or without a fresh reading\n", + p.safe(), + p.ceiling(), + governor.mhz(p.safe()), + governor.mhz(p.ceiling()), + Celsius(cool), + Celsius(hot), + p.safe(), + Celsius(critical) + ), + } + Ok(governor) + } + + fn mhz(&self, state: u32) -> u32 { + self.freqs.get(state as usize).copied().unwrap_or(0) + } + + fn cap(&self) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.cap(), + } + } + + fn note_source(&mut self, dev: &device::Device, source: Source, temp: Option) { + if source == self.source { + return; + } + self.source = source; + let safe = self.policy.safe(); + match (source, temp) { + (Source::Ok(zones), Some(t)) => { + let smc = if zones & 1 != 0 { + " macsmc_soc_die (SMC die sensors)" + } else { + "" + }; + let pmp = if zones & 2 != 0 { + " apple_pmp_hotspot (PMP CPU cluster sensors)" + } else { + "" + }; + dev_info!( + dev, + "M3: thermal: SoC die temperature {} from zones:{}{}\n", + Celsius(t), + smc, + pmp + ) + } + (Source::Failed(e), _) => dev_info!( + dev, + "M3: thermal: {} (error {}); runtime cap at most {} ({} MHz)\n", + describe(source), + e, + safe, + self.mhz(safe) + ), + _ => dev_info!( + dev, + "M3: thermal: {}; runtime cap at most {} ({} MHz)\n", + describe(source), + safe, + self.mhz(safe) + ), + } + } + + fn summary(&mut self, dev: &device::Device, now: i64, temp: Option) { + if self + .last_summary + .is_some_and(|t| now - t < SUMMARY_EVERY_NS) + { + return; + } + self.last_summary = Some(now); + let peak = core::mem::take(&mut self.peak); + let cap = self.policy.cap(); + let (raised, lowered) = self.policy.counts(); + let (safe, ceiling) = (self.policy.safe(), self.policy.ceiling()); + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: SoC die {}, runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + Celsius(t), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + None => dev_info!( + dev, + "M3: thermal: no SoC die temperature ({}), runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + describe(self.source), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + } + } + + fn log_change(&mut self, dev: &device::Device, change: Change, temp: Option) { + if self.lines < LOG_LINES { + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): SoC die {} ({})\n", + change.from, + change.to, + self.mhz(change.to), + Celsius(t), + describe_why(change.why) + ), + None => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): {}\n", + change.from, + change.to, + self.mhz(change.to), + describe_why(change.why) + ), + } + } else if self.lines == LOG_LINES { + dev_info!( + dev, + "M3: thermal: further runtime cap changes are not logged\n" + ); + } + self.lines = self.lines.saturating_add(1); + } + + /// Recompute the runtime cap after a job. Returns the new cap if it changed; the caller + /// publishes it. + pub(crate) fn update(&mut self, dev: &device::Device, now: Time) -> Option { + let Some(sensor) = &self.sensor else { + return None; + }; + let reading = sensor.state.lock().reading; + let now = ns(self.base, now); + let (usable, change) = self.policy.update(now, reading); + let temp = usable.ok().map(|(t, _, _)| t); + match usable { + Err(source) => self.note_source(dev, source, None), + Ok((t, _, zones)) => self.note_source(dev, Source::Ok(zones), Some(t)), + } + self.summary(dev, now, temp); + let change = change?; + self.log_change(dev, change, temp); + Some(change.to) + } + + /// The highest state the firmware may report now: the runtime cap, or the highest cap of + /// the grace period after a lowering. + pub(crate) fn allowed(&self, now: Time) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.allowed(ns(self.base, now)), + } + } + + /// Account one check of the reported state; `over` says whether it stayed above + /// [`Self::allowed`], `peak` is the highest state it reported. Fails with ERANGE when the + /// firmware does not follow the runtime cap. + pub(crate) fn note_check( + &mut self, + dev: &device::Device, + now: Time, + over: bool, + peak: u32, + ) -> Result { + self.peak = self.peak.max(peak); + if let Some((count, span)) = self.policy.note_check(ns(self.base, now), over) { + let cap = self.cap(); + dev_err!( + dev, + "M3: thermal: the firmware reported a state above the runtime cap {} ({} MHz) in {} consecutive checks over {} ms: it does not follow the runtime cap; marking the GPU failed\n", + cap, + self.mhz(cap), + count, + span / 1_000_000 + ); + return Err(ERANGE); + } + Ok(()) + } + + /// Note that the GPU retired a job, so the sensor is read at the busy rate. + pub(crate) fn busy(&self, now: Time) { + if let Some(sensor) = &self.sensor { + sensor.state.lock().busy = ns(self.base, now); + } + } +} + +impl Drop for Governor { + fn drop(&mut self) { + if let Some(sensor) = &self.sensor { + sensor.stopped.store(true, Ordering::Release); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs new file mode 100644 index 0000000000000..f9c3ff08006d5 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The parameters of the T8122 start experiment (`t8122_start`): their text, defaults, accepted +//! values and their meaning. Plain data only, so that the host tests run it (tools/asahi). + +/// A parameter that is not given. +pub(crate) const UNSET: u64 = u64::MAX; +/// A parameter whose text is not one of its accepted values. The experiment refuses to start +/// with it rather than fall back to a default. +pub(crate) const INVALID: u64 = u64::MAX - 1; + +/// The Fender window sizes: the ADT sgx reg[1] size less 0x28000 (the difference on T6030), and +/// the whole ADT sgx reg[1]. +pub(crate) const FENDER_RULE: u32 = 0x10_4000; +pub(crate) const FENDER_ADT: u32 = 0x12_c000; +/// The GPU clock-generator offsets in SGX: the T8122 table's (`T8122_IO_MAPPINGS`, the default), +/// and the T6030 one. +pub(crate) const CLOCK_GEN_E1C: u64 = 0xe1_c000; +pub(crate) const CLOCK_GEN_E5C: u64 = 0xe5_c000; +/// The SGX setup write of T6030 and T8122 (offset, value). +pub(crate) const T6030_SGX_SETUP: (usize, u32) = (0xd1_4000, 0x7_0001); +/// The default unit masks: the T8122 table's (`m3_soc::T8122_HWDATA_B`), which are T6030's. +pub(crate) const UNIT_MASK_A: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B: u32 = 0x7; +/// No unit-mask bit outside these is accepted: the default masks. +pub(crate) const UNIT_MASK_A_LIMIT: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B_LIMIT: u64 = 0x7; +/// The default performance-state ceiling: the lowest two states. +pub(crate) const PSTATE_CAP: u32 = 2; +/// The highest performance-state ceiling the experiment accepts. The same as the driver's own +/// limit when the fast-die temperature controller is off (`m3_adt_config::ADT_MAX_PSTATE_LIMIT`): +/// J613/J615 have no SoC die temperature zone yet, so there is no thermal feedback above it. +pub(crate) const PSTATE_CAP_MAX: u32 = 5; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. A leading sign is not accepted. +fn number(text: &str) -> Option { + if text.starts_with(['+', '-']) { + return None; + } + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) if !hex.starts_with(['+', '-']) => u64::from_str_radix(hex, 16).ok(), + Some(_) => None, + None => text.parse::().ok(), + } +} + +/// Parse a parameter's text: one of `names`, else a number. Anything else, and the two +/// reserved values, give [`INVALID`]. +fn parse(text: &str, names: &[(&str, u64)]) -> u64 { + let text = text.trim(); + if let Some(&(_, value)) = names.iter().find(|(name, _)| *name == text) { + return value; + } + number(text).filter(|v| *v < INVALID).unwrap_or(INVALID) +} + +pub(crate) fn parse_number(text: &str) -> u64 { + parse(text, &[]) +} + +pub(crate) fn parse_fender(text: &str) -> u64 { + parse(text, &[("rule", FENDER_RULE as u64), ("adt", FENDER_ADT as u64)]) +} + +pub(crate) fn parse_clkgen(text: &str) -> u64 { + parse(text, &[("e1c", CLOCK_GEN_E1C), ("e5c", CLOCK_GEN_E5C), ("none", 0)]) +} + +pub(crate) fn parse_sgx_setup(text: &str) -> u64 { + parse(text, &[("none", 0), ("t6030", 1)]) +} + +/// The parameters as given: [`UNSET`], [`INVALID`] or a value. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Raw { + pub(crate) start: u64, + pub(crate) initdata_version: u64, + pub(crate) fender: u64, + pub(crate) clkgen: u64, + pub(crate) sgx_setup: u64, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u64, + pub(crate) pstate_cap: u64, +} + +impl Raw { + /// No parameter given. + #[cfg(test)] + pub(crate) const NONE: Raw = Raw { + start: UNSET, + initdata_version: UNSET, + fender: UNSET, + clkgen: UNSET, + sgx_setup: UNSET, + unit_mask_a: UNSET, + unit_mask_b: UNSET, + pstate_cap: UNSET, + }; + + /// Whether any parameter other than `asahi.t8122_start` was given. + pub(crate) fn values_given(&self) -> bool { + [ + self.initdata_version, + self.fender, + self.clkgen, + self.sgx_setup, + self.unit_mask_a, + self.unit_mask_b, + self.pstate_cap, + ] + .iter() + .any(|v| *v != UNSET) + } +} + +/// What `asahi.t8122_start` asks for. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Start { + /// Not given, or 0. + Off, + /// 1. + On, + /// Anything else. + Invalid, +} + +pub(crate) fn start(raw: &Raw) -> Start { + match raw.start { + UNSET | 0 => Start::Off, + 1 => Start::On, + _ => Start::Invalid, + } +} + +/// The GPU clock-generator IO mapping. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ClockGen { + /// SGX + the offset, mapped read-only for the firmware. + At(u64), + /// No mapping: HwDataB slot 29 stays zero. + Absent, +} + +/// The values of one boot. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Values { + pub(crate) initdata_version: u64, + pub(crate) fender: u32, + pub(crate) clock_gen: ClockGen, + pub(crate) sgx_setup: Option<(usize, u32)>, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) pstate_cap: u32, +} + +/// A parameter given with a value it does not accept: its name and what it accepts. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Refusal { + pub(crate) name: &'static str, + pub(crate) accepts: &'static str, +} + +/// A parameter's value, its default when not given; None when invalid. +fn given(param: u64, default: u64) -> Option { + match param { + UNSET => Some(default), + INVALID => None, + v => Some(v), + } +} + +/// Resolve the parameters. The first parameter with a value it does not accept is the refusal. +pub(crate) fn resolve(raw: &Raw) -> Result { + let refuse = |name, accepts| Refusal { name, accepts }; + let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + .filter(|v| *v != 0) + .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; + let fender = match given(raw.fender, FENDER_RULE as u64) { + Some(v) if v == FENDER_RULE as u64 || v == FENDER_ADT as u64 => v as u32, + _ => return Err(refuse("t8122_fender", "0x104000 or rule, 0x12c000 or adt")), + }; + let clock_gen = match given(raw.clkgen, CLOCK_GEN_E1C) { + Some(0) => ClockGen::Absent, + Some(v) if v == CLOCK_GEN_E1C || v == CLOCK_GEN_E5C => ClockGen::At(v), + _ => return Err(refuse("t8122_clkgen", "e1c, e5c or none")), + }; + let sgx_setup = match given(raw.sgx_setup, 1) { + Some(0) => None, + Some(1) => Some(T6030_SGX_SETUP), + _ => return Err(refuse("t8122_sgx_setup", "none or t6030")), + }; + let unit_mask_a = given(raw.unit_mask_a, UNIT_MASK_A) + .filter(|v| *v != 0 && *v & !UNIT_MASK_A_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_a", "nonzero, no bits outside 0x700000003"))?; + let unit_mask_b = given(raw.unit_mask_b, UNIT_MASK_B as u64) + .filter(|v| *v != 0 && *v & !UNIT_MASK_B_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_b", "nonzero, no bits outside 0x7"))? as u32; + let pstate_cap = given(raw.pstate_cap, PSTATE_CAP as u64) + .filter(|v| (1..=u64::from(PSTATE_CAP_MAX)).contains(v)) + .ok_or(refuse("t8122_pstate_cap", "1 to 5"))? as u32; + Ok(Values { + initdata_version, + fender, + clock_gen, + sgx_setup, + unit_mask_a, + unit_mask_b, + pstate_cap, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with(edit: impl Fn(&mut Raw)) -> Raw { + let mut raw = Raw { start: 1, ..Raw::NONE }; + edit(&mut raw); + raw + } + + #[test] + fn nothing_given_is_off_and_gives_no_values() { + assert_eq!(start(&Raw::NONE), Start::Off); + assert!(!Raw::NONE.values_given()); + assert_eq!(start(&with(|r| r.start = 0)), Start::Off); + assert_eq!(start(&with(|_| {})), Start::On); + assert_eq!(start(&with(|r| r.start = 2)), Start::Invalid); + assert_eq!(start(&with(|r| r.start = INVALID)), Start::Invalid); + assert!(!with(|_| {}).values_given()); + assert!(with(|r| r.pstate_cap = 3).values_given()); + assert!(with(|r| r.fender = INVALID).values_given()); + } + + #[test] + fn defaults_are_the_plan_baseline() { + let v = resolve(&with(|_| {})).unwrap(); + assert_eq!(v.initdata_version, 0x0c08_e21e_8380_0490); + assert_eq!(v.fender, 0x10_4000); + assert_eq!(v.clock_gen, ClockGen::At(0xe1_c000)); + assert_eq!(v.sgx_setup, Some((0xd1_4000, 0x7_0001))); + assert_eq!((v.unit_mask_a, v.unit_mask_b), (0x7_0000_0003, 7)); + assert_eq!(v.pstate_cap, 2); + } + + #[test] + fn every_alternative_resolves() { + let v = resolve(&with(|r| { + r.initdata_version = parse_number("0x123456789abcdef0"); + r.fender = parse_fender("adt"); + r.clkgen = parse_clkgen("e5c"); + r.sgx_setup = parse_sgx_setup("none"); + r.unit_mask_a = parse_number("1"); + r.unit_mask_b = parse_number("0x1"); + r.pstate_cap = parse_number("5"); + })) + .unwrap(); + assert_eq!(v.initdata_version, 0x1234_5678_9abc_def0); + assert_eq!(v.fender, 0x12_c000); + assert_eq!(v.clock_gen, ClockGen::At(0xe5_c000)); + assert_eq!(v.sgx_setup, None); + assert_eq!((v.unit_mask_a, v.unit_mask_b, v.pstate_cap), (1, 1, 5)); + let v = resolve(&with(|r| r.clkgen = parse_clkgen("none"))).unwrap(); + assert_eq!(v.clock_gen, ClockGen::Absent); + // Numbers and names give the same values. + assert_eq!(parse_fender("0x104000"), parse_fender("rule")); + assert_eq!(parse_fender(" 0x12C000 "), parse_fender("adt")); + assert_eq!(parse_clkgen("0xe5c000"), parse_clkgen("e5c")); + assert_eq!(parse_clkgen("0xe1c000"), parse_clkgen("e1c")); + assert_eq!(parse_clkgen("0"), parse_clkgen("none")); + assert_eq!(parse_sgx_setup("1"), parse_sgx_setup("t6030")); + } + + #[test] + fn unaccepted_values_refuse_and_name_the_parameter() { + let cases: [(fn(&mut Raw), &str); 15] = [ + (|r| r.initdata_version = parse_number("g15s"), "t8122_initdata_version"), + (|r| r.initdata_version = 0, "t8122_initdata_version"), + (|r| r.fender = parse_fender("0x144000"), "t8122_fender"), + (|r| r.fender = parse_fender("big"), "t8122_fender"), + (|r| r.clkgen = parse_clkgen("0xe00000"), "t8122_clkgen"), + (|r| r.clkgen = parse_clkgen("e5"), "t8122_clkgen"), + (|r| r.sgx_setup = parse_sgx_setup("2"), "t8122_sgx_setup"), + (|r| r.unit_mask_a = parse_number("0x700000007"), "t8122_unit_mask_a"), + (|r| r.unit_mask_a = 0, "t8122_unit_mask_a"), + (|r| r.unit_mask_b = parse_number("0xf"), "t8122_unit_mask_b"), + (|r| r.unit_mask_b = 0, "t8122_unit_mask_b"), + (|r| r.pstate_cap = 0, "t8122_pstate_cap"), + (|r| r.pstate_cap = 6, "t8122_pstate_cap"), + (|r| r.pstate_cap = 16, "t8122_pstate_cap"), + (|r| r.pstate_cap = parse_number("-1"), "t8122_pstate_cap"), + ]; + for (edit, name) in cases { + assert_eq!(resolve(&with(edit)).unwrap_err().name, name); + } + } + + #[test] + fn pstate_cap_accepts_one_to_five_only() { + for cap in 1..=PSTATE_CAP_MAX { + assert_eq!(resolve(&with(|r| r.pstate_cap = u64::from(cap))).unwrap().pstate_cap, cap); + } + for cap in [0u64, 6, 8, 15, 16] { + assert!(resolve(&with(|r| r.pstate_cap = cap)).is_err(), "cap {cap}"); + } + assert_eq!(PSTATE_CAP_MAX, 5); + } + + #[test] + fn a_leading_sign_is_rejected() { + for text in ["+1", "-1", "0x+1", "0x-1", "+0x10"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert!(resolve(&with(|r| r.pstate_cap = parse_number("+3"))).is_err()); + } + + #[test] + fn reserved_numbers_and_garbage_parse_as_invalid() { + for text in ["", "0x", "x1", "18446744073709551615", "18446744073709551614", "0x1g"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert_eq!(parse_number("18446744073709551613"), INVALID - 1); + assert_eq!(parse_number(" 0X10 "), 16); + } + + #[test] + fn default_masks_are_within_the_limits() { + assert_eq!(UNIT_MASK_A & !UNIT_MASK_A_LIMIT, 0); + assert_eq!(u64::from(UNIT_MASK_B) & !UNIT_MASK_B_LIMIT, 0); + assert_eq!(UNIT_MASK_A_LIMIT, 0x7_0000_0003); + assert_eq!(UNIT_MASK_B_LIMIT, 0x7); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs new file mode 100644 index 0000000000000..cf0ee658e8da5 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The T8122 (M3, G15G) start, `asahi.t8122_start=1`. +//! +//! The T8122 table (`m3_soc::T8122`) is complete, but the runtime has not passed on an M3 +//! MacBook Air yet, so the SoC is refused unless `asahi.t8122_start=1` arms the start. On a T8122 +//! whose GPU the boot loader handed over (resource admission passed), an armed start uses the +//! table's values, with a performance-state cap. Each value can be overridden for a lab A/B by +//! rebooting with a parameter: +//! +//! | Parameter | Default | Others | +//! |---|---|---| +//! | `asahi.t8122_initdata_version` | `0x0c08e21e83800490`, the G15 14.8.3 version | any | +//! | `asahi.t8122_fender` | `0x104000` (`rule`) | `0x12c000` (`adt`) | +//! | `asahi.t8122_clkgen` | `e1c`: SGX+0xe1c000, read-only | `e5c`: SGX+0xe5c000, read-only; `none` | +//! | `asahi.t8122_sgx_setup` | `t6030`: SGX+0xd14000 = 0x70001 | `none` | +//! | `asahi.t8122_unit_mask_a` | `0x700000003` | nonzero, within `0x700000003` | +//! | `asahi.t8122_unit_mask_b` | `0x7` | nonzero, within `0x7` | +//! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 | +//! +//! The defaults are the T8122 table's values. A parameter given with a value it does not accept +//! refuses the start. Without `asahi.t8122_start=1` nothing here runs, and T8122 is refused. +//! Nothing here applies to any other SoC. + +use core::mem::offset_of; + +use kernel::{device, prelude::*}; + +use crate::{ + fw::initdata::raw, + initdata::G15RuntimeHwDataB, + m3_adt_config::T8122_IO_MAPPINGS, + m3_init_storage::{self as storage, IoMap}, + m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + pgtable::{prot, Prot}, + t8122_knobs::{ + self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, + FENDER_RULE, + }, +}; + +/// The GPU register window (SGX) of T8122. +const SGX: u64 = 0x2_9000_0000; +/// Index of the Fender window and of the GPU clock generator in [`T8122_IO_MAPPINGS`]. +const FENDER: usize = 0; +const CLOCK_GEN: usize = 11; +/// The HwDataB slot of the GPU clock generator. +const CLOCK_GEN_SLOT: usize = 29; + + +/// The firmware IO mappings with the Fender window `fender` and the clock generator at +/// SGX + `clock_gen`. +const fn mappings(fender: u32, clock_gen: u64) -> [IoMapping; 12] { + let mut m = T8122_IO_MAPPINGS; + m[FENDER].2 = fender; + m[FENDER].3 = fender; + m[CLOCK_GEN].1 = SGX + clock_gen; + m +} + +/// The runtime IO maps, laid out once for the larger Fender window so that no slot's firmware +/// VA depends on the parameters. +const LAYOUT: [IoMap; 12] = + storage::pack_iomaps(&mappings(FENDER_ADT, CLOCK_GEN_E1C), storage::IOMAP_BASE); + +const fn iomaps(fender: u32, clock_gen: u64) -> [IoMap; 12] { + let mut m = LAYOUT; + m[FENDER].size = fender as usize; + m[CLOCK_GEN].physical = SGX + clock_gen; + m +} + +/// Every Fender window and clock-generator address, in [`Experiment::variant`] order. +static IO_MAPPINGS: [[IoMapping; 12]; 4] = [ + mappings(FENDER_RULE, CLOCK_GEN_E5C), + mappings(FENDER_RULE, CLOCK_GEN_E1C), + mappings(FENDER_ADT, CLOCK_GEN_E5C), + mappings(FENDER_ADT, CLOCK_GEN_E1C), +]; +static IOMAPS: [[IoMap; 12]; 4] = [ + iomaps(FENDER_RULE, CLOCK_GEN_E5C), + iomaps(FENDER_RULE, CLOCK_GEN_E1C), + iomaps(FENDER_ADT, CLOCK_GEN_E5C), + iomaps(FENDER_ADT, CLOCK_GEN_E1C), +]; + +/// Whether each IO map holds its IO mapping: same slot, the mapping's 16 KiB-aligned physical +/// base, its subpage offset, and room for its total size. +const fn maps_cover(mappings: &[IoMapping; 12], maps: &[IoMap; 12]) -> bool { + let mut i = 0; + while i < 12 { + let (slot, phys, total, _, _) = mappings[i]; + let io = maps[i]; + if io.slot != slot + || io.physical + io.offset as u64 != phys + || io.offset + total as usize > io.size + || io.size % 0x4000 != 0 + { + return false; + } + i += 1; + } + true +} + +const _: () = { + // The default variant's IO mappings are the T8122 table's. + let defaults = mappings(FENDER_RULE, CLOCK_GEN_E1C); + let mut j = 0; + while j < 12 { + let (a, b) = (defaults[j], T8122_IO_MAPPINGS[j]); + assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2 && a.3 == b.3 && a.4 == b.4); + j += 1; + } + // The T8122 table: the Fender window at the rule size, the clock generator last, at + // SGX+0xe1c000, 16 KiB, read-only. + assert!(T8122_IO_MAPPINGS[FENDER].0 == 0 && T8122_IO_MAPPINGS[FENDER].1 == SGX + 0xd0_0000); + assert!(T8122_IO_MAPPINGS[FENDER].2 == FENDER_RULE); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].0 == CLOCK_GEN_SLOT); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].1 == SGX + CLOCK_GEN_E1C); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].2 == 0x4000 && !T8122_IO_MAPPINGS[CLOCK_GEN].4); + assert!(CLOCK_GEN + 1 == T8122_IO_MAPPINGS.len()); + // Both windows are whole pages and hold the Fender scratch area (+0x60000..+0x80000). + assert!(FENDER_RULE % 0x4000 == 0 && FENDER_ADT % 0x4000 == 0 && FENDER_RULE >= 0x8_0000); + assert!(FENDER_RULE < FENDER_ADT); + // Both clock-generator blocks are whole pages inside the first 16 MiB of SGX. + assert!(CLOCK_GEN_E5C % 0x4000 == 0 && CLOCK_GEN_E1C % 0x4000 == 0); + assert!(CLOCK_GEN_E5C + 0x4000 <= 0x100_0000 && CLOCK_GEN_E1C + 0x4000 <= 0x100_0000); + let mut i = 0; + while i < 4 { + assert!(maps_cover(&IO_MAPPINGS[i], &IOMAPS[i])); + i += 1; + } + // The default unit masks are the T8122 table's, and the limits allow them. + assert!(T8122_HWDATA_B.unit_mask_a == knobs::UNIT_MASK_A); + assert!(T8122_HWDATA_B.unit_mask_b == knobs::UNIT_MASK_B); + assert!(knobs::UNIT_MASK_A & !knobs::UNIT_MASK_A_LIMIT == 0); + assert!(knobs::UNIT_MASK_B as u64 & !knobs::UNIT_MASK_B_LIMIT == 0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); + // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the + // opt-in itself; if another is added, this fails the build until the start accounts for it. + assert!(crate::m3_soc::T8122.unported.len() == 1); +}; + +/// The values of one armed boot. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Experiment(Values); + +/// Whether `soc` is the T8122 table. +pub(crate) fn is_t8122(soc: &Soc) -> bool { + core::ptr::eq(soc, &crate::m3_soc::T8122) +} + +impl Experiment { + /// Index of this boot's IO-mapping table in [`IO_MAPPINGS`] and [`IOMAPS`]. + fn variant(&self) -> usize { + let adt = usize::from(self.0.fender == FENDER_ADT); + let e1c = usize::from(self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C)); + 2 * adt + e1c + } + + /// How many IO mappings this boot gives the firmware: all, or all but the clock generator. + fn mapping_count(&self) -> usize { + match self.0.clock_gen { + ClockGen::At(_) => CLOCK_GEN + 1, + ClockGen::Absent => CLOCK_GEN, + } + } + + /// The InitData version given to the firmware (`asahi.t8122_initdata_version`). + pub(crate) fn initdata_version(&self) -> u64 { + self.0.initdata_version + } + + /// The firmware IO mappings of the InitData. + pub(crate) fn io_mappings(&self) -> &'static [IoMapping] { + &IO_MAPPINGS[self.variant()][..self.mapping_count()] + } + + /// The runtime's mapping of each of them. With the default Fender window and clock generator + /// they are the T8122 table's own IO maps. + pub(crate) fn iomaps(&self) -> &'static [IoMap] { + if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { + return &crate::m3_adt_config::T8122_IOMAPS; + } + &IOMAPS[self.variant()][..self.mapping_count()] + } + + /// The register block the IO mapping of `slot` must lie in, given the HwConfig's block + /// `base` for it: the selected clock generator for slot 29. + pub(crate) fn io_block(&self, slot: usize, base: u64) -> u64 { + match self.0.clock_gen { + ClockGen::At(offset) if slot == CLOCK_GEN_SLOT => SGX + offset, + _ => base, + } + } + + /// The HwDataB slots the firmware maps read-only, as a bit mask: the clock generator's. + pub(crate) fn read_only_slots(&self) -> u32 { + match self.0.clock_gen { + ClockGen::At(_) => 1 << CLOCK_GEN_SLOT, + ClockGen::Absent => 0, + } + } + + /// The runtime HwDataB words: the T8122 table's. The unit masks are written over them + /// ([`Self::unit_masks`]). + pub(crate) fn hwdata_b(&self) -> &'static G15RuntimeHwDataB { + &T8122_HWDATA_B + } + + /// The HwDataB unit masks A (+0x17c0) and B (+0x17c8). + pub(crate) fn unit_masks(&self) -> (u64, u32) { + (self.0.unit_mask_a, self.0.unit_mask_b) + } + + /// The SGX write (offset, value) made before the firmware starts, if any. + pub(crate) fn sgx_setup(&self) -> Option<(usize, u32)> { + self.0.sgx_setup + } + + /// The highest performance state the firmware may use (`asahi.t8122_pstate_cap`). + pub(crate) fn pstate_cap(&self) -> u32 { + self.0.pstate_cap + } +} + +/// Firmware page protection of the IO map of `slot`, with the read-only slots `read_only`. +pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot { + if slot < 32 && read_only & (1 << slot) != 0 { + prot::PROT_FW_MMIO_RO + } else { + prot::PROT_FW_MMIO_RW + } +} + +/// Decide, after resource admission, whether `soc` starts with the experiment values. +/// +/// Any SoC but T8122: Ok(None), silently. T8122 without `asahi.t8122_start=1`: Ok(None), so the +/// caller refuses it as before (one extra line when experiment parameters were given). T8122 +/// with `asahi.t8122_start=1`: the experiment, after logging every value, or a refusal for a +/// parameter value it does not accept. +pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> { + if !is_t8122(soc) { + return Ok(None); + } + let raw = crate::m3_params::t8122_params(); + match knobs::start(&raw) { + Start::On => {} + Start::Off => { + if raw.values_given() { + dev_info!( + dev, + "M3 G15G start: asahi.t8122_* values given without asahi.t8122_start=1; ignored\n" + ); + } + return Ok(None); + } + Start::Invalid => { + dev_err!( + dev, + "M3 G15G start: refused: asahi.t8122_start has a value it does not accept (0 or 1); GPU startup disabled\n" + ); + return Err(ENODEV); + } + } + // Admission has required the boot loader's handoff: its firmware segments and reserved + // regions, and on this SoC its power configuration. + if !soc.power_from_boot_loader || soc.firmware.is_none() || soc.hwcfg.is_none() { + dev_err!(dev, "M3 G15G start: refused: the T8122 table is not the one this experiment expects\n"); + return Err(ENODEV); + } + let v = knobs::resolve(&raw).map_err(|refusal| { + dev_err!( + dev, + "M3 G15G start: refused: asahi.{} has a value it does not accept ({}); GPU startup disabled\n", + refusal.name, + refusal.accepts + ); + ENODEV + })?; + let e = Experiment(v); + dev_warn!( + dev, + "M3 G15G start: armed (asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", + v.initdata_version, + v.fender, + clkgen_label(v.clock_gen, v.fender), + match v.sgx_setup { + Some(_) => "t6030(sgx+0xd14000=0x70001)", + None => "none", + }, + v.unit_mask_a, + v.unit_mask_b, + v.pstate_cap + ); + dev_warn!( + dev, + "M3 G15G start: the T8122 table's InitData words, HwData placement and register lists; {} IO mappings at firmware VA {:#x}\n", + e.mapping_count(), + storage::IOMAP_BASE + ); + // J613/J615 have no SoC die temperature zone yet, so a cap above the no-feedback limit runs + // the fanless GPU without any temperature feedback (`m3_thermal` holds the cap but gets no + // reading). The runtime cap and the firmware ceiling stay equal, because the cap is at most + // the no-feedback limit. + if v.pstate_cap > knobs::PSTATE_CAP { + dev_warn!( + dev, + "M3 G15G start: pstate_cap={} is above {}: no SoC die temperature zone on this board, so states up to {} run with no temperature feedback\n", + v.pstate_cap, + knobs::PSTATE_CAP, + v.pstate_cap + ); + } + Ok(Some(e)) +} + +/// The armed line's clock-generator field. It says `ro` for the firmware's own slot-29 mapping, +/// and adds that the Fender window reaches the block read-write when the chosen window covers it +/// (SGX+0xe1c000 lies under the 0x12c000 window but not the 0x104000 one; SGX+0xe5c000 under +/// neither). +fn clkgen_label(clock_gen: ClockGen, fender: u32) -> &'static str { + let fender_end = 0xd0_0000u64 + u64::from(fender); + let reaches = |off: u64| off + 0x4000 <= fender_end; + match clock_gen { + ClockGen::At(CLOCK_GEN_E1C) if reaches(CLOCK_GEN_E1C) => { + "e1c(sgx+0xe1c000,ro; also rw in the fender window)" + } + ClockGen::At(CLOCK_GEN_E1C) => "e1c(sgx+0xe1c000,ro)", + ClockGen::At(_) if reaches(CLOCK_GEN_E5C) => { + "e5c(sgx+0xe5c000,ro; also rw in the fender window)" + } + ClockGen::At(_) => "e5c(sgx+0xe5c000,ro)", + ClockGen::Absent if reaches(CLOCK_GEN_E1C) => { + "none (but the fender window reaches sgx+0xe1c000 rw)" + } + ClockGen::Absent => "none", + } +} + +/// Log, in an armed start, that the probe stopped at `stage` before the firmware started. +pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, stage: &str, error: Error) { + if experiment.is_some() { + dev_err!( + dev, + "M3 G15G start: refused at {} ({:?}); the firmware was not started, see the lines above\n", + stage, + error + ); + } +} + +/// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. +pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { + if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + dev_err!( + dev, + "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" + ); + } +} + +// One `M3 G15G verdict:` line per outcome of an armed start, so that a tester can tell them +// apart from the log alone. They run on T8122 only (callers check `is_t8122`); nothing here +// changes what the runtime does. + +/// The GPU coprocessor's start or the driver's InitData upload failed (`m3_runtime::Runtime::new`). +/// `started`: the coprocessor ran and offered its endpoints. +pub(crate) fn prepare_verdict(dev: &device::Device, soc: &Soc, started: bool, error: Error) { + if !is_t8122(soc) { + return; + } + if started { + dev_err!( + dev, + "M3 G15G verdict: driver-refused ({:?}): the GPU firmware is up, but the driver did not publish the InitData (UAT, upload checks or thermal setup, see above)\n", + error + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-boot-failed ({:?}): the GPU coprocessor did not start, or its RTKit did not offer endpoints 0x20/0x21; no InitData was published\n", + error + ); + } +} + +/// The step the firmware boot sequence reached (`m3_runtime::Runtime::boot_inner`). The verdict +/// uses it to name a failure as a publish, a device-control or an InitData-acknowledgement +/// problem, rather than lumping them all under "initdata-rejected". +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum BootStep { + /// About to send, or sending, the InitData publish message (endpoint 0x20). + Publish, + /// A device-control message (0x13 or 9) before the firmware signalled ready. + DeviceControl, + /// Waiting (up to 2 s) for the firmware to write its ready words. + AwaitReady, + /// The firmware signalled ready; a control or check after that. + PostReady, +} + +/// The firmware's answer to the published InitData (`m3_runtime::Runtime::boot`). `accepted`: +/// the firmware wrote its ready words. `step`: the furthest step the boot reached. +pub(crate) fn boot_verdict( + dev: &device::Device, + version: u64, + step: BootStep, + accepted: bool, + crashed: bool, + result: Result, +) { + let Err(e) = result else { + dev_info!( + dev, + "M3 G15G verdict: firmware-running: the firmware accepted the InitData (version {:#x}) and the device controls; no job has run yet\n", + version + ); + return; + }; + // The firmware signalled ready, then a later control or check failed: the InitData was + // accepted, so this is not a rejection. + if accepted || step == BootStep::PostReady { + if crashed { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}) then crashed during a post-boot control or check; see the crash lines above\n", + e, + version + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}), then a post-boot control or check failed (see above)\n", + e, + version + ); + } + return; + } + match step { + BootStep::Publish => dev_err!( + dev, + "M3 G15G verdict: publish-failed ({:?}): the InitData publish message (endpoint 0x20) could not be sent; the firmware was not told about the InitData (version {:#x})\n", + e, + version + ), + BootStep::DeviceControl => dev_err!( + dev, + "M3 G15G verdict: device-control-failed ({:?}): a device-control queue, mailbox send or acknowledgement failed after the InitData (version {:#x}) was published; see the device-control diagnostic above\n", + e, + version + ), + // AwaitReady (and PostReady is handled above). + _ if crashed => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware crashed after the InitData (version {:#x}) was published; see the crash lines above\n", + e, + version + ), + _ => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware did not acknowledge the InitData (version {:#x}) within 2 s (its ready words stayed unset); see M3 firmware readiness above\n", + e, + version + ), + } +} + +/// A GPU coprocessor setup step failed before the firmware could be started +/// (`m3_runtime::Runtime::new`): the RTKit state, the mailbox transport or the DRM device. +pub(crate) fn coproc_setup_refused(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G start: refused at the GPU coprocessor setup (RTKit/mailbox) ({:?}); the firmware was not started\n", + error + ); +} + +/// The firmware ran a job above the performance-state cap (`m3_runtime`, the after-job pstate +/// check). The `M3: firmware ... stays above the cap ...` line precedes this. +pub(crate) fn cap_violated_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: cap-violated ({:?}): after a job the firmware reported a performance state above the cap; the GPU was marked failed (see the 'stays above the cap' line above)\n", + error + ); +} + +/// A job-path step failed outside the completion wait (`m3_runtime::Runtime::execute`): a submit, +/// a doorbell, or a post-completion read. No dispatch outcome was determined. +pub(crate) fn job_setup_failed_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: job-failed-before-wait ({:?}): a submit, doorbell or diagnostic step failed, so no dispatch outcome was determined; see the lines above\n", + error + ); +} + +/// A job that failed (`m3_runtime::Inner::fail`). `primary` is ETIMEDOUT when it did not retire +/// within the runtime's per-batch bound; `start_stamp` is the job's first GPU start (CDM +/// dispatch) timestamp, if readable; `pstate` is the GPU performance-state register, if readable. +pub(crate) fn job_failed_verdict( + dev: &device::Device, + primary: Error, + start_stamp: Option, + pstate: Option, + crashed: bool, +) { + if primary != ETIMEDOUT || crashed { + dev_err!( + dev, + "M3 G15G verdict: job-faulted ({:?}): the firmware crashed, reported an error, or the GPU reported a fault while the job ran; see the lines around this one\n", + primary + ); + return; + } + // A nonzero start timestamp means the firmware dispatched the job on the GPU, so this was not + // a dispatch gap however the power state reads now (the GPU may have idled off since). The + // stamp and pstate print as `Some(hex)` or `None`. + if start_stamp.is_some_and(|start| start != 0) { + dev_err!( + dev, + "M3 G15G verdict: job-ran-completion-missed: the job was dispatched (GPU start timestamp {:x?}) but its completion was not seen within the per-batch bound (pstate register {:x?})\n", + start_stamp, + pstate + ); + return; + } + // The GPU start timestamp is 0 (or unreadable): the job was not dispatched. + match pstate { + Some(p) if p & 0xf == 0 => dev_err!( + dev, + "M3 G15G verdict: job-accepted-never-dispatched: the job was not dispatched (GPU start timestamp {:x?}) and the GPU reads powered down (pstate register {:#x})\n", + start_stamp, + p + ), + Some(p) => dev_err!( + dev, + "M3 G15G verdict: job-timed-out-powered: the job did not retire with the GPU powered (pstate register {:#x}, GPU start timestamp {:x?}); see the engine snapshot below\n", + p, + start_stamp + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-timed-out: the job did not retire (pstate register and GPU start timestamp both unreadable)\n" + ), + } +} + +/// The first job of a kind (0 render, 1 compute) retired (`m3_runtime::Runtime::execute`): +/// `span` is its first start and last end GPU timestamp (24 MHz ticks), if readable. +pub(crate) fn job_completed_verdict(dev: &device::Device, kind: usize, gpu_ns: u64, span: Option<[u64; 2]>) { + let what = if kind == 0 { "render" } else { "compute" }; + match span { + Some([start, end]) if start != 0 && end != 0 => dev_info!( + dev, + "M3 G15G verdict: job-completed: the first {} job finished, GPU timestamps {:#x}..{:#x}, {} ns of GPU time\n", + what, + start, + end, + gpu_ns + ), + Some([start, end]) => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are {:#x}..{:#x}\n", + what, + start, + end + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are unreadable\n", + what + ), + } +} From 7660bed77d0b2c5fbf0422f6d12af56c228a385a Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:51 -0500 Subject: [PATCH 002/392] Prepare the original context for carried T8122 commit ad52c9b317f6 Keep the public dependency patch unchanged. The separate 7.3 adaptation restores Aurora integration after the carried series. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/Kconfig | 23 +- drivers/gpu/drm/asahi/debug.rs | 2 +- drivers/gpu/drm/asahi/driver.rs | 252 ++---- drivers/gpu/drm/asahi/file.rs | 273 +----- drivers/gpu/drm/asahi/float.rs | 9 +- drivers/gpu/drm/asahi/g15/mod.rs | 71 +- drivers/gpu/drm/asahi/g15/uat.rs | 8 +- drivers/gpu/drm/asahi/gpu.rs | 119 +-- drivers/gpu/drm/asahi/mmu.rs | 1348 ++++++------------------------ rust/kernel/soc/apple/rtkit.rs | 90 +- 10 files changed, 444 insertions(+), 1751 deletions(-) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..958600a91b06b 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Aurora (DRM support for Apple AGX GPUs)" + tristate "Asahi (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -21,27 +21,8 @@ config DRM_ASAHI select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM select RUST_APPLE_RTKIT - select CRYPTO_LIB_SHA256 - help - DRM driver for Apple AGX GPUs, from the M1 family onwards. - - To compile this driver as a module, choose M here: the module will - be called aurora. It keeps "asahi" as a module alias. - -config DRM_ASAHI_G16 - bool "Apple M4 (G16G) support (experimental)" - depends on DRM_ASAHI && ARM64 - depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI - depends on APPLE_CLPC=y || APPLE_CLPC=DRM_ASAHI || APPLE_CLPC=n - depends on THERMAL_OF help - Rust backend for the qualified J713/T8132 GPU firmware. - Load aurora explicitly after the native display and GFX TVM supplier - are ready. Firmware-visible storage remains retained until a verified - retirement or reboot. Keep the working kernel available during bring-up. - A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. - An aborted receipt disables that command class until reboot. - Suspend, hibernation and kexec are blocked while native graphics owns DMA. + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 479d9763cc7ed..50628ade5ab8e 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -84,7 +84,7 @@ pub(crate) enum DebugFlags { /// Update the cached global debug flags from the module parameter pub(crate) fn update_debug_flags() { - let flags = module_parameters::debug_flags.value(); + let flags = *module_parameters::debug_flags.value(); DEBUG_FLAGS.store(flags, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d6e461a7c3210..c092595180cf8 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,13 +45,10 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { - binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) g16: Option>, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, @@ -79,18 +76,12 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata -/// -/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h -/// and only changes if that ABI ever breaks. The minor version goes up with -/// each new ioctl, flag or feature bit, so userspace can tell what a kernel -/// accepts without probing. The patch level marks fixes userspace may need to -/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 1, + major: 0, minor: 0, patchlevel: 0, - name: c"aurora", - desc: c"Aurora GPU for Apple AGX", + name: c"asahi", + desc: c"Apple AGX Graphics", }; /// DRM Driver implementation for `AsahiDriver`. @@ -105,6 +96,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -144,82 +136,68 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on -// its entries, so the entries that the native G15 backend owns when it is -// built are passed in by configuration. -macro_rules! asahi_of_device_table { - ($($g15_owned:tt)*) => { - kernel::of_device_table!( - OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - $($g15_owned)* - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] - ); - }; -} - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -asahi_of_device_table!( - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), +// OF Device ID table. +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] ); -#[cfg(CONFIG_DRM_ASAHI_G15)] -asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -227,12 +205,14 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + fn unbind<'bound>( + _pdev: &'bound platform::Device>, + this: Pin<&Self::Data<'bound>>, + ) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } - this.close_binding(); } /// Device probe function. @@ -241,7 +221,6 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); - crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -251,7 +230,6 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; - // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -268,9 +246,6 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -302,20 +277,12 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); - let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - // The DRM core created this card's debugfs directory when the device was registered; - // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and - // waits for its readers, before the registration data that owns the snapshot is dropped. - if let Some(snapshot) = debug_stats.as_ref() { - crate::stats_raw::register(reg.device(), snapshot); - } - // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -323,16 +290,13 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && module_parameters::stats_export.value() != 0, + stats.is_some() && *module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev, raw_stats, stats, @@ -352,9 +316,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -364,12 +325,9 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -385,9 +343,6 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -403,73 +358,19 @@ impl AsahiPlatformDriver { return Err(error); } }; - let stats = gpu::Gpu::stats_snapshot(&*gpu); + // G17's firmware channel format has no supported stats decoder yet. let raw_dev = pdev.as_ref().as_raw(); - let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); - let raw_stats = match crate::sysfs_exports::register_g17( - raw_dev, - snapshot, - module_parameters::stats_export.value() != 0, - ) { + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; - if let Err(error) = - crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) - { - dev_warn!( - pdev.as_ref(), - "idle_off_delay_ms not registered: {:?}\n", - error - ); - } Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, - raw_dev, - raw_stats, - stats, - }) - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl AsahiPlatformDriver { - pub(crate) fn probe_g16<'bound>( - pdev: &'bound platform::Device>, - cfg: &'static crate::g16::Config, - ) -> Result> { - // SAFETY: the qualified UAT output width constrains this device's DMA. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - let resources = regs::Resources::new(pdev)?; - let res = regs::G16Resources::new(pdev)?; - let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; - let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiDrmRegistrationData { - gpu: data_gpu, - resources, - }); - // SAFETY: registration is stored in bound platform data and never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; - Ok(AsahiPlatformDriverData { - binding, - _device: reg.device().into(), - _reg: reg, - g17: None, - g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -477,16 +378,9 @@ impl AsahiPlatformDriver { } } -impl AsahiPlatformDriverData<'_> { - pub(crate) fn close_binding(&self) { - self.binding.close(); - } -} - impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - self.binding.close(); - // `self.raw_dev` was set in `probe` from the platform + // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 244dca39e41ac..608864949c976 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,6 @@ use crate::{ AsahiDrmDevice, // }, gem, - gpu, - memacct, mmu, queue, util::{ @@ -45,7 +43,6 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; -use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -71,7 +68,6 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, - gpu: Arc, } impl Drop for Vm { @@ -93,7 +89,6 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); - self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -127,23 +122,10 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, - pub(crate) cumulative: Option, - pub(crate) timeline: bool, } impl SyncItem { - fn parse_one( - file: &DrmFile, - mut data: uapi::drm_asahi_sync, - out: bool, - bundled: bool, - ) -> Result { - if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { - return Err(EINVAL); - } - data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; - } + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -164,8 +146,6 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, - cumulative: None, - timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -188,10 +168,12 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: None, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, timeline_value: data.timeline_value, - cumulative: None, - timeline: true, }) } _ => { @@ -206,52 +188,28 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, - allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; - let mut anchor = None; - // Read each record once so validation and use see the same retained bytes. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !allow_cumulative || i < in_count || anchor.is_some() { - return Err(EINVAL); - } - anchor = Some(i as usize); - } - vec.push( - Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, - GFP_KERNEL, - )?; - } - if let Some(anchor) = anchor { - let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], - anchor - in_count as usize, - )?; - vec[anchor].cumulative = Some(bundle); - } else { - for sync in &mut vec[in_count as usize..] { - if sync.timeline { - sync.chain_fence = Some(dma_fence::FenceChain::new()?); - } - } + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; } Ok(vec) @@ -260,7 +218,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -275,10 +233,6 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, - /// GPU memory this client created and holds. - mem: Arc, - /// Completed engine time, retained by queues after handle destruction. - usage: Arc, } /// Convenience type alias for our DRM `File` type. @@ -298,35 +252,12 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); - let mem = memacct::Client::new()?; - let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) - }) - } - - /// The objects this client created, which `memacct` attributes to it, not every object it - /// has a handle to: an imported or shared buffer is counted once, by its creator. - fn memory_stats(&self) -> Option { - Some(drm::file::MemoryStats { - private: self.mem.committed(), - resident: Some(self.mem.resident()), - ..Default::default() + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) }) } - fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { - let names = [ - kernel::c_str!("vertex"), - kernel::c_str!("fragment"), - kernel::c_str!("compute"), - ]; - for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { - emit(name, ns); - } - } - fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -336,41 +267,25 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new( - id: u64, - user_range: Range, - mem: Arc, - usage: Arc, - ) -> impl PinInit { - // SAFETY: The initializer supplies aligned, pinned storage for Self. - // XArray::new is infallible; each pinned array is initialized in place, - // then every remaining field is written before returning success. + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - pin_init::raw_try_init( - raw_vms, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; let raw_queues = addr_of_mut!((*slot).queues); - pin_init::raw_try_init( - raw_queues, - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ), - )?; + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; let raw_objects = addr_of_mut!((*slot).objects); - pin_init::raw_try_init( - raw_objects, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).mem).write(mem); - addr_of_mut!((*slot).usage).write(usage); Ok(()) }) } @@ -422,7 +337,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as usize), + UserPtr::from_addr(data.pointer as _), core::mem::size_of_val(&words), ) .writer() @@ -441,11 +356,8 @@ impl File { return Err(ENODEV); } - let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, - memory_limit: memory_limits.device, - client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -465,13 +377,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // Write the retained result to this user slice once. + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; Ok(0) @@ -483,18 +395,15 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as usize); + let user = UserPtr::from_addr(data.pointer as _); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size).writer().write(&[ - vm_id, - 0, - status.reported_error() as u32, - 0, - ])?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } @@ -505,7 +414,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -569,22 +478,7 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; - // G16 registers a retained backend owner before returning this VM. Any later - // allocation can fail before file::Vm exists to run its normal close path. - let failed_vm = ScopeGuard::new_with_data( - (vm.clone(), user_range.clone(), kernel_range.clone()), - |(vm, user_range, kernel_range)| { - if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { - vm.close_idle_reclaim(); - if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { - pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); - } - vm.bo_deferred_cleanup(); - gpu.vm_closed(&vm); - } - }, - ); + let vm = gpu.new_vm(kernel_range.clone())?; let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -660,12 +554,10 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, - gpu: gpu.clone(), }, GFP_KERNEL, )?)?; - drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -674,7 +566,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -683,6 +575,7 @@ impl File { Err(ENOENT) } else { drop(vm); + reg_data.gpu.vm_closed(); Ok(0) } } @@ -725,48 +618,11 @@ impl File { None }; - let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER - .admit(&file.inner().mem, size as u64) - .inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; - // Large G17 user objects are populated in full by the first VM_BIND. - // Do not multiply direct reclaim/compaction latency by their folio - // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = - gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); - let gem = gem::new_object( - device, - size, - data.flags, - resv_obj, - reclaim_cpu, - pages_no_reclaim, - commit, - )?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; + if reg_data.gpu.prepare_new_objects() { - #[cfg(CONFIG_DRM_ASAHI_G15)] - if reg_data.gpu.as_any().is::() { - gem::prepare_new_object_g15(&gem)?; - } else { - let registered = device.registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let registered = device.registration_guard().ok_or(ENODEV)?; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - { - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } + gem::prepare_new_object(&gem)?; } - let handle = gem.create_handle(file)?; data.handle = handle; @@ -843,17 +699,11 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 - || data.num_binds > limit - || data.userptr == 0 - || data.stride % 8 != 0 - || data.stride as usize > kernel::page::PAGE_SIZE - { - cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); + if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 + || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { return Err(EINVAL); } } - if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -864,7 +714,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -882,7 +732,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1330,33 +1180,11 @@ impl File { return Err(EINVAL); } - // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and - // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority is not supported on this GPU\n" - ); - return Err(EINVAL); - } - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - // SAFETY: capable() reads the current ioctl task's credentials. - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } - && !file.is_current_master() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" - ); - return Err(EACCES); - } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1377,7 +1205,6 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, - file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1419,9 +1246,6 @@ impl File { } let gpu = ®_data.gpu; - if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { - return Err(E2BIG); - } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1454,7 +1278,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as usize), + UserPtr::from_addr(data.cmdbuf as _), data.cmdbuf_size as usize, ) .reader() @@ -1466,13 +1290,8 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = SyncItem::parse_array( - file, - data.syncs, - data.in_sync_count, - data.out_sync_count, - gpu.as_any().is::(), - )?; + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; mod_dev_dbg!( device, @@ -1536,7 +1355,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = u64::from(gpu.base_clock_hz()); + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 52261d10757d1..46c05699d7722 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,11 +44,6 @@ struct F32U { } impl F32 { - /// Return the IEEE-754 binary32 representation. - pub(crate) const fn to_bits(self) -> u32 { - self.0 - } - /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) @@ -140,12 +135,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(i64::from(v)).pack() + F32U::from_i64(v as i64).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(u64::from(v)).pack() + F32U::from_u64(v as u64).pack() } } }; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 9ffd858598b1c..75d3ecc30ffe5 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for -//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting -//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait +//! for the native display and PMP supplier before booting firmware. T6031 retains explicit +//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -144,23 +143,22 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &&hw::t6030::CONFIG, + &hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &&hw::t6031::CONFIG, + &hw::t6031::CONFIG, ), ]); -// Probe admission requires the native display before firmware starts (T6030 -// also its PMP supplier), so both may be loaded at device creation. +// T6030 probe admission requires the native display and PMP supplier before +// firmware starts. T6031 retains explicit loading until it has an equivalent +// display-order admission gate. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - @none, - [ - of::DeviceId::new(c_str!("apple,agx-t6030")), - of::DeviceId::new(c_str!("apple,agx-t6031")), - ] + MODULE_OF_TABLE_G15, + (), + [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -179,12 +177,6 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { - if let Some(driver) = &this._driver { - driver.close_binding(); - } - } - /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -219,6 +211,7 @@ impl platform::Driver for Driver<'_> { }, }) } + } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -229,8 +222,6 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); - // SAFETY: The helper only reads the device tree. - unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -381,7 +372,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and the client aliases of every user VM. + /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -446,7 +437,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -459,18 +450,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain()?; + gpu.retain(); Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once; `boot::start` already took it before touching the - /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) -> Result { - boot::glue::retain_module() + pub(crate) fn retain(&self) { + boot::glue::retain_module(); } } @@ -545,21 +535,7 @@ impl Firmware { vms, }, ); - let result = firmware.init_graphs().and_then(|_| { - if cfg.platform != hw::Platform::AdtWithPmp { - return Ok(()); - } - // A report before firmware startup is ACKed without enabling bandwidth scaling. - // Reassert it after setup, including the backend's final device control. - // SAFETY: This session retains its acknowledged T6030 AGX power vote. - let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; - if ret == 0 { - Ok(()) - } else { - Err(Error::from_errno(ret)) - } - }); - if let Err(e) = result { + if let Err(e) = firmware.init_graphs() { let Firmware { session, init, @@ -654,7 +630,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self, _vm: &mmu::Vm) { + fn vm_closed(&self) { vm::closed(self) } @@ -669,10 +645,9 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, + queue::Queue::new(self, vm, priority, usc_exec_base)?, GFP_KERNEL, )?) } @@ -681,7 +656,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) + ) -> Result { + timestamp::map(self, bo, range) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 53e99f5a772b2..854ff34e729a3 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,10 +25,8 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement. Failed published jobs retain their slots until -//! their last reference drops (until reboot for jobs retained after a fail-stop). -//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` +//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -98,7 +96,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(None, pa, count, f) } + unsafe { pgtable::with_root_entries(pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a91f267f9e1e0..90bb25451019e 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,6 +11,7 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. +use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -35,7 +36,6 @@ use kernel::{ }, macros::versions, new_mutex, - new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,7 +45,6 @@ use kernel::{ }, Arc, Mutex, - SpinLock, UniqueArc, // }, time::{ @@ -86,7 +85,6 @@ use crate::{ queue, regs, stats, - stats_raw, workqueue, // }; @@ -261,30 +259,18 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, - /// Last raw firmware statistics of this device, exported read-only - /// through `stats_raw`. - raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, + #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>, + garbage_contexts: Mutex>>>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { - /// Admission bound checked before copying the userspace command stream. - fn max_cmdbuf_size(&self) -> usize { - usize::MAX - } - /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; - /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. - fn raw_stats_snapshot(&self) -> Option>> { - None - } - /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -299,10 +285,6 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; - /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). - fn supports_soft_faults(&self) -> bool { - crate::module_parameters::fault_control.value() == 0xb - } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -315,14 +297,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; } if self.queue_limits().is_some() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && self.supports_soft_faults() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); + if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags } @@ -330,46 +312,34 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Backend-specific user address aperture; other GPUs keep their geometry's range. fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which - /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against - /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before - /// resolving syncs because the queue installs a fence in every output sync even on failure. + /// Use tracked command publication and sticky VM status on this backend. fn tracked_submission(&self) -> bool { false } - /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. + /// Whether faults may be reported as recoverable to userspace. fn soft_faults(&self) -> bool { true } - /// Whether new user objects must be populated, cleaned to the point of coherency and pinned - /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose - /// default CPU mappings are write-combined). + /// Populate and clean fresh object backing before any WC CPU mapping. fn prepare_new_objects(&self) -> bool { false } - /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also - /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at - /// most a page (EINVAL otherwise). `None` keeps the shared rules. + /// Optional backend-specific upper bound for a VM_BIND batch. fn bind_batch_limit(&self) -> Option { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was closed; release backend-only registry references. + fn vm_closed(&self) {} /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Create a VM charged to a DRM file when the backend bounds retained bindings. - fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { - self.new_vm(kernel_range) - } - /// Release the backend's handle owner after closing user mapping admission. - /// Accepted jobs and firmware resource leases retain their own references. - fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -378,14 +348,13 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result>; - /// Bind a BO range as a firmware- or CPU-written timestamp buffer. + /// Map a BO as a timestamp buffer fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -427,8 +396,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context, with the objects it retains, for invalidation. - fn free_context(&self, ctx: workqueue::RetiredContext); + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); } /// Private generic trait for functions that don't need to escape this module. @@ -773,7 +742,6 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, - mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -838,12 +806,6 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); - // Per-device raw statistics snapshot, shared with the debugfs export. - let raw_stats_snap = Arc::pin_init( - new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), - GFP_KERNEL, - )?; - let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -859,7 +821,6 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, - raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -887,7 +848,6 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), - raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1119,7 +1079,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(u64::from(map.writable)), + readwrite: U64(map.writable as u64), }; }); @@ -1169,14 +1129,20 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let guard = self.dev.registration_guard()?; + let Some(guard) = self.dev.registration_guard() else { + return None; + }; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if let Some(fault) = &info { - dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); } info }) @@ -1343,10 +1309,6 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } - fn raw_stats_snapshot(&self) -> Option>> { - Some(self.raw_stats.clone()) - } - fn as_any(&self) -> &dyn Any { self } @@ -1429,7 +1391,6 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1447,7 +1408,6 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, - usage, )?, GFP_KERNEL, )?) @@ -1457,15 +1417,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(bo.map_range_into_range( + ) -> Result { + bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - )?) + ) } } @@ -1490,7 +1450,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx.data).is_err() { + if self.invalidate_context(&ctx).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1499,9 +1459,6 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } - // The firmware no longer knows the context's queues, so their structures may go now, - // ahead of the context data they point to. - core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1770,7 +1727,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: workqueue::RetiredContext) { + fn free_context(&self, ctx: KBox>) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f146b2762d812..e868e30baca4c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::{size_of, ManuallyDrop}; +use core::mem::size_of; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,6 +29,7 @@ use core::sync::atomic::{ use kernel::{ addr::{ + DmaAddr, PhysicalAddr, // }, c_str, @@ -108,17 +109,18 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; +use pin_init; + const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod shared; mod lifetime; -mod ppl; mod residency; -mod shared; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -152,81 +154,23 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// One or two coprocessor instances that never touch the handoff region (G17). + /// Two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at - /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. - /// TTBAT context 1 starts as an alias of the kernel context. + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: Option, + secondary_root_offset: u64, }, } -/// UAT policy of a SoC where it differs from the A18 Pro's. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct UatQuirks { - /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and - /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of - /// `struct page`. - /// - /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the - /// regions lose `no-map` and this path goes away. - pub(crate) reserved_tables: bool, - // The following settings keep a SoC on the only policy it was tested with. Each one is - // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. - /// Write back every page-table and context-table change: the GPU table walker is not - /// assumed to snoop the CPU caches. - pub(crate) clean_tables: bool, - /// Invalidate ranges one page at a time, never with range TLBI operations. - pub(crate) page_tlbi: bool, - /// Take the handoff lock around context-table edits. Where the firmware leaves its half of - /// the handoff page alone, its lock bytes are not initialized and are never taken. - pub(crate) handoff_lock: bool, - /// Handoff "current slot" written at initialization. - pub(crate) handoff_slot: u32, - /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in - /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. - pub(crate) native_gpu_permissions: bool, -} - -impl UatQuirks { - /// The A18 Pro and earlier policy. - pub(crate) const DEFAULT: UatQuirks = UatQuirks { - reserved_tables: false, - clean_tables: false, - page_tlbi: false, - handoff_lock: true, - handoff_slot: 0, - native_gpu_permissions: false, - }; -} - const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// M4's nonzero wire incarnation of a host-owned root installation. -const fn binding_wire_generation(incarnation: u64) -> u8 { - if incarnation == 0 { - 0 - } else { - (1 + (incarnation - 1) % 255) as u8 - } -} - -/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in -/// context 0; the existing G15 handoff shares them with user contexts. -const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { - if map_kernel_to_user { - root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT - } else { - 0 - } -} - /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -366,7 +310,6 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, - bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -412,54 +355,19 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -pub(crate) struct VmReadGuard(VmExecGuard); - -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -impl VmReadGuard { - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { - Vm::covers_range_locked(&mut self.0, address, size, read, write) - } - - pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { - Vm::read_user_locked(&mut self.0, va, buf) - } -} - /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, - /// Nonzero M4 wire incarnation of the root currently owning this slot. - generation: u8, ttb: u64, } /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { - /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. - /// - /// Map steps update this monotonically under the VM reservation before - /// linking the GPUVA. Remap pieces keep their original sequence. - newest_bind: AtomicU64, #[pin] - segments: Mutex>, -} - -/// Copied DMA addresses and the lease covering every alias of this VM/BO association. -/// The lease is dropped only after all aliases have been unmapped and invalidated. -struct SgSegments { - segments: KVVec<(DmaAddress, ResourceSize)>, - _lease: kernel::drm::gem::shmem::SGLease, -} - -impl SgSegments { - fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { - self.segments.iter() - } + segments: Mutex>>, } /// Preallocated resources needed to execute a VM operation. @@ -500,9 +408,6 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, - - /// Sequence assigned by the successful map for its related metadata. - bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -515,7 +420,6 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, - bind_sequence: 0, }) } @@ -546,7 +450,6 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, - bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -565,11 +468,6 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { - let bind_sequence = self.bind_sequence.fetch_update( - Ordering::AcqRel, - Ordering::Acquire, - |value| value.checked_add(1), - ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -578,16 +476,22 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - if ctx.dev.is_none() { + let Some(dev) = ctx.dev else { + dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); return Err(ENODEV); - } - // Bind preparation retained the table's backing before taking the VM reservation. - // Never acquire the object reservation from inside the GPUVM map callback. - let segments = op.vm_bo().data().segments.lock(); - let sgt = segments.as_ref().ok_or(EFAULT)?; - for &(dma_address, dma_len) in sgt.iter() { - let mut addr = dma_address as usize; - let mut len = dma_len as usize; + }; + + let sgt = op.obj().sg_table(dev).inspect_err(|e| { + dev_err!( + self.dev.as_ref(), + "sm_step_map() Failed to get SG table: {:?}", + e + ); + })?; + for sgt_entry in sgt.iter() { + // TODO: proper DMA address/length handling + let mut addr = sgt_entry.dma_address() as usize; + let mut len: usize = sgt_entry.dma_len() as usize; if left == 0 { break; @@ -634,7 +538,6 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } - drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -642,18 +545,12 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; - op.vm_bo() - .data() - .newest_bind - .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), - bind_sequence, }, ); - ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -679,7 +576,6 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; - let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -720,8 +616,7 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table - .discard_partial_map(unmap_start..unmap_end)?; + self.page_table.discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -745,11 +640,9 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), - bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), - bind_sequence: orig_bind_sequence, }, ); @@ -764,12 +657,7 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!( - self.dev, - "MMU: unmap: {:#x}:{:#x}\n", - va.addr(), - va.length() - ); + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -804,7 +692,14 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - self.finish_failed_map(addr, size as usize); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -814,19 +709,6 @@ impl VmInner { } } - /// Complete discarded stores and invalidate both the legacy bound slot and - /// every M4/G17 context naming this root. Call only after releasing PPL. - fn finish_failed_map(&self, addr: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size); - } - fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -881,19 +763,15 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind - /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those - /// contexts before allowing an unmapped object's backing to be released. + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { + if self.uat_inner.firmware == UatFirmware::Handoff { mem::sync(); return; } - let contexts = self - .uat_inner - .lock() - .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); self.tlbi_context_mask(iova, size, contexts); } @@ -921,35 +799,10 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); - // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not - // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; - let uat_inner = self.uat_inner.clone(); - let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); - let discard = if result.is_err() { - // Rollback still changes live leaves and must remain under the PPL lease. - self.page_table.discard_partial_map(node.start()..end) - } else { - Ok(()) - }; - // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. - drop(handoff); if result.is_err() { - self.finish_failed_map(node.start(), node.mapped_size); - } else { - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); - } - if let Err(error) = discard { - if self.uat_inner.m4_adopted { - // The caller retains its node and backing when this latch is set. - self.uat_inner.failed.store(true, Ordering::Release); - } - dev_err!( - self.dev.as_ref(), - "Could not discard failed kernel mapping: {:?}\n", - error - ); + self.rollback_failed_map(node.start(), node.mapped_size as u64); } result } @@ -957,14 +810,7 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node - .bo - .as_ref() - .ok_or(EINVAL)? - .get()? - .data() - .segments - .lock(); + let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -1022,7 +868,8 @@ impl VmInner { } } - // The caller invalidates negative translations after releasing its PPL lease. + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -1030,11 +877,9 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { - device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, - bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -1101,7 +946,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1109,9 +954,8 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); - inner.publish_slot(id); } - inner.handoff_unlock(); + inner.handoff().unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1152,12 +996,6 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } - - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn generation(&self) -> u8 { - self.0.binding.lock().generation - } } impl Drop for VmBind { @@ -1198,7 +1036,10 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new(bo: ARef>, owner: ARef>) -> Self { + fn new( + bo: ARef>, + owner: ARef>, + ) -> Self { Self { bo: Some(bo), owner, @@ -1241,90 +1082,29 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's -/// reservation lock, so it must not be dropped where fence producers run. -pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); - -impl TimestampVmap { - pub(crate) fn new(gem: &gem::Object) -> Result { - Ok(Self(gem.owned_vmap::<0>()?)) - } - - fn size(&self) -> usize { - use kernel::drm::gem::BaseObject; - self.0.owner().size() - } - - /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete - /// atomic store; byte stores support unaligned destinations without creating mutable - /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused - /// when the destination is prepared, and ignored here. - pub(crate) fn write(&self, offset: usize, value: u64) { - if offset - .checked_add(size_of::()) - .is_none_or(|end| end > self.size()) - { - return; - } - // Completion runs in task context. Bindings of the same BO share this lock, - // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.0.owner().timestamp_lock.lock(); - let base = self.0.addr.cast::(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. - let pointer = unsafe { base.add(offset) }; - if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: All eight bytes were checked above; the vmap pins the backing. - // A naturally aligned query result becomes available in one complete store. - let destination = unsafe { &*pointer.cast::() }; - destination.store(value.to_le(), Ordering::Relaxed); - } else { - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The complete eight-byte extent was checked above. The owned - // vmap pins its backing; byte atomics permit unaligned and overlapping - // timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(offset + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - } - // A CPU-retired query can be consumed by a later GPU command. WRITEBACK - // objects use a cacheable vmap, while GPU mappings need not snoop that cache. - // Publish the bytes before signalling the completion fence or queue frontier. - if self.0.owner().is_writeback() { - crate::pgtable::clean_poc(pointer, size_of::()); - } - crate::pgtable::poc_barrier(); - } -} - -/// Prepared CPU timestamp access with its own vmap; destruction happens after completion -/// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +/// Prepared CPU timestamp access; destruction happens after completion signalling. pub(crate) struct TimestampMapping { - mapping: TimestampVmap, + mapping: kernel::drm::gem::shmem::VMapOwned, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { - use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: TimestampVmap::new(gem)?, - offset, - }) - } - pub(crate) fn write(&self, value: u64) { - self.mapping.write(self.offset, value); + let base = self.mapping.addr.cast::(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*base.add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(ManuallyDrop>); +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1348,22 +1128,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - self.0.owner.lock_inner().page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Aligned results - /// use a complete atomic store; byte stores support unaligned destinations without - /// creating mutable aliases. An unaligned complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - let gem = self.0._gem.as_ref().ok_or(EINVAL)?; - TimestampMapping::new(gem, self.timestamp_offset(offset)?) - } - - /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete - /// extent lies inside both the mapping and the object. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1374,12 +1146,10 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(start) - } - - /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. - pub(crate) fn timestamp_vmap(&self) -> Result { - TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) + Ok(TimestampMapping { + mapping: gem.owned_vmap::<0>()?, + offset: start, + }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1491,13 +1261,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff_lock(); + uat_inner.handoff().lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff_unlock(); + uat_inner.handoff().unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1587,19 +1357,6 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { - // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected - // failed construction must retain its complete node/backing instead of invoking the - // unrelated G13/G14 FwCtl protocol. - if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { - self.0.uat_inner.failed.store(true, Ordering::Release); - pr_err!( - "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", - self.iova(), - self.size() - ); - return; - } - // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1642,27 +1399,8 @@ impl Drop for KernelMapping { self.size() ); - let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { - Ok(guard) => guard, - Err(error) => { - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after PPL failure: {:?}\n", - error - ); - return; - } - }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { - if self.0.uat_inner.m4_adopted { - self.0.uat_inner.failed.store(true, Ordering::Release); - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after partial unmap failure\n" - ); - return; - } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1671,8 +1409,6 @@ impl Drop for KernelMapping { ); } - drop(upper_handoff); - if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1689,11 +1425,6 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); - drop(_mutation); - drop(owner); - // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have - // been released. Uncertain M4 paths return earlier, retaining the node and its pins. - unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1702,73 +1433,32 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], - /// Firmware-observed incarnations of application roots installed by the VM slot allocator. - binding_generations: [u64; UAT_NUM_CTX], - binding_published: [bool; UAT_NUM_CTX], - /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by - /// successful host publications, never learned by accepting observed hardware words. - owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, - quirks: UatQuirks, } impl UatShared { - /// Takes the handoff lock, where this SoC uses it. - fn handoff_lock(&self) { - if self.quirks.handoff_lock { - self.handoff().lock(); - } - } - - /// Releases the handoff lock taken by [`Self::handoff_lock`]. - fn handoff_unlock(&self) { - if self.quirks.handoff_lock { - self.handoff().unlock(); - } - } - - /// Writes back context-table slot `idx` after it changed, where this SoC needs it. - fn publish_slot(&self, idx: usize) { - if self.quirks.clean_tables { - let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); - pgtable::clean_poc(slot, size_of::()); - pgtable::poc_barrier(); - } - } - /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { - self.ttbs_rgn - .map - .ptr() - .cast::<[SlotTTBS; UAT_NUM_CTX]>() - .as_ref() - } - .unwrap() + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { + fn contexts_naming_root(&self, ttb: u64) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| { - let expected = tagged_root(ttb, *ctx); - let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. - slot.ttb0.load(Ordering::Acquire) == expected - }) + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1785,138 +1475,20 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. - m4_adopted: bool, - /// An uncertain mapping/root is retained; future M4 allocation and publication fail. - failed: AtomicBool, - /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. + #[cfg(CONFIG_DRM_ASAHI_G15)] clean_poc: bool, - /// This SoC's UAT policy. - quirks: UatQuirks, - /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). - reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } -/// A transport owns the sole host-side PPL mutex before touching AP interest. -struct M4Transport<'a> { - shared: Guard<'a, UatShared, MutexBackend>, - start: Instant, - failed: &'a AtomicBool, -} - -impl ppl::Memory for M4Transport<'_> { - fn firmware_interested(&self) -> bool { - self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 - } - fn firmware_turn(&self) -> bool { - self.shared.handoff().turn.load(Ordering::Relaxed) != 0 - } - fn ap_interest(&self, interested: bool) { - self.shared - .handoff() - .lock_ap - .store(u8::from(interested), Ordering::Relaxed); - } - fn give_firmware_turn(&self) { - self.shared.handoff().turn.store(1, Ordering::Relaxed); - } - fn barrier(&self) { - mem::sync(); - } - fn expired(&self) -> bool { - self.start.elapsed() > Delta::from_millis(100) - } - fn relax(&self) { - core::hint::spin_loop(); - } - fn fail(&self) { - self.failed.store(true, Ordering::Release); - } -} - -/// Every AP firmware transaction owns the same mutex, including G16 Window operations. -/// M4 drops its protocol guard before its transport releases the host mutex. The legacy -/// handoff path keeps the preexisting protocol and lock ordering for other generations. -#[must_use = "retain the guard until protected table mutations have completed"] -enum HandoffGuard<'a> { - M4(ppl::Guard>), - Legacy(Guard<'a, UatShared, MutexBackend>), -} - -impl HandoffGuard<'_> { - fn shared_mut(&mut self) -> &mut UatShared { - match self { - Self::M4(guard) => &mut guard.memory_mut().shared, - Self::Legacy(shared) => shared, - } - } -} - -impl Drop for HandoffGuard<'_> { - fn drop(&mut self) { - if let Self::Legacy(shared) = self { - shared.handoff_unlock(); - } - } -} - -/// Opaque host-and-firmware lease for G16 window transactions. No caller may -/// independently manipulate the AP-interest or turn words. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[must_use = "retain the guard throughout the reserved-table window transaction"] -pub(crate) struct M4HandoffGuard<'a> { - _transaction: HandoffGuard<'a>, -} - impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } - fn check_m4(&self) -> Result { - if self.m4_adopted && self.failed.load(Ordering::Acquire) { - Err(EIO) - } else { - Ok(()) - } - } - - /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw - /// AP interest while still holding the mutex and latch M4 failure before releasing it. - fn lock_handoff(&self) -> Result> { - self.check_m4()?; - let shared = self.lock(); - self.check_m4()?; - if !self.m4_adopted { - shared.handoff_lock(); - return Ok(HandoffGuard::Legacy(shared)); - } - let transport = M4Transport { - shared, - start: Instant::::now(), - failed: &self.failed, - }; - match ppl::Guard::acquire(transport) { - Ok(guard) => Ok(HandoffGuard::M4(guard)), - Err(()) => Err(ETIMEDOUT), - } - } - - /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. - fn lock_m4_upper(&self, upper: bool) -> Result>> { - if upper && self.m4_adopted { - self.lock_handoff().map(Some) - } else { - Ok(None) - } - } - /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1944,9 +1516,7 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 { - core::hint::spin_loop() - } + while self.turn.load(Ordering::Relaxed) != 0 {} self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1971,9 +1541,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(slot, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2053,39 +1623,19 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { +fn prefetch_sgt_segments( + dev: &device::Device, + gem: &gem::Object, +) -> Result> { let sgt = crate::gem::held_sg_table(gem, dev)?; + // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { - segments, - _lease: sgt.into_lease(), - }) -} - -/// Reuse cached segments without allocating another copy for each bind. The bind lease was -/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, -/// and its extra count excludes new eviction until preparation/commit finishes. -fn prepare_sgt_segments( - dev: &device::Device, - gem: &gem::Object, - bo: &gpuvm::GpuVmBo, - _bind_lease: &kernel::drm::gem::shmem::SGLease, -) -> Result { - let needs_segments = bo.data().segments.lock().is_none(); - if needs_segments { - // Never enter the object reservation while holding the BO mutex. - let prefetched = prefetch_sgt_segments(dev, gem)?; - let mut segments = bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetched); - } - } - Ok(()) + Ok(segments) } impl Vm { @@ -2108,12 +1658,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, - uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? - .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; + + #[cfg(CONFIG_DRM_ASAHI_G15)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2139,7 +1689,6 @@ impl Vm { VmBinding { binding: None, bind_token: None, - generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -2153,7 +1702,6 @@ impl Vm { } else { None }; - let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -2166,7 +1714,6 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), - bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -2176,14 +1723,11 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); - let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { - device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, mapping_epoch, - bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -2196,10 +1740,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -2210,18 +1751,15 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout - /// attribution compares and logs VMs by it). + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn id(&self) -> u64 { self.id } @@ -2233,18 +1771,9 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - self.read_lock()?.read_user(va, buf) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - pub(crate) fn read_lock(&self) -> Result { - Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { + let mut inner = lock_vm(&self.gpuvm, None, false)?; let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2314,7 +1843,9 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2329,22 +1860,26 @@ impl Vm { if stride <= block_size { return false; } - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) }) } - /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - // Every mapping change holds the VM reservation, so readers need only that one. The full - // exec lock would also lock every external BO, making each submit O(external BOs). - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return (false, None); + }; let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2357,10 +1892,6 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } - fn current_bind_sequence(&self) -> u64 { - self.bind_sequence.load(Ordering::Acquire) - } - /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2396,49 +1927,6 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Best, - ) - } - - /// Lowest-address placement for a monotonic search across several VMs. - pub(crate) fn map_in_range_lowest( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Low, - ) - } - - #[allow(clippy::too_many_arguments)] - fn map_in_range_with_mode( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -2453,30 +1941,29 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; let size = object_range.range(); - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2497,7 +1984,7 @@ impl Vm { 0, range.start, range.end, - mode, + mm::InsertMode::Best, ); let mut node = match result { Ok(node) => node, @@ -2515,16 +2002,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a GEM object into this Vm at a specific address. @@ -2542,32 +2022,33 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock_vm(/*Some(&gem)*/ None, false) + .exec_lock(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2602,16 +2083,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2629,16 +2103,14 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2651,13 +2123,19 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2700,22 +2178,6 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } - /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn mark_binding_published(&self) -> Result { - let uat = self.gpuvm.lock_inner().uat_inner.clone(); - let binding = self.binding.lock(); - let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; - let mut shared = uat.lock(); - if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) - || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] - { - return Err(EIO); - } - shared.binding_published[slot] = true; - Ok(()) - } - /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2725,9 +2187,7 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2761,35 +2221,22 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; - let transaction_uat = inner.uat_inner.clone(); - let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner - .page_table - .map_pages(iova..end, phys as PhysicalAddr, prot, false); - let discard = if ret.is_err() { - inner.page_table.discard_partial_map(iova..end) - } else { - Ok(()) - }; - drop(handoff); + let ret = inner.page_table.map_pages( + iova..end, + phys as PhysicalAddr, + prot, + false, + ); if ret.is_err() { - inner.finish_failed_map(iova, size); - } else { - inner.tlbi_contexts_naming_root(iova, size); - } - let retain = discard.is_err() && inner.uat_inner.m4_adopted; - if retain { - inner.uat_inner.failed.store(true, Ordering::Release); + inner.rollback_failed_map(iova, size as u64); } + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. drop(mutation); - drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } + let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Unmap everything in an address range. @@ -2831,9 +2278,7 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2864,7 +2309,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2891,66 +2336,6 @@ impl Vm { Ok(()) } - fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { - let _residency = self.enter_retirement_residency(); - // Removing whole mappings only does unmaps, so no preallocated VAs - // are needed for the indexed path. - let mut resources = VmOpResources { - preallocated_gpuvas: [None, None, None], - }; - let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; - - // The BO's newest sequence is checked under the same lock as binds. If - // it is not newer than the close, every mapping belongs to that close - // and the per-object list is safe and substantially cheaper to walk. - // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { - Err(error) if error == ENOMEM => { - return self.drop_mappings_filtered(gem, sequence); - } - result => result?, - }; - let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { - return Ok(()); - }; - if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { - drop(vm_exec); - drop(bo); - return self.drop_mappings_filtered(gem, sequence); - } - - mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); - let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); - if result.is_ok() { - self.untrack_context_object_before(gem, sequence); - self.untrack_shared_object_before(gem, sequence); - } - drop(vm_exec); - drop(bo); - if result == Err(ENOMEM) { - return self.drop_mappings_filtered(gem, sequence); - } - result - } - - /// A newer binding shares the BO list, so select the closed generation by - /// walking the VM without allocating. This is also the ENOMEM fallback. - fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { - let mut inner = self.gpuvm.lock_inner(); - let result = inner.unmap_object_where( - gem, - |va| va.data().bind_sequence <= sequence, - VmInner::unmap_gpuva, - ); - if result.is_ok() { - self.untrack_context_object_before(gem, sequence); - self.untrack_shared_object_before(gem, sequence); - } - drop(inner); - self.bo_deferred_cleanup(); - result - } - /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { @@ -2997,18 +2382,8 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = match self.uat_inner.lock_handoff() { - Ok(guard) => guard, - Err(_) => { - self.page_table.retain(); - pr_err!( - "MMU: retaining M4 VM {} page tables after PPL failure\n", - self.id - ); - return; - } - }; - let uat_inner = transaction.shared_mut(); + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -3021,10 +2396,9 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); - uat_inner.owned_user_roots[idx] = 0; - uat_inner.publish_slot(idx); } - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -3044,7 +2418,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -3055,56 +2429,34 @@ pub(crate) struct AdoptedRegions { } impl Uat { - /// Common AP and firmware exclusion for G16's reserved-table window. Acquire - /// immutable root addresses before this guard, preserving binding -> UAT lock order. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn lock_m4_handoff(&self) -> Result> { - if !self.inner.m4_adopted { - return Err(EINVAL); - } - Ok(M4HandoffGuard { - _transaction: self.inner.lock_handoff()?, - }) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Map a region given by its physical range, write-combined. - fn map_physical( - range: &Range, - min_size: usize, - reserved_ram: bool, - ) -> Result { + fn map_physical(range: &Range, min_size: usize) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as usize; + raw.flags = kernel::bindings::IORESOURCE_MEM as _; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - let flags = io::mem::MemFlag::WC; - #[cfg(CONFIG_DRM_ASAHI_G15)] - let flags = if reserved_ram { - // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. - if crate::g15::memory_mode(range.start, size as u64)? - == kernel::bindings::MEMREMAP_WB as u32 - { - io::mem::MemFlag::WB - } else { - flags - } + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + let mode = crate::g15::memory_mode(range.start, size as u64)?; + let ram = mode == kernel::bindings::MEMREMAP_WB as u32; + let flags = if ram { + io::mem::MemFlag::WB } else { - flags + io::mem::MemFlag::WC }; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let _ = reserved_ram; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; - // the GPU reaches them through its own translation. + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU + // reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3112,7 +2464,7 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3126,52 +2478,23 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. - #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, true, true) - } - - /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also - /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; - /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn new_adopted_with_user_upper( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn new_adopted_inner( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - reserved_ram: bool, - ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, - m4_adopted: !map_kernel_to_user, - failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, - reserved: None, - quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3180,14 +2503,10 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user, + map_kernel_to_user: true, handoff_rgn, ttbs_rgn, - quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3199,14 +2518,8 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = Vm::new( - dev.into(), - inner.clone(), - geometry.user_range(), - geometry, - None, - 1, - )?; + let kernel_lower_vm = + Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -3233,13 +2546,13 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3303,16 +2616,13 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self - .inner - .lock() - .contexts_naming_root(root, self.inner.m4_adopted)) + Ok(self.inner.lock().contexts_naming_root(root)) } /// Publish a completed alias set before its descriptors become visible. @@ -3341,24 +2651,12 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, true) - } - - /// Binds a VM without waiting for another slot owner to retire. - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn try_bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, false) - } - - fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { - self.inner.check_m4()?; let mut binding = vm.binding.lock(); - self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = module_parameters::robust_isolation.value() != 0; + let isolation = *module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3366,34 +2664,22 @@ impl Uat { None }); - let slot = if wait { - self.slots.get(binding.bind_token)? - } else { - self.slots.try_get(binding.bind_token)? - }; + let slot = self.slots.get(binding.bind_token)?; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = self.inner.lock_handoff()?; - let uat_inner = transaction.shared_mut(); - // On M4, empty bind/close churn must not wrap the wire generation - // back to firmware's last observed identity for a different root. - let previous = uat_inner.binding_generations[idx]; - let incarnation = if !self.inner.m4_adopted - || previous == 0 - || uat_inner.binding_published[idx] - { - previous.checked_add(1).ok_or(EOVERFLOW)? + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT } else { - previous + 0 }; - let ttb1 = - application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); - let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -3402,12 +2688,8 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.binding_generations[idx] = incarnation; - uat_inner.binding_published[idx] = false; - uat_inner.owned_user_roots[idx] = ttb; - binding.generation = binding_wire_generation(incarnation); - uat_inner.publish_slot(idx); - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -3426,35 +2708,6 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } - /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every - /// host root installation/removal; firmware does not write these words. An unexpected - /// observed root is an error, never a new ownership grant. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn check_adopted_bindings(&self) -> Result { - self.inner.check_m4()?; - // Binding locks precede the global UAT lock, as in bind(). - let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; - let inner = self.inner.lock(); - self.inner.check_m4()?; - if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { - return Err(EINVAL); - } - let ttbs = inner.ttbs(); - if ttbs[0].ttb0.load(Ordering::Acquire) != lower - || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID - { - return Err(EIO); - } - for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { - if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || table.ttb1.load(Ordering::Acquire) != 0 - { - return Err(EIO); - } - } - Ok(()) - } - /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -3462,7 +2715,7 @@ impl Uat { return Err(EINVAL); } { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3506,7 +2759,6 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); - inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3516,17 +2768,14 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - self.inner.check_m4()?; - let vm = Vm::new( + Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - )?; - self.inner.check_m4()?; - Ok(vm) + ) } /// Returns the address space geometry of this UAT. @@ -3548,19 +2797,14 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - UAT_PGSZ / size_of::(), - |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) }?; mem::sync(); @@ -3573,16 +2817,11 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - snapshot.len(), - |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) }?; if !matches { return Err(EIO); @@ -3594,19 +2833,15 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let Some(secondary) = self.inner.lock().secondary_ttb1 else { - // A single-instance firmware has no secondary root to mirror. - return Ok(()); - }; + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(reserved, secondary, count, |secondary| { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3623,33 +2858,24 @@ impl Uat { Ok(()) } - /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). - pub(crate) fn invalidate_reserved_tables(&self) { - if let Some(reserved) = self.inner.reserved.as_ref() { - reserved.invalidate(); - } - } - /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff_unlock(); + inner.handoff().unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.publish_slot(KERNEL_ALIAS_CTX); - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3659,30 +2885,21 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner( - dev: driver::AsahiDevice, - firmware: UatFirmware, - quirks: UatQuirks, - reserved: Option>, - ) -> Result> { + fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); - let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - m4_adopted: false, - failed: AtomicBool::new(false), - clean_poc: quirks.clean_tables, - quirks, - reserved, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: false, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3691,14 +2908,10 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, - quirks: shared_quirks, }, "uat_shared" ), @@ -3714,27 +2927,12 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, - quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - mem::set_page_tlbi(quirks.page_tlbi); - let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let reserved = if quirks.reserved_tables { - let mut tables = pgtable::ReservedTables::new(); - for name in [c"pagetables", c"l2-pagetables"] { - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - // SAFETY: The boot loader reserved these regions (no-map) for the firmware's - // page tables; they stay reserved until reboot. - unsafe { tables.add(res)? }; - } - Some(Arc::new(tables, GFP_KERNEL)?) - } else { - None - }; - - let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let inner = Self::make_inner(dev.clone(), firmware)?; + let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3742,11 +2940,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset: Some(offset), - } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), - UatFirmware::PublishedRoots { - secondary_root_offset: None, - } => (UAT_PGSZ, None), + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3797,18 +2995,11 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; - if quirks.clean_tables { - pgtable::clean_poc( - core::ptr::from_ref(inner.handoff()).cast::(), - HANDOFF_SIZE, - ); - pgtable::poc_barrier(); - } + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); @@ -3820,7 +3011,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if matches!(firmware, UatFirmware::PublishedRoots { .. }) { + if secondary_ttb1.is_some() { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3829,12 +3020,8 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } - if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); - pgtable::poc_barrier(); - } - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); @@ -3852,30 +3039,3 @@ impl Drop for Uat { mem::sync(); } } - -impl Vm { - /// Diagnostic reservation occupancy (including guards), not a placement promise. - /// Removal can progress between queries; allocations are excluded by the VM lock. - pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = self.gpuvm.lock_inner(); - let mut next = range.start; - let mut used = 0; - let mut largest = 0; - while next < range.end { - let Some(node) = inner.mm.first_overlap(next..range.end)? else { - largest = largest.max(range.end - next); - break; - }; - largest = largest.max(node.start.saturating_sub(next)); - let end = node.end.min(range.end); - used += end - node.start.max(next); - next = end; - } - Ok((used, largest)) - } -} -impl Uat { - pub(crate) fn execution_contexts_used(&self) -> u32 { - (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() - } -} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..5f7e572a3a462 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT //! Support for Apple RTKit coprocessors. //! @@ -222,7 +222,6 @@ impl RtKit { } else { 0 }, - syslog_is_debug: None, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), @@ -231,18 +230,11 @@ impl RtKit { } else { None }, - shmem_setup_ep: None, shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { Some(shmem_destroy_callback::) } else { None }, - protocol_version: 0, - }; - - const COREANALYTICS_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { - flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_NEEDS_COREANALYTICS, - ..Self::VTABLE }; /// Creates a new RTKit client for a given device and optional mailbox name or index. @@ -251,29 +243,6 @@ impl RtKit { mbox_name: Option<&'static CStr>, mbox_idx: usize, data: T::Data, - ) -> Result { - Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::VTABLE) - } - - /// Creates a client that requires CoreAnalytics setup during boot. - /// - /// The caller must qualify this dependency for the particular firmware - /// instance. This selects immutable operations before mailbox RX starts. - pub fn new_with_coreanalytics( - dev: &device::Device, - mbox_name: Option<&'static CStr>, - mbox_idx: usize, - data: T::Data, - ) -> Result { - Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) - } - - fn new_with_ops( - dev: &device::Device, - mbox_name: Option<&'static CStr>, - mbox_idx: usize, - data: T::Data, - ops: &'static bindings::apple_rtkit_ops, ) -> Result { let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); let guard = ScopeGuard::new(|| { @@ -291,7 +260,7 @@ impl RtKit { None => ptr::null(), }, mbox_idx.try_into()?, - ops, + &Self::VTABLE, )) }?; @@ -305,10 +274,6 @@ impl RtKit { }) } - /// Borrows the C handle without transferring ownership. - /// The owner must outlive each unsafe C operation and its callbacks. - pub fn as_raw(&self) -> *mut bindings::apple_rtkit { self.rtk } - /// Requests acknowledged RTKit shutdown using the C core's bounded waits. /// /// The caller must not hold a lock needed by its receive callbacks: the @@ -357,57 +322,6 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } - - /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or - /// restores the scheduler's default with `None`. - /// - /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a - /// mailbox without a threaded receive handler. - pub fn set_rx_util_min(&self, util_min: Option) -> Result { - // INVARIANT: `rtk` is valid per the type invariant. - RxUtilMin { rtk: self.rtk }.set(util_min) - } - - /// Returns a handle that sets the receive thread's minimum utilization (see - /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. - /// - /// # Safety - /// - /// The caller must not use the handle after this [`RtKit`] is dropped. - pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { - // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. - RxUtilMin { rtk: self.rtk } - } -} - -/// Sets the minimum utilization of an RTKit coprocessor's receive thread. -/// -/// The request is a single store that the receive thread applies to itself when it next runs, -/// so it may be made from any context. -/// -/// # Invariants -/// -/// `rtk` is valid while the handle is used. -#[derive(Clone, Copy)] -pub struct RxUtilMin { - rtk: *mut bindings::apple_rtkit, -} - -// SAFETY: The request is one store into the mailbox, valid from any thread. -unsafe impl Send for RxUtilMin {} -// SAFETY: As above; the handle has no state of its own. -unsafe impl Sync for RxUtilMin {} - -impl RxUtilMin { - /// See [`RtKit::set_rx_util_min`]. - pub fn set(&self, util_min: Option) -> Result { - let util_min = match util_min { - Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, - None => -1, - }; - // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. - to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) - } } // SAFETY: `RtKit` operations require a mutable reference From e45af192f61d218a59fc2ac06514d1bdd6a6e267 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 01:11:35 -0400 Subject: [PATCH 003/392] drm/asahi: m3: start the T8122 runtime on 7.2 Wire the restored 7.1.12 M3 runtime to the 7.2 driver behind CONFIG_DRM_ASAHI_M3_T8122 (default n), so the M3 MacBook Air (J613, J615) GPU runs again on its 14.8.3 firmware. A T8122 probe still passes the G15G identity gate and the backend selection first, and the runtime still starts only with asahi.t8122_start=1; otherwise the probe is refused as before. Other GPUs are not affected: T6030/T6031 keep the G15 backend or the refusal, and the T6030/T6031 start paths and the T6031 start experiment are dropped from the runtime. On 7.2 the runtime: - registers through drm::Registration as a gpu::Gpu backend, like G15 and G17, and is stopped on unbind while its device resources are live; the completion worker moves from the DRM device data into the runtime's RTKit state; - keeps user mappings alive across accepted work with the shared VM job lifetime (VmJobGuard, retain_job, deferred unmaps), the generalized form of 7.1.12's M3 VM job lifetime, on VMs created with tracked status; - adopts the firmware's UAT as the G15 backend does on 7.2 (Uat::new_adopted and g15::uat: the PPL exchange, TTBAT context 0, the host window at root entry 2 through the shared L2) instead of 7.1.12's M3-only handoff mode in mmu.rs/pgtable.rs. The firmware- walked pages are cleaned to the point of coherency, the host window is linked before the InitData and each batch reach the firmware, and context 0 is invalidated when a job's command aliases go. The option therefore depends on DRM_ASAHI_G15; - refuses VMs to userspace that did not ask GET_PARAMS for the whole global parameter structure (7.1.12's per-file client gate), through a new gpu::Gpu::admit_vm hook that admits every client elsewhere. Smaller 7.2 adaptations: the IoMem/Devres, IoSysMap and RTKit buffer APIs; Queue::submit's registration argument; quiet MMIO mappings in an mmu::m3 helper module; the M3 debug flags, F32::to_bits, Completion::complete/try_wait_for_completion/wait_for_completion_timeout and RtKit::is_running/is_crashed, which 7.2 dropped; the M3 module parameters move into m3_params, and m3_soc.rs gets back its full 7.1.12 table, cfg-gated with the runtime. asahi.m3_expose=0 is refused before anything starts. The devcoredump fault snapshot is dropped (7.2's kernel crate has no devcoredump binding): the first failure is logged. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/Kconfig | 15 + drivers/gpu/drm/asahi/debug.rs | 7 + drivers/gpu/drm/asahi/driver.rs | 57 ++++ drivers/gpu/drm/asahi/file.rs | 14 + drivers/gpu/drm/asahi/float.rs | 5 + drivers/gpu/drm/asahi/g15/mod.rs | 4 +- drivers/gpu/drm/asahi/g15/regs.rs | 15 + drivers/gpu/drm/asahi/g15/uat.rs | 33 +- drivers/gpu/drm/asahi/gpu.rs | 6 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 ----- drivers/gpu/drm/asahi/m3/g15_raw.rs | 30 +- drivers/gpu/drm/asahi/m3/hw.rs | 18 +- drivers/gpu/drm/asahi/m3/hw/agx3.rs | 2 +- drivers/gpu/drm/asahi/m3/initdata.rs | 346 +------------------- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 146 +++------ drivers/gpu/drm/asahi/m3/m3_board.rs | 27 +- drivers/gpu/drm/asahi/m3/m3_client.rs | 40 +-- drivers/gpu/drm/asahi/m3/m3_completion.rs | 8 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 49 +-- drivers/gpu/drm/asahi/m3/m3_config.rs | 123 +++----- drivers/gpu/drm/asahi/m3/m3_device.rs | 117 +++---- drivers/gpu/drm/asahi/m3/m3_drm.rs | 197 +++++++----- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 57 +--- drivers/gpu/drm/asahi/m3/m3_memory.rs | 54 +++- drivers/gpu/drm/asahi/m3/m3_params.rs | 341 ++++---------------- drivers/gpu/drm/asahi/m3/m3_pass.rs | 6 +- drivers/gpu/drm/asahi/m3/m3_pool.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_render.rs | 73 ++--- drivers/gpu/drm/asahi/m3/m3_resources.rs | 12 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 46 +-- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 288 +++++++++-------- drivers/gpu/drm/asahi/m3/m3_soc.rs | 367 +++++++++++++++++++--- drivers/gpu/drm/asahi/m3/m3_submit.rs | 48 +-- drivers/gpu/drm/asahi/m3/m3_thermal.rs | 17 +- drivers/gpu/drm/asahi/m3/mod.rs | 94 +++++- drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 2 +- drivers/gpu/drm/asahi/m3/t8122_start.rs | 22 +- drivers/gpu/drm/asahi/mmu.rs | 2 + drivers/gpu/drm/asahi/mmu/m3.rs | 142 +++++++++ rust/kernel/soc/apple/rtkit.rs | 12 + rust/kernel/sync/completion.rs | 22 ++ 41 files changed, 1467 insertions(+), 1486 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/mmu/m3.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 958600a91b06b..4cfe26f28b891 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -36,6 +36,21 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. +config DRM_ASAHI_M3_T8122 + bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" + depends on DRM_ASAHI + # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. + depends on DRM_ASAHI_G15 + select CRYPTO_LIB_SHA256 + default n + help + Build the M3 runtime that starts the GPU of the M3 MacBook Air + (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves + loaded. It starts only with asahi.t8122_start=1 on the kernel + command line, after the boot loader handed the GPU over; without + it the T8122 probe is refused as before. Other GPUs are not + affected. + config DRM_ASAHI_G15 bool "Native G15 firmware backend (M3 Pro/Max)" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 50628ade5ab8e..e8087442d5320 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -54,6 +54,13 @@ pub(crate) enum DebugFlags { DeviceControlCh = 21, FwCtlCh = 22, + // Host submission/completion timestamps without verbose firmware tracing. + SubmitTiming = 23, + // Low-volume M3 command/geometry totals without per-IRQ printk. + M3SubmitSummary = 24, + // Per-slot retired timestamps, without enabling firmware ktrace. + M3PassTiming = 25, + // 32-35: Allocator debugging FillAllocations = 32, DebugAllocations = 33, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index c092595180cf8..43fa17412815d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -49,6 +49,8 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, + /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, @@ -62,6 +64,11 @@ unsafe impl Send for AsahiPlatformDriverData<'_> {} // SAFETY: As above. unsafe impl Sync for AsahiPlatformDriverData<'_> {} +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +type M3Backend = Arc; +#[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] +type M3Backend = core::convert::Infallible; + /// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiDrmRegistrationData<'drm> { @@ -213,6 +220,11 @@ impl platform::Driver for AsahiPlatformDriver { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = this.m3.as_ref() { + // As for G17: the runtime stops its coprocessor through device resources. + m3.stop(); + } } /// Device probe function. @@ -227,6 +239,8 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; @@ -297,6 +311,7 @@ impl platform::Driver for AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, raw_dev, raw_stats, stats, @@ -328,6 +343,43 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } + + /// The M3 MacBook Air (T8122): the M3 runtime, started only when its identity gate, the + /// backend selection and the start experiment (`asahi.t8122_start=1`) admit it; otherwise + /// the probe is refused as without the runtime. + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + fn probe_t8122<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + let soc = crate::m3::t8122_admitted(pdev)?; + // SAFETY: The admitted M3 GPU's UAT outputs 42-bit physical addresses. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; + let gpu = backend.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); + // SAFETY: The registration is stored and unregistered by platform data destruction, + // which drops the runtime with it. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + backend.stop(); + return Err(error); + } + }; + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: Some(backend), raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -371,6 +423,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: Some(gpu), + m3: None, raw_dev, raw_stats, stats: None, @@ -390,5 +443,9 @@ impl Drop for AsahiPlatformDriverData<'_> { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = self.m3.as_ref() { + m3.stop(); + } } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 608864949c976..242f6cdcd3f83 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -233,6 +233,9 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// Largest size this file has asked GET_PARAMS for the global parameters with + /// ([`gpu::Gpu::admit_vm`]). + params_size: core::sync::atomic::AtomicU64, } /// Convenience type alias for our DRM `File` type. @@ -286,6 +289,7 @@ impl File { (*slot).id = id; (*slot).user_range = user_range; + addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) } @@ -386,6 +390,9 @@ impl File { core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; + file.inner() + .params_size + .fetch_max(data.size, core::sync::atomic::Ordering::Relaxed); Ok(0) } @@ -454,6 +461,13 @@ impl File { // Every other runtime here reports DRM_ASAHI_GPU_HAL_LEGACY. return Err(EINVAL); } + reg_data.gpu.admit_vm( + device.as_ref().as_ref(), + file.inner() + .params_size + .load(core::sync::atomic::Ordering::Relaxed), + data.flags, + )?; let kernel_range = data.kernel_start..data.kernel_end; let user_range = file.inner().user_range.clone(); diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 46c05699d7722..a740ad3bb945d 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -49,6 +49,11 @@ impl F32 { F32(u) } + /// Convert to the raw IEEE 754 bits. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + // Convert a `f32` value into an F32 // // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 75d3ecc30ffe5..324bc08194e53 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -68,10 +68,10 @@ mod pools; mod power; mod queue; mod recovery; -mod regs; +pub(crate) mod regs; mod slots; mod timestamp; -mod uat; +pub(crate) mod uat; mod vm; mod wire; diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs index d5435e9f1c7ea..05a1b390b3dff 100644 --- a/drivers/gpu/drm/asahi/g15/regs.rs +++ b/drivers/gpu/drm/asahi/g15/regs.rs @@ -152,6 +152,21 @@ impl Mapping { pub(crate) fn phys(&self) -> u64 { self.phys } + + /// Cleans the whole mapping to the point of coherency when it is the kernel's cacheable + /// mapping of reserved System RAM ([`Self::region`]); device and write-combined mappings + /// need nothing. + pub(crate) fn clean_poc(&self) { + if self.kind != Kind::WriteBack { + return; + } + let start = self.raw.addr(); + // SAFETY: `start..start + maxsize` is this live mapping; cleaning a cacheable range has + // no other effect. + unsafe { + bindings::dcache_clean_poc(start as _, (start + self.raw.size()) as _); + } + } } impl Drop for Mapping { diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 854ff34e729a3..3855bd0beb9cb 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -200,16 +200,41 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { + Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) + } + + /// Maps the UAT handoff regions at the given physical addresses: the handoff page, the + /// TTBAT, the firmware kernel root and the shared L2 page. The M3 T8122 runtime takes them + /// from its admitted reserved-memory regions. + pub(crate) fn from_regions(handoff: u64, ttbs: u64, root: u64, shared_l2: u64) -> Result { Ok(Self { - handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, - ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?, - root: Mapping::region(adt.tables.start, hw::PAGE as usize)?, - shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, + handoff: Mapping::region(handoff, hw::HANDOFF_SIZE as usize)?, + ttbs: Mapping::region(ttbs, hw::TTBS_SIZE as usize)?, + root: Mapping::region(root, hw::PAGE as usize)?, + shared_l2: Mapping::region(shared_l2, hw::SHARED_L2_SIZE as usize)?, image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, }) } + /// [`Self::sync_range`] over the whole host window, then [`Self::clean_tables`]. + pub(crate) fn sync_all(&mut self, uat: &mmu::Uat) -> Result { + self.sync_inner(uat, None)?; + self.clean_tables(); + Ok(()) + } + + /// Cleans the firmware-walked pages this window writes (the kernel root, the shared L2, + /// the TTBAT and the handoff page) to the point of coherency. Only a region that is + /// reserved System RAM, mapped cacheable, has anything to clean: the UAT walker does not + /// snoop the CPU caches. The M3 T8122 runtime calls it after [`Self::bootstrap`]. + pub(crate) fn clean_tables(&self) { + for map in [&self.root, &self.shared_l2, &self.ttbs, &self.handoff] { + map.clean_poc(); + } + mb(); + } + /// Cold-start check: all 128 TTBAT words are zero. pub(crate) fn ttbat_is_clear(&self) -> Result { let t = self.ttbs.io(); diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 90bb25451019e..b9283efb470a9 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -338,6 +338,12 @@ pub(crate) trait Gpu: Send + Sync { } /// A userspace VM handle was closed; release backend-only registry references. fn vm_closed(&self) {} + /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS + /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 + /// T8122 runtime admits any client. + fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { + Ok(()) + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs deleted file mode 100644 index 9c37cfa98dce9..0000000000000 --- a/drivers/gpu/drm/asahi/m3/agx_fault.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Immutable fault-only snapshots of owned M3 firmware memory. -//! -//! Records contain name[32], firmware VA, length, begin/end monotonic ns, -//! then exactly `length` bytes. No MMIO or application BOs are read. Reads -//! are sequential observations, not atomic with running firmware. - -use kernel::{prelude::*, time::{ClockSource, Monotonic}}; - -pub(crate) struct Dump { - bytes: KVVec, - count: u32, - limit: usize, -} - -impl Dump { - - /// M3 snapshots have no scheduler job ID yet and contain only bounded - /// driver-owned firmware configuration and channel storage. - pub(crate) fn new_m3() -> Result { - Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) - } - - fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { - // Reserve before firmware/job publication, while healthy. Record appends - // must never enter reclaim waiting on the GPU that just faulted. - let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; - bytes.resize(40, 0, GFP_KERNEL)?; - bytes[..8].copy_from_slice(magic); - bytes[16..24].copy_from_slice(&job.to_le_bytes()); - bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - Ok(Self { bytes, count: 0, limit }) - } - - /// Start the one-shot capture; construction may precede the fault by hours. - pub(crate) fn begin(&mut self, job: u64) { - self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); - self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - } - - pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, - read: impl FnOnce(&mut [u8]) -> Result) -> Result { - if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } - let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; - let start = self.bytes.len(); - let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; - if end > self.limit { return Err(E2BIG); } - // Capacity was reserved at construction. NOWAIT is defense in depth; - // the checked limit above guarantees resize cannot allocate here. - self.bytes.resize(end, 0, GFP_NOWAIT)?; - self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); - self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); - self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); - let begin = Monotonic::ktime_get() as u64; - if let Err(error) = read(&mut self.bytes[start+64..end]) { - self.bytes.truncate(start); - return Err(error); - } - let finish = Monotonic::ktime_get() as u64; - self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); - self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); - self.count = count; - Ok(()) - } - - pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { - let length = self.bytes.len(); - self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); - self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); - self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); - let format = "M3FWD001"; - let owned = KBox::new(self, GFP_NOWAIT)?; - kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, - kernel::devcoredump::DEFAULT_TIMEOUT); - dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); - Ok(()) - } -} - -impl kernel::devcoredump::DevCoreDump for Dump { - fn read(&self, output: &mut [u8], offset: usize) -> Result { - if offset >= self.bytes.len() { return Ok(0); } - let length = output.len().min(self.bytes.len()-offset); - output[..length].copy_from_slice(&self.bytes[offset..offset+length]); - Ok(length) - } -} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs index a876cad1ec283..785b7f2314c8f 100644 --- a/drivers/gpu/drm/asahi/m3/g15_raw.rs +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -2,8 +2,8 @@ //! GPU initialization / global structures -use super::channels; -use super::types::*; +use crate::fw::channels; +use crate::fw::types::*; use crate::{ default_zeroed, gem, @@ -1158,7 +1158,9 @@ pub(crate) mod raw { pub(crate) event: ChannelRing, pub(crate) fw_log: ChannelRing, pub(crate) ktrace: ChannelRing, - pub(crate) stats: ChannelRing, + // The G15 stats record has the 13.x size (STATS_SZ for V >= V13_0B4); 7.2 generates + // only the 13.x instances. + pub(crate) stats: ChannelRing, /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 @@ -1548,7 +1550,7 @@ impl GpuStruct for InitDataG15V14_8_3 { // G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. mod g15_layout { - use super::channels; + use crate::fw::channels; use super::raw::*; use core::mem::{offset_of, size_of}; use kernel::static_assert; @@ -1602,7 +1604,7 @@ mod g15_layout { static_assert!(size_of::() == 0xe00); static_assert!(size_of::() == 0x18); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); - static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3::m3_globals_debug::OFFSET); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); @@ -1778,23 +1780,5 @@ mod g15_layout { == 0xc30 ); - // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). - // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these - // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X - // sizes. - static_assert!(size_of::>() == 0x6bc0); - static_assert!(size_of::>() == 0xb3b0); - static_assert!(size_of::() == 0x12394); - static_assert!(size_of::() == 0x1715c); - static_assert!(size_of::() == 0x421c); - static_assert!(size_of::() == 0x6c34); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xb4); - static_assert!(size_of::() == 0x30); - static_assert!(size_of::() == 0x1124); - static_assert!(size_of::() == 0x1a34); static_assert!(size_of::>() == 0x30); } diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs index c90156bd1244b..a774f5de6573f 100644 --- a/drivers/gpu/drm/asahi/m3/hw.rs +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -5,21 +5,18 @@ //! This module contains the definitions used to store per-GPU and per-SoC configuration data. use crate::fw::types::*; -pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +pub(crate) use crate::m3::identity::{GpuGen, GpuHalGeneration, GpuVariant}; use kernel::c_str; use kernel::prelude::*; const MAX_POWERZONES: usize = 5; +/// G15 SRAM rail floor for a device tree without `apple,min-sram-microvolt`, in microvolts: the +/// J516S runtime ADT perf-states-sram table floors SRAM at 810 mV (7.1.12 `hw::t6030`). +const G15_MIN_SRAM_MICROVOLT: u32 = 810_000; + pub(crate) mod agx3; -pub(crate) mod t600x; -pub(crate) mod t602x; -pub(crate) mod t6030; -pub(crate) mod t6031; -pub(crate) mod t8103; -pub(crate) mod t8112; pub(crate) mod t8122; -pub(crate) mod t8140; /// GPU revision enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] @@ -559,6 +556,9 @@ impl PwrConfig { return Err(EINVAL); } + // Kept as on the 7.1.12 kernel this runtime was qualified with: it steps by 3 over the + // zone count, which reads the same zones as one zone per triple for up to one zone and + // fewer beyond. (7.2's shared copy reads one zone per triple.) let pz_count = pz_data.len() / 3; let mut power_zones = KVec::new(); for i in (0..pz_count).step_by(3) { @@ -610,7 +610,7 @@ impl PwrConfig { min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. // Tolerate a DT without the property on G15 only. - prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + prop!("apple,min-sram-microvolt", G15_MIN_SRAM_MICROVOLT) } else { prop!("apple,min-sram-microvolt") }, diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs index cb0c4fdbfa012..f5e6ad607ec25 100644 --- a/drivers/gpu/drm/asahi/m3/hw/agx3.rs +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -2,7 +2,7 @@ #[cfg(not(test))] -use crate::identity::{GpuGen, GpuVariant}; +use crate::m3::identity::{GpuGen, GpuVariant}; #[cfg(test)] #[path = "../identity.rs"] mod identity; diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index dfeffd900eb12..2348d075a731c 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -12,21 +12,16 @@ //! derived from observing the InitData structures that macOS generates. use crate::f32; -use crate::fw::initdata::*; +use crate::m3::g15_raw::*; use crate::fw::types::*; -use crate::module_parameters; use crate::{ - driver::AsahiDevice, - gem, - gpu, - hw, + m3::hw, mmu, // }; use kernel::error::{ Error, Result, // }; -use kernel::macros::versions; use kernel::prelude::*; use kernel::try_init; @@ -35,6 +30,10 @@ use ::pin_init::Init; /// Builder helper for the global GPU InitData. +/// The "unknown page" of the 39-bit user layout (7.1.12 `mmu::IOVA_UNK_PAGE`): two pages below +/// the top of the user range. +const IOVA_UNK_PAGE: u64 = (1 << 39) - 2 * 0x4000; + /// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live /// in Fender scratch SRAM without a backup/restore path. /// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the @@ -104,24 +103,6 @@ pub(crate) struct G15RuntimeHwDataB { } impl G15Options { - /// The options of the G15 manager backend: its validated defaults, the performance-state - /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. - fn manager(cfg: &'static hw::HwConfig) -> G15Options { - let g15 = cfg.gpu_gen == hw::GpuGen::G15; - G15Options { - layout: G15Layout::Manager, - split_pstates: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), - cap: g15.then(|| { - crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) - }), - timestamp_base: None, - reference_ppm: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), - runtime_hwdata_b: None, - per_cluster_voltages: false, - } - } /// The published performance-state tables for `pwr`. pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { @@ -318,43 +299,12 @@ impl PStateTables { } } -/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its -/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. -pub(crate) fn g15_manager_max_frequency_khz( - cfg: &'static hw::HwConfig, - pwr: &hw::PwrConfig, -) -> Option { - if cfg.gpu_gen != hw::GpuGen::G15 { - return None; - } - let tables = G15Options::manager(cfg).tables(pwr).ok()?; - Some(tables.max_frequency_khz(pwr)) -} -pub(crate) struct InitDataBuilderG15V14_8_3<'a> { - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - g15: G15Options, -} +/// The G15 InitData builders. The M3 runtime places the firmware objects itself, so only the +/// builders into plain memory are kept ([`Self::g15_contents`]). +pub(crate) struct InitDataBuilderG15V14_8_3<'a>(core::marker::PhantomData<&'a ()>); impl<'a> InitDataBuilderG15V14_8_3<'a> { - /// Create a new InitData builder - pub(crate) fn new( - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> InitDataBuilderG15V14_8_3<'a> { - InitDataBuilderG15V14_8_3 { - dev, - alloc, - cfg, - dyncfg, - g15: G15Options::manager(cfg), - } - } /// Create an unknown T81xx-specific data structure. fn t81xx_data( @@ -591,47 +541,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { })) } - /// Create the HwDataA structure. This mostly contains power-related configuration. - fn hwdata_a(&mut self) -> Result> { - let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - if !dyncfg.hw_data_b.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataAG15V14_8_3 as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_data_a.len() { - matches = false; - dev_err!( - dev.as_ref(), - "!!! Hwdata A size mismatch: {} {}", - sla.len(), - dyncfg.hw_data_a.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { - matches = false; - dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(dev.as_ref(), "!!! Hwdata A match"); - } - } - } - Ok(()) - }) - }) - } /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device @@ -698,30 +607,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_429c[0] = 1; } - // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any - // (debug aid, same as the G13/G14 builders do inline). - fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { - if blob.is_empty() { - return; - } - // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. - let sla = unsafe { - core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) - }; - if sla.len() != blob.len() { - dev_err!( - dev.as_ref(), - "!!! {} size mismatch: {} {}", - what, - sla.len(), - blob.len() - ); - } - match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { - Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), - None => dev_info!(dev.as_ref(), "!!! {} match", what), - } - } // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose // source is not known yet are left 0 and marked TODO. @@ -737,7 +622,8 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { let runtime_hwdata_b = g15.runtime_hwdata_b; let timestamp_base = match g15.timestamp_base { Some(base) => base, - None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // The runtime always names its timestamp arena. + None => return Err(EINVAL), }; Ok(try_init!(raw::HwDataBG15V14_8_3 { // VA-layout words, +0x0/+0x8 @@ -750,7 +636,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its // 42-bit user layout; the manager keeps a 39-bit user range and maps the page // at IOVA_UNK_PAGE, so it passes that. - unk_30: U64(mmu::IOVA_UNK_PAGE), + unk_30: U64(IOVA_UNK_PAGE), // FW VA of the timestamp area, +0x28 timestamp_area_base: U64(timestamp_base), // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) @@ -960,20 +846,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_1818 = words.unk_1818; } - // Create the G15 (14.8.3) HwDataB structure. - fn hwdata_b(&mut self) -> Result> { - let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); - Ok(()) - }) - }) - } // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields // with a known init value are set; the rest start at 0. @@ -1102,123 +974,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { Ok(()) } - // Create the G15 (14.8.3) Globals structure. - fn globals(&mut self) -> Result> { - let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); - Ok(()) - }) - }) - } - // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by - // the GPU manager. - fn runtime_pointers(&mut self) -> Result> { - let hwa = self.hwdata_a()?; - let hwb = self.hwdata_b()?; - let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); - let rp_2d0 = self.g15.runtime_pointers_2d0(); - - let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; - buffer_mgr_ctl.vmap()?.memset(0); - - GpuObject::new_init_prealloc( - self.alloc.private.alloc_object()?, - |_ptr| { - let alloc = &mut *self.alloc; - try_init!(RuntimePointersG15V14_8_3 { - // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 - // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. - stats <- { - let alloc = &mut *alloc; - try_init!(StatsG15V14_8_3 { - vtx: alloc.private.new_default::()?, - frag: alloc.private.new_init( - pin_init::init_zeroed::(), - |_inner, _ptr| { - try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { - total_cmds: 0, - unk_4: 0, - stats: Default::default(), - }) - } - )?, - comp: alloc.private.new_default::()?, - }) - }, - - hwdata_a: hwa, - hwdata_b: hwb, - - // BRN (erratum) table. - // TODO: size and contents unknown; one zeroed page (no errata). - brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, - // E7: 0x88 bytes, zeroed - unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, - // E5: 0x60 bytes, zeroed - unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, - // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). - // TODO: size/format unknown; one zeroed page. The firmware takes a - // GPU VA and a FW VA; both point at this kernel mapping for now. - uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, - - buffer_mgr_ctl, - buffer_mgr_ctl_low_mapping: None, - buffer_mgr_ctl_high_mapping: None, - }) - }, - |inner, _ptr| { - try_init!(raw::RuntimePointersG15V14_8_3 { - hwdata_b: inner.hwdata_b.gpu_pointer(), - brn_table: inner.brn_table.gpu_pointer(), - unkptr_e7: inner.unkptr_e7.gpu_pointer(), - - pipes: Default::default(), - device_control: Default::default(), - event: Default::default(), - fw_log: Default::default(), - ktrace: Default::default(), - stats: Default::default(), - fwlog_buf: None, - __pad_200: Default::default(), - // FWLog enable: the firmware writes no log records while this is 0. - // Opt-in via asahi.g15_debug bit 43. - unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, - - stats_vtx: inner.stats.vtx.gpu_pointer(), - stats_frag: inner.stats.frag.gpu_pointer(), - stats_comp: inner.stats.comp.gpu_pointer(), - unkptr_e5: inner.unkptr_e5.gpu_pointer(), - __pad_254: Default::default(), - // Optional pointer, left 0 - unkptr_2a8: U64(0), - buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), - buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), - uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), - uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), - // TODO: board configuration words; +0x2d4 left 0. - unk_2d0: rp_2d0, - unk_2d4: 0, - __pad_2d8: Default::default(), - // +0x3b0: 0xff, then 0x90 zero bytes - cswitch_hist_ctl: 0xff, - unk_3b1: Default::default(), - hwdata_a: inner.hwdata_a.gpu_pointer(), - // TODO: 16 board configuration bytes; left 0. - unk_449: Default::default(), - __pad_459: Default::default(), - unk_46d: Default::default(), - __pad_48d: Default::default(), - }) - }, - ) - } // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle @@ -1230,13 +986,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { } } - // Create the G15 P0 status block. - fn status_block(&mut self) -> Result> { - let g15 = self.g15; - let mut block = self.alloc.shared.new_default::()?; - block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); - Ok(block) - } // Build the G15 P1 power-controller block without allocating it. The fields keep the order // and the formulas of the G13 13.5 Globals power fields (inferred). @@ -1306,18 +1055,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { }) } - // Create the G15 P1 power-controller block. - fn power_ctl_block(&mut self) -> Result> { - let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); - self.alloc - .shared - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) - } - // Create the G15 P2 debug block. ktrace stays disabled, as on G13. - fn debug_block(&mut self) -> Result> { - self.alloc.shared.new_default::() - } /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. fn uat_level_info( @@ -1349,66 +1087,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { ] } - /// Build the top-level InitData object. - #[inline(never)] - pub(crate) fn build(&mut self) -> Result>> { - { - let pwr = &self.dyncfg.pwr; - let tables = self.g15.tables(pwr)?; - dev_info!( - self.dev.as_ref(), - "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", - self.g15.cap.unwrap_or(0), - if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, - tables.max, - tables.len - 1, - tables.max_frequency_khz(pwr) / 1000 - ); - } - let runtime_pointers = self.runtime_pointers()?; - let globals = self.globals()?; - // 14.x: FwStatus lives inside the P0 status block - let fw_status = self.status_block()?; - let power_ctl_block = self.power_ctl_block()?; - let debug_block = self.debug_block()?; - let shared_ro = &mut self.alloc.shared_ro; - - let obj = self.alloc.private.new_init( - try_init!(InitDataG15V14_8_3 { - unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, - runtime_pointers, - globals, - fw_status, - power_ctl_block, - debug_block, - }), - |inner, _ptr| { - let cfg = &self.cfg; - try_init!(raw::InitDataG15V14_8_3 { - ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), - unk_buf: inner.unk_buf.gpu_pointer(), - unk_8: 0, - unk_c: 0, - runtime_pointers: inner.runtime_pointers.gpu_pointer(), - globals: inner.globals.gpu_pointer(), - // InitData +0x28/+0x2c: u32 0, u32 1 - unk_28: 0, - unk_2c: 1, - uat_page_size: 0x4000, - uat_page_bits: 14, - uat_num_levels: 3, - uat_level_info: Array::new(Self::uat_levels(cfg)), - // TODO: pmap descriptor +0x7c, meaning unknown; left 0. - unk_94: 0, - __pad_98: Default::default(), - debug_block: inner.debug_block.gpu_pointer(), - status_block: inner.fw_status.gpu_pointer(), - power_ctl_block: inner.power_ctl_block.gpu_pointer(), - }) - }, - )?; - Ok(KBox::new(obj, GFP_KERNEL)?) - } } /// G15 InitData contents built into plain memory, for a backend that places the firmware objects diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 2fa384c910688..1893c113cf2a7 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -13,18 +13,18 @@ use kernel::{ use crate::{ float::F32, - fw::initdata::raw, - hw, initdata, - initdata::{G15Layout, G15Options}, - m3_config::Records, - m3_firmware::Firmware, - m3_init_layout as init, - m3_init_storage as storage, - m3_memory::Buffer, - m3_params::{self, G15Debug, InitDataParam, ThermalMode}, - m3_soc::{IoMapping, PstateTable, Soc}, - t8122_hwdata, - t8122_start::Experiment, + m3::g15_raw::raw, + m3::hw, m3::initdata, + m3::initdata::{G15Layout, G15Options}, + m3::m3_config::Records, + m3::m3_firmware::Firmware, + m3::m3_init_layout as init, + m3::m3_init_storage as storage, + m3::m3_memory::Buffer, + m3::m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3::m3_soc::{IoMapping, PstateTable, Soc}, + m3::t8122_hwdata, + m3::t8122_start::Experiment, }; // The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage @@ -275,7 +275,7 @@ pub(crate) struct Contents { /// map read-write) except in the T8122 start experiment. pub(crate) read_only_slots: u32, /// The HwData object's placement and mapping (`Soc::hwdata_object`). - pub(crate) hwdata_object: Option, + pub(crate) hwdata_object: Option, } fn read_u32(bytes: &[u8], offset: usize) -> Result { @@ -296,36 +296,27 @@ impl Contents { /// any GPU register is touched. /// /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 - /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an - /// armed T6031 (`t6031_start`). + /// only; it then stands in for the SoC table's missing ones. pub(crate) fn select( - pdev: &platform::Device, + pdev: &platform::Device>, firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let dev = pdev.as_ref(); let param = m3_params::initdata_param(); - check_version(dev, firmware, experiment, t6031)?; + check_version(dev, firmware, experiment)?; let iomaps = match experiment { Some(e) => e.iomaps(), - None => match t6031 { - Some(e) => e.iomaps(), - None => soc.iomaps.ok_or(ENODEV)?, - }, + None => soc.iomaps.ok_or(ENODEV)?, }; storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; - let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let images = build_images(dev, firmware, soc, experiment, iomaps)?; let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; - let pstates = PstatePolicy::new( - dev, - hwdata, - experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), - )?; + let pstates = PstatePolicy::new(dev, hwdata, experiment.map(|e| e.pstate_cap()))?; let table = match soc.pstates { PstateTable::Fixed { states, top_mhz } => (states, top_mhz), - PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + PstateTable::DeviceTree => crate::m3::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { dev_err!( dev, "M3: the device tree's operating points give no performance-state table\n" @@ -346,7 +337,7 @@ impl Contents { return Err(EINVAL); } // The published states never go above the device tree's highest operating point. - if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if let Some(dt_khz) = crate::m3::m3_board::max_frequency_khz(pdev) { if 1000 * pstates.max_mhz > dt_khz { dev_err!( dev, @@ -377,14 +368,8 @@ impl Contents { pstates.max, pstates.max_mhz ); - let read_only_slots = experiment - .map(|e| e.read_only_slots()) - .or_else(|| t6031.map(|e| e.read_only_slots())) - .unwrap_or(0); - let hwdata_object = match t6031 { - Some(e) => e.hwdata_object(), - None => soc.hwdata_object, - }; + let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0); + let hwdata_object = soc.hwdata_object; Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) } @@ -435,7 +420,6 @@ fn check_version( dev: &device::Device, firmware: &Firmware, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let version = constructed_version()?; if version != firmware.initdata_magic && experiment.is_some() { @@ -447,15 +431,6 @@ fn check_version( ); return Ok(()); } - if version != firmware.initdata_magic && t6031.is_some() { - dev_warn!( - dev, - "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", - firmware.initdata_magic, - version - ); - return Ok(()); - } if version != firmware.initdata_magic { dev_err!( dev, @@ -572,7 +547,6 @@ fn fill_io_mappings( dev: &device::Device, cfg: &'static hw::HwConfig, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, mappings: &[IoMapping], iomaps: &[storage::IoMap], hwdata: &mut [u8], @@ -585,10 +559,9 @@ fn fill_io_mappings( let virt = entry + offset_of!(raw::IOMapping, virt_addr); // The same register block as the manager's table. let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { - let base = match (experiment, t6031) { - (Some(e), _) => e.io_block(slot, m.base as u64), - (None, Some(e)) => e.io_block(slot, m.base as u64), - (None, None) => m.base as u64, + let base = match experiment { + Some(e) => e.io_block(slot, m.base as u64), + None => m.base as u64, }; base & !0x3fff == phys & !0x3fff && m.writable == writable }); @@ -720,7 +693,7 @@ fn whole(v: F32) -> u32 { /// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported /// them. Returns them when the boot loader's switch is set. With the switch set and no two /// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. -fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { +fn leak_fuse(dev: &device::Device, lf: &crate::m3::m3_soc::LeakFuse) -> Result> { let chosen = kernel::of::chosen(); let switch = chosen .as_ref() @@ -790,29 +763,19 @@ fn build_images( firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, iomaps: &[storage::IoMap], ) -> Result>>> { check_layout().inspect_err(|_| { dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") })?; - let cfg: &'static hw::HwConfig = match t6031 { - Some(e) => e.hwcfg(), - None => soc.hwcfg.ok_or(ENODEV)?, - }; + let cfg: &'static hw::HwConfig = soc.hwcfg.ok_or(ENODEV)?; let io_mappings = match experiment { Some(e) => e.io_mappings(), - None => match t6031 { - Some(e) => e.io_mappings(), - None => soc.io_mappings.ok_or(ENODEV)?, - }, + None => soc.io_mappings.ok_or(ENODEV)?, }; let runtime_hwdata_b = match experiment { Some(e) => e.hwdata_b(), - None => match t6031 { - Some(e) => e.hwdata_b(), - None => soc.hwdata_b.ok_or(ENODEV)?, - }, + None => soc.hwdata_b.ok_or(ENODEV)?, }; let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) @@ -841,10 +804,7 @@ fn build_images( } // A SoC whose power model is still a stand-in caps the power target: every operating point's // power is scaled by the same factor, so the relative powers stay the boot loader's. - let power_cap_mw = match t6031 { - Some(e) => Some(e.power_cap_mw()), - None => soc.power_target_cap_mw, - }; + let power_cap_mw = soc.power_target_cap_mw; if let Some(cap) = power_cap_mw { let max = pwr.max_power_mw; if max > cap { @@ -863,7 +823,7 @@ fn build_images( let node = dev.of_node().ok_or(ENODEV)?; let dyncfg = hw::DynConfig { uat_ttb_base: firmware.resources.regions[0].base, - id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + id: soc_identity(soc, cfg, None), pwr, firmware_version: node .get_property::>(c_str!("apple,firmware-version")) @@ -877,7 +837,7 @@ fn build_images( split_pstates: true, // The cap is applied to the uploaded objects, for either source (`publish_pstates`). cap: None, - timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + timestamp_base: Some(crate::m3::agx_memory::TIMESTAMP_RANGE.start), reference_ppm: true, runtime_hwdata_b: Some(runtime_hwdata_b), per_cluster_voltages: soc.per_cluster_voltages, @@ -922,12 +882,6 @@ fn build_images( write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); - } else if let Some(e) = t6031 { - type B = raw::HwDataBG15V14_8_3; - let (a, b) = e.unit_masks(); - write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; - write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; - dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); } write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; // The system counter value at InitData creation, the base of the firmware's first power and @@ -953,14 +907,6 @@ fn build_images( m.fast_die, m.alarm ); - } else if let Some(e) = t6031 { - let (fast, alarm) = e.mtr_masks(dev)?; - type A = raw::HwDataAG15V14_8_3; - for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { - write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; - } - write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; - dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); } // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first // table holds the core coefficient, from the builder). @@ -969,23 +915,15 @@ fn build_images( let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; } - // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies - // the T8122 words only when asahi.t6031_fw_words=t8122. - let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { - Some(e) if e.fw_words() => ( - &crate::m3_soc::T8122_HWDATA_WORDS, - &crate::m3_soc::T8122_GLOBALS_WORDS, - ), - Some(_) => (&[], &[]), - None => (soc.hwdata_words, soc.globals_words), - }; + // The SoC's own values for words the shared builder writes otherwise. + let (hwdata_words, globals_words) = (soc.hwdata_words, soc.globals_words); for &(at, value) in hwdata_words { write(&mut hwdata, at, &value.to_le_bytes())?; } // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 - // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. - if crate::t8122_start::is_t8122(soc) - && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + // words of crate::m3::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::m3::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT) && cfg.gpu_gen == hw::GpuGen::G15 { if m3_params::t8122_hwdata_418c() { @@ -1006,7 +944,7 @@ fn build_images( } // On T8122, log and require the two HwDataB words the firmware's power management depends // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. - if core::ptr::eq(soc, &crate::m3_soc::T8122) { + if core::ptr::eq(soc, &crate::m3::m3_soc::T8122) { type B = raw::HwDataBG15V14_8_3; let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; @@ -1019,7 +957,7 @@ fn build_images( return Err(EINVAL); } } - fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + fill_io_mappings(dev, cfg, experiment, io_mappings, iomaps, &mut hwdata)?; images[HWDATA] = Some(hwdata); let mut globals = zeroed(GLOBALS)?; @@ -1047,7 +985,7 @@ fn build_images( if given { " (asahi.m3_ut_engagement)" } else { "" } ); } - if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + if crate::m3::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) .map_err(|_|EINVAL)? { dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); } @@ -1237,7 +1175,7 @@ pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { for off in GLOBALS_RUNTIME_CAP { globals.u32(off, 100 * cap)?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index 5ecfd820b4e4f..c44b2a42d99d5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Board facts for the M3 runtime, read from the hardware and the device tree instead of being -//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. +//! fixed to one board. The per-SoC facts are in [`crate::m3::m3_soc`]. use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; -use crate::m3_resources::{ +use crate::m3::m3_resources::{ reserved_resource, Region, Resources, // }; -use crate::m3_soc::Soc; +use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. fn board_is(compatible: &[u8]) -> bool { @@ -60,7 +60,7 @@ pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUS /// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node /// has no populated OPP table. -pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { +pub(crate) fn max_frequency_khz(pdev: &platform::Device>) -> Option { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut max_hz: u64 = 0; @@ -77,7 +77,7 @@ pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { /// Every cluster of a point must have the same voltage, except on a SoC with per-cluster /// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable /// OPP table. -pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { +pub(crate) fn opp_table_shape(pdev: &platform::Device>, soc: &Soc) -> Option<(u32, u32)> { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut points = KVec::new(); @@ -99,7 +99,7 @@ pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Optio }; points.push((hz, uv), GFP_KERNEL).ok()?; } - crate::t8122_admission::opp_table_shape(points) + crate::m3::t8122_admission::opp_table_shape(points) } /// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the @@ -340,7 +340,7 @@ pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option, soc: &Soc) -> Result { +pub(crate) fn admit(pdev: &platform::Device>, soc: &Soc) -> Result { let dev = pdev.as_ref(); let refuse = |what: &str, err: Error| { dev_info!(dev, "M3: not admitted: {}\n", what); @@ -460,14 +460,14 @@ pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result) -> bool { +pub(crate) fn has_pmp_link(pdev: &platform::Device>) -> bool { pdev.as_ref() .of_node() .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) } -pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; -use crate::m3_firmware::image_info; +pub(crate) use crate::m3::m3_firmware::{KnownImage, KNOWN_IMAGES_T8122}; +use crate::m3::m3_firmware::image_info; fn sha256(data: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; @@ -551,9 +551,8 @@ pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { /// board, a render node there would let a stock userspace driver submit work the board may not /// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. pub(crate) fn expose_render_node(soc: &Soc) -> bool { - match *crate::module_parameters::m3_expose.value() { - 0 => false, - v if v > 0 => true, - _ => runtime_validated_board(soc), + match crate::m3::m3_params::expose() { + Some(expose) => expose, + None => runtime_validated_board(soc), } } diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 6661222530a38..8fc7791e939ba 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -13,7 +13,6 @@ use core::sync::atomic::{ AtomicU32, - AtomicU64, Ordering, // }; @@ -23,8 +22,7 @@ use kernel::{ }; use crate::{ - driver::AsahiDevice, - agx_uapi::{ + m3::agx_uapi::{ GpuAccess, GpuAddressSpace, // }, @@ -38,28 +36,14 @@ const AGX3_PARAMS_SIZE: u64 = core::mem::size_of:: Self { - Self { - params_size: AtomicU64::new(0), - } - } - - /// Record a successful GET_PARAMS for the global parameters. - pub(crate) fn note_params(&self, size: u64) { - self.params_size.fetch_max(size, Ordering::Relaxed); - } - - /// Whether this file may create a VM. Only G15 devices are gated. - pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { - let size = self.params_size.load(Ordering::Relaxed); - if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the +/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. +pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { + { + // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`): + // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime. + if size >= AGX3_PARAMS_SIZE && flags == 0 { return Ok(()); } let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); @@ -74,7 +58,7 @@ impl ClientGate { "" }; dev_info!( - device.as_ref(), + device, "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", name, task.group_leader().pid(), @@ -200,7 +184,9 @@ pub(crate) fn check_compute_launches( /// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the /// pages cannot be unmapped and freed while they are cleaned and read. fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { - vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) + // 7.2's `Vm::read_user` cleans and invalidates each line to the point of coherency before + // the copy, under the VM execution lock. + vm.read_user(iova, out) } /// Clean and invalidate the data cache lines covering `start..start + len` to the point of diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index 08114c4d8cc8a..e06fcded9047e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -3,7 +3,7 @@ //! Completion work owned only by the M3 runtime. use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; -use crate::{driver, m3_drm::Shared}; +use crate::m3::{m3_drm::Shared, m3_rtkit::State}; #[pin_data] pub(crate) struct Completion { @@ -23,7 +23,7 @@ impl_has_work! { impl HasWork for Completion { self.work } } impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { - use crate::m3_runtime::Service; + use crate::m3::m3_runtime::Service; // Snapshots after a notification before sleeping for the next one: the completion // stamps can trail the firmware's event message. const TRAILING_POLLS: u32 = 4; @@ -59,8 +59,8 @@ impl WorkItem for Completion { } } } -pub(crate) fn queue(dev: &driver::AsahiDevice) { - if let Some(work) = dev.completion.as_ref() { +pub(crate) fn queue(state: &State) { + if let Some(work) = state.completion.as_ref() { let _ = workqueue::system_highpri().enqueue(work.clone()); } } diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index e7469d31b496f..eab8bf71f7bcd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -1,15 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_state_layout as state; -use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, - m3_pool_layout::GpuRegion, m3_init_layout::Region}; -use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_state_layout as state; +use crate::{m3::m3_compute_layout as command, m3::m3_compute_sequence as microsequence, + m3::m3_pool_layout::GpuRegion, m3::m3_init_layout::Region}; +use crate::{driver,mmu,m3::agx_memory::{self},m3::m3_memory::Buffer,m3::m3_compute_storage as storage}; +use crate::m3::m3_timeline as timeline; use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; -const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { +const UMA_POOL:crate::m3::m3_pool::Owners=crate::m3::m3_pool::Owners { backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, manager:storage::POOL,counter:storage::COUNTER,table_index:2, }; @@ -40,11 +40,14 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + /// Last: drops after the command aliases. + _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, - stats: Region, control: crate::agx_compute::Control, + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::m3::agx_compute::Control, registers: command::RegisterSet) -> Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; @@ -101,10 +104,10 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, - control: crate::agx_compute::Control, sequence: u64, slot: usize, + control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, register_set: command::RegisterSet) -> Result { let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); @@ -164,7 +167,7 @@ impl Compute { objects[PREEMPTION].fill(0xcc) } pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, - control: crate::agx_compute::Control) -> Result { + control: crate::m3::agx_compute::Control) -> Result { if !self.complete()? { return Err(EBUSY); } // All prior flushed stamps/events and engine idle were checked by // Runtime before entry. Keep pool/queue ownership, remap only client @@ -211,7 +214,7 @@ impl Compute { /// Append only while the previous batch is fully retired and this one /// remains unpublished. Firmware serializes each command, including its /// original completion flush, in this one compute queue. - pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + pub(crate) fn append(&mut self,control:crate::m3::agx_compute::Control)->Result { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { @@ -268,20 +271,20 @@ impl Compute { } agx_memory::publish();Ok(()) } - pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } - pub(crate) fn completion_dependency(&self)->Result { - Ok(crate::m3_sync_layout::CompletionDependency { + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3::m3_sync_layout::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, value:self.stamp(),event:2, }) } - pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat)->Result { if self.barriers.is_none() { self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); } @@ -290,17 +293,17 @@ impl Compute { /// Insert a dependency before the last appended, still unpublished CDM. /// Its command ordinal and timestamp slot are unchanged; only the ring /// gains an entry. The whole batch must retire before this storage is reused. - pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3::m3_sync_layout::CompletionDependency)->Result { if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { return Err(EINVAL); } let value=self.stamp(); let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; - let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; - crate::m3_sync_layout::Barrier { + let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; + crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, - self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, }.encode(&mut bytes).map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; @@ -333,7 +336,7 @@ impl Compute { pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } - pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { @@ -360,7 +363,7 @@ impl Compute { && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index 59277aaefb681..c4a44c4b4f3b0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419.140.12 configuration with named initialization references. use kernel::prelude::*; -use crate::m3_init_layout as init; -use crate::m3_init_storage as storage; +use crate::m3::m3_init_layout as init; +use crate::m3::m3_init_storage as storage; use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, FW_CONTROL_STATE,FW_CONTROL_RING}; -use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +use crate::{driver,m3::agx_memory::{self},m3::m3_memory::Buffer,mmu,pgtable::prot}; kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); @@ -49,12 +49,10 @@ impl Records { pub(crate) struct Config { objects: KVec, - #[cfg(CONFIG_DEV_COREDUMP)] - fault_reserve: Option, _iomaps: KVec, pub(crate) completed_events: u64, - pstates: crate::m3_adt_config::PstateWatch, - thermal: crate::m3_thermal::Governor, + pstates: crate::m3::m3_adt_config::PstateWatch, + thermal: crate::m3::m3_thermal::Governor, request_top: u32, request_value: u32, request_checked: bool, @@ -63,47 +61,9 @@ pub(crate) struct Config { request_observed: u32, } impl Config { - /// Snapshot owned RAM only. Firmware can still update these buffers; - /// per-record time intervals explicitly describe sequential observations. - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { - let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; - dump.begin(0); - let mut cause = [0u8; 16]; - cause[..4].copy_from_slice(&1u32.to_le_bytes()); - cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); - cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); - dump.record("host-first-error", 0, cause.len(), |out| { - out.copy_from_slice(&cause); Ok(()) - })?; - for (name, index) in [ - ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), - ("hardware", HARDWARE_DATA), ("globals", GLOBALS), - ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), - // The power-controller block, and the host-to-firmware device-control and - // firmware-control queues: their state words show whether the firmware read a - // queued control (read index) and what it was sent. - ("power", GLOBALS_POWER), - ("devctl-state", storage::DEVICE_CONTROL * 2), - ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), - ("fwctl-state", storage::FW_CONTROL_STATE), - ("fwctl-ring", storage::FW_CONTROL_RING), - ("event-state", storage::EVENT * 2), - ("event-ring", storage::EVENT * 2 + 1), - ("fwlog-state", storage::FIRMWARE_LOG * 2), - ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), - ("fwlog-payload", FWLOG_PAYLOAD), - ("trace-state", storage::TRACE * 2), - ("trace-ring", storage::TRACE * 2 + 1), - ] { - let buffer = &mut self.objects[index]; - dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; - } - dump.publish(dev.as_ref()) - } - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, - contents: &crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware, + contents: &crate::m3::m3_adt_config::Contents) -> Result { contents.check_for_upload()?; let mut objects = KVec::new(); let mut iomaps = KVec::new(); @@ -125,8 +85,8 @@ impl Config { for io in contents.iomaps { // Read-write unless the contents say otherwise (only the T8122 start experiment). let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} - else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; - let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + else {crate::m3::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io_m3(io.address,io.physical.try_into()?,io.size,protection)?; let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; let field=io.pointer_field().map_err(|_|EINVAL)?; objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; @@ -139,7 +99,7 @@ impl Config { // The root record carries the G15 14.8.3 InitData version, which is every T6030 // image's. Only the T8122 start experiment gives the firmware another one // (`asahi.t8122_initdata_version`). - if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + if firmware.initdata_magic!=crate::m3::m3_firmware::G15_V14_8_3_INITDATA { objects[storage::ROOT].u64(0,firmware.initdata_magic)?; } // Vertex/tessellation loops need not retire a primitive at every @@ -171,26 +131,17 @@ impl Config { return Err(EINVAL); } let (hwdata, globals) = objects.split_at_mut(GLOBALS); - crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + crate::m3::m3_adt_config::publish_pstates(dev.as_ref().as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; // Firmware ktrace is for explicit detailed diagnostics. Normal games // still drain error/events, but need not generate trace records for // every firmware operation on the command-processing core. objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( crate::debug::DebugFlags::SubmitTiming)))?; - crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); - let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; - #[cfg(CONFIG_DEV_COREDUMP)] - let fault_reserve = match crate::agx_fault::Dump::new_m3() { - Ok(dump) => Some(dump), - Err(error) => { - dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); - None - } - }; + let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; Ok(Self { objects, _iomaps: iomaps, completed_events: 0, - #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, - pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, request_check_at:None,request_writes:0,request_observed:0 }) } @@ -203,22 +154,22 @@ impl Config { objects[CONTROL_REGION].write(0, &records.control) } pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } - pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { - let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", - u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + u8::from(crate::m3::m3_params::globals_debug_off()),value,crate::m3::m3_params::consistent_perf()); Ok(()) } pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { - let Some(value)=crate::m3_consistent_perf::request(chip, - crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + let Some(value)=crate::m3::m3_consistent_perf::request(chip, + crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; if value==0 && self.request_value==0 {return Ok(());} let now=kernel::time::Instant::::now(); let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); - if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + if !crate::m3::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { return Ok(()); } - type P=crate::fw::initdata::raw::PowerCtlBlock; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; let at=core::mem::offset_of!(P,consistent_perf_state); if !self.request_checked { let mut flag=[0]; @@ -240,14 +191,14 @@ impl Config { /// Move the runtime cap by temperature (the thermal limit), then check the performance /// state the firmware reports against the ceiling and the runtime cap. Called after boot and /// after every job, by the job thread. - pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result { let now = kernel::time::Instant::::now(); - if let Some(cap) = self.thermal.update(dev.as_ref(), now) { - crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) { + crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; } let allowed = self.thermal.allowed(now); - let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; - self.thermal.note_check(dev.as_ref(), now, over, peak)?; + let (over, peak) = self.pstates.check(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref().as_ref(), now, over, peak)?; self.thermal.busy(now); Ok(()) } @@ -256,14 +207,14 @@ impl Config { init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) } - pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, - device: &crate::m3_device::Device, + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice, + device: &crate::m3::m3_device::Device, sequence: u32) -> Result { let cap = self.thermal.allowed(kernel::time::Instant::::now()); - self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], device.pstate_register()?, cap, sequence)?; - type P=crate::fw::initdata::raw::PowerCtlBlock; - type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; + type G=crate::m3::g15_raw::raw::GlobalsG15V14_8_3; let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; let power_if=core::mem::offset_of!(G,power_if_target); @@ -272,13 +223,13 @@ impl Config { let mut override_active=[0]; self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", - sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + sequence,crate::m3::m3_params::consistent_perf(),request,cap_word,target1,target2, self.request_writes,self.request_observed,override_active[0]); Ok(()) } pub(crate) fn render_pb(&mut self) -> Result { - self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?) } pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { if kind>2 {return Err(EINVAL);} @@ -316,7 +267,7 @@ impl Config { Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) } - pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result { let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; @@ -345,7 +296,7 @@ impl Config { /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. /// Nothing is acknowledged, cleared or resumed here. - pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result { let control=&mut self.objects[CONTROL_REGION]; let halt_count=control.read_u64(0x4580)?; let halted=control.read_u32(0x4590)?; @@ -360,7 +311,7 @@ impl Config { /// Read-only: log the event ring entries from the host's read index to the firmware's write /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the /// completion count is not changed. - pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result { const SIZE: usize = 0x38; const COUNT: usize = 256; let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; @@ -395,7 +346,7 @@ impl Config { pub(crate) fn control_done(&mut self, next: u32) -> Result { Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) } - pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Event, trace, stats and six independent firmware log rings. for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 1c2f0b1e8866d..c43017a8933c0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -12,6 +12,7 @@ use kernel::{ io::{ mem::IoMem, Io, + IoBase, }, platform, prelude::*, @@ -19,16 +20,16 @@ use kernel::{ time::{delay::fsleep, Delta, Instant, Monotonic}, }; -use crate::m3_firmware::Firmware; -use crate::m3_soc::Soc; +use crate::m3::m3_firmware::Firmware; +use crate::m3::m3_soc::Soc; const ASC_CPU_CONTROL: usize = 0x44; const ASC_CPU_RUN: u32 = 1 << 4; pub(crate) struct Device { dev: ARef, - asc: Pin>>>, - sgx: Pin>>, + asc: Devres>, + sgx: Devres>, firmware: Firmware, core_mask: u64, soc: &'static Soc, @@ -37,34 +38,28 @@ pub(crate) struct Device { impl Device { /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides - /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. - pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>) -> Result { + /// the SGX setup write, which the T8122 table lacks. + pub(crate) fn new(pdev: &platform::Device>, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::m3::t8122_start::Experiment>) -> Result { // Map ASC before taking a vote. The larger SGX aperture contains // this control window and the separate mailbox provider, so it is // mapped without a conflicting claim over those child resources. // Only the 16 KiB control window is used. Some device trees describe // the ASC block as a larger range that also covers the mailbox, which // the mailbox driver has already claimed, so do not claim the range. - let asc = KBox::pin_init( - pdev.io_request_by_name(c_str!("asc")) - .ok_or(EINVAL)? - .iomap_sized::<0x4000>(), - GFP_KERNEL, - )?; + let asc = pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); - let sgx = KBox::pin_init( - pdev.io_request_by_name(c_str!("sgx")) - .ok_or(EINVAL)? - .iomap(), - GFP_KERNEL, - )?; + let sgx = pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); - let pmp = crate::m3_board::has_pmp_link(pdev); - let t8122 = crate::t8122_start::is_t8122(soc); - let t6031_soc = crate::t6031_start::is_t6031(soc); - if pmp && !t8122 && !t6031_soc { + let pmp = crate::m3::m3_board::has_pmp_link(pdev); + let t8122 = crate::m3::t8122_start::is_t8122(soc); + if pmp && !t8122 { dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); return Err(ENOTSUPP); } @@ -78,7 +73,7 @@ impl Device { } else { dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); } - if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { dev_err!( pdev.as_ref(), "M3 {}: ASC already running; refusing to take ownership\n", @@ -103,19 +98,15 @@ impl Device { // failed acquisition releases it only after the ASC-stopped check in Drop. device.power_vote = true; to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; - if t6031_soc { - dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); - } else { - dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); - } + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); } - let registers = device.sgx.access(pdev.as_ref())?; + let registers = device.sgx.access(pdev.as_ref())?.as_view(); let version = registers.try_read32(0xd04000)?; let counts = registers.try_read32(0xd04010)?; // The core-enable words: the second only on a SoC with more than 32 core slots. let mut core_mask = u64::from(registers.try_read32(0xe01500)?); - if crate::m3_board::core_slots(soc) > 32 { + if crate::m3::m3_board::core_slots(soc) > 32 { core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; } dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", @@ -124,7 +115,7 @@ impl Device { // fused core mask varies with SKU; absent cores must never be enabled. if version & soc.id.version_mask != soc.id.version || counts & soc.id.counts_mask != soc.id.counts - || !crate::m3_board::core_mask_valid(soc, core_mask) + || !crate::m3::m3_board::core_mask_valid(soc, core_mask) { return Err(ENODEV); } @@ -138,21 +129,7 @@ impl Device { } e.sgx_setup() } - None => match t6031 { - Some(e) => { - match e.sgx_setup() { - Some((offset, value)) => dev_warn!( - pdev.as_ref(), - "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", - offset, - value - ), - None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), - } - e.sgx_setup() - } - None => Some(soc.sgx_setup.ok_or(ENODEV)?), - }, + None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { registers.try_write32(setup_value, setup_offset)?; @@ -173,7 +150,8 @@ impl Device { /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { if !busy && !faults {return Ok(());} - let sgx=self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard=self.sgx.try_access().ok_or(ENODEV)?; + let sgx=sgx_guard.as_view(); if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { return Err(EBUSY); } @@ -190,34 +168,16 @@ impl Device { if fault!=0 {return Err(EIO);} Ok(()) } - pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { - use kernel::dma::{Device as _, DmaMask}; - // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; - crate::mmu::check_handoff_guard()?; - dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); - for _ in 0..2 { - let drm: ARef = kernel::drm::Device::new( - pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; - if drm.gpu().is_ok() { return Err(EIO); } - let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; - object.vmap()?.memset(0); - drop(object); - drop(drm); - } - dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); - Ok(()) - } - /// The raw performance-state register (SGX+0xe01000). pub(crate) fn pstate_register(&self) -> Result { - self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + self.sgx.try_access().ok_or(ENODEV)?.as_view().try_read32(0xe01000) } /// Read-only snapshot. Do not select or acknowledge fault banks here; /// the firmware owns those registers until recovery is ported. pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = sgx_guard.as_view(); let pstate = sgx.try_read32(0xe01000)? & 0xf; if pstate == 0 { dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); @@ -240,10 +200,10 @@ impl Device { let service=sgx.try_read32(0xa010)?; let debug=sgx.try_read32(0xa000)?; let mut cores=u64::from(sgx.try_read32(0xe01500)?); - if crate::m3_board::core_slots(self.soc) > 32 { + if crate::m3::m3_board::core_slots(self.soc) > 32 { cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; } - for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + for core in 0..crate::m3::m3_board::core_slots(self.soc).min(64) { if cores & (1u64< &Firmware { &self.firmware } - pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { - if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + pub(crate) fn require_stopped(&self, pdev: &platform::Device>) -> Result { + if self.asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { return Err(EBUSY); } Ok(()) } - pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + pub(crate) fn start_asc(&self, pdev: &platform::Device>) -> Result { self.require_stopped(pdev)?; - let asc = self.asc.access(pdev.as_ref())?; + let asc = self.asc.access(pdev.as_ref())?.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); Ok(()) } pub(crate) fn stop_asc(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; + let asc_guard = self.asc.try_access().ok_or(ENODEV)?; + let asc = asc_guard.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); let start = Instant::::now(); while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { @@ -301,7 +262,7 @@ impl Drop for Device { } // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. match self.asc.try_access() { - Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + Some(asc) if asc.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} _ => { dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); return; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 25e65c6599084..2d93823e67d01 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 through the common Asahi GEM, GPUVM and render-node frontend. use core::ops::Range; -use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; -use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; +use core::any::Any; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}, uapi}; +use crate::{alloc, driver, gem, gpu, m3::hw, mmu, queue}; -pub(crate) type Shared = Arc>>; -struct ProgressView(Arc); +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); impl kernel::debugfs::Writer for ProgressView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); @@ -14,7 +15,7 @@ impl kernel::debugfs::Writer for ProgressView { } } // Separate endpoint preserves the strict completion-v1 ABI for old guards. -struct ActivityView(Arc); +struct ActivityView(Arc); impl kernel::debugfs::Writer for ActivityView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, epoch, healthy) = self.0.activity_snapshot(); @@ -23,7 +24,7 @@ impl kernel::debugfs::Writer for ActivityView { } } -struct TimingView(Arc); +struct TimingView(Arc); impl kernel::debugfs::Writer for TimingView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, _, _, healthy) = self.0.progress_snapshot(); @@ -33,7 +34,7 @@ impl kernel::debugfs::Writer for TimingView { // Root-only writable debugfs policy; no mutable module-parameter references. // Keep ownership independent of the runtime lock and remove the file on stop. -struct CompletionView(Arc); +struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); @@ -53,7 +54,7 @@ impl kernel::debugfs::Reader for CompletionView { } } -struct SubmitOverlapView(Arc); +struct SubmitOverlapView(Arc); impl kernel::debugfs::Writer for SubmitOverlapView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { writeln!(f, "{}", u8::from(self.0.cpu_overlap())) @@ -74,137 +75,179 @@ impl kernel::debugfs::Reader for SubmitOverlapView { } } -pub(crate) struct Registered { - registration: Pin>>>>, +/// The started T8122 runtime, as the shared DRM layer sees it ([`gpu::Gpu`]). It is created at +/// probe with the runtime it wraps and handed to the DRM registration, which owns it: dropping it +/// (unbind, or a failed registration) stops the runtime. The debugfs files share its lifetime. +pub(crate) struct Backend { shared: Shared, - health: Arc, + health: Arc, + ids: gpu::SequenceIDs, + scheduler: Arc>, + core_mask: u64, + max_frequency_khz: u32, + soc: &'static crate::m3::m3_soc::Soc, _progress: Pin>>, _activity: Pin>>, - _memory: Pin>>, + _memory: Pin>>, _timing: Pin>>, _completion: Pin>>, _overlap: Pin>>, } -impl Registered { - pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { - let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { - crate::t8122_start::not_admitted(pdev.as_ref(), soc); - crate::t6031_start::not_admitted(pdev.as_ref(), soc); + +impl Backend { + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the + /// caller registers afterwards. + pub(crate) fn start( + pdev: &platform::Device>, + drm: &driver::AsahiDrmDevice, + soc: &'static crate::m3::m3_soc::Soc, + ) -> Result> { + // The runtime only exists to back its render node: without one (`asahi.m3_expose=0`) + // nothing is started, rather than leaving a running firmware behind a failed probe. + if !crate::m3::m3_board::expose_render_node(soc) { + dev_info!(pdev.as_ref(), "M3: render node not registered on this board (asahi.m3_expose=1 registers it); GPU not started\n"); + return Err(ENODEV); + } + let resources = crate::m3::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, - // or an unarmed T8122 or T6031, needs a complete table. - let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; - let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() && t6031.is_none() { + // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 + // needs a complete table, which it does not have: it is refused here. + let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); - let t6031 = t6031.as_ref(); let refused = |stage: &'static str| { - move |e: &Error| { - crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); - crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); - } + move |e: &Error| crate::m3::t8122_start::refused(pdev.as_ref(), experiment, stage, *e) }; - let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment) .inspect_err(refused("the firmware identity"))?; dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); // InitData source and contents, before any GPU register access. - let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment) .inspect_err(refused("the InitData contents"))?; - let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + let device = crate::m3::m3_device::Device::new(pdev, firmware, soc, experiment) .inspect_err(refused("the GPU registers and identity"))?; - device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; let core_mask = device.core_mask(); let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); - let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + let mut runtime = crate::m3::m3_runtime::Runtime::new(pdev, drm, device, contents)?; runtime.boot(pdev)?; dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", - u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), - u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); - let drm = runtime.drm(); + u8::from(crate::m3::m3_params::unlocked_wait()), u8::from(crate::m3::m3_params::timeout_nohang()), + u8::from(crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault))); let health = runtime.health(); - // Health holds no device references. The read-only file owns an Arc - // and is removed with registration; teardown latches failure first. + let state = runtime.rtkit_state(); + // Health holds no device references. The read-only files own an Arc and are removed + // with the backend; teardown latches failure first. let directory = kernel::debugfs::Dir::new(c"asahi-m3"); let progress = KBox::pin_init(directory.read_only_file(c"progress", ProgressView(health.clone())), GFP_KERNEL)?; let activity = KBox::pin_init(directory.read_only_file(c"activity", ActivityView(health.clone())), GFP_KERNEL)?; let memory = KBox::pin_init(directory.read_only_file(c"memory", - crate::agx_memory_stats::View), GFP_KERNEL)?; + crate::m3::agx_memory_stats::View), GFP_KERNEL)?; let timing = KBox::pin_init(directory.read_only_file(c"timing", TimingView(health.clone())), GFP_KERNEL)?; let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", CompletionView(runtime.completion_wait())), GFP_KERNEL)?; let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; - let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; - if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + let scheduler = Arc::new(drm::sched::Scheduler::new(drm.as_ref().as_ref(), 4, 8, 0, 3000, + kernel::c_str!("asahi_m3_sched"))?, GFP_KERNEL)?; + let shared: Shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?; + // From here on every failure drops the backend, which stops the runtime. + let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(), + scheduler, core_mask, max_frequency_khz, soc, + _progress: progress, _activity: activity, _memory: memory, _timing: timing, + _completion: completion, _overlap: overlap }, GFP_KERNEL)?; + // The completion worker services this runtime from now on; the RTKit callbacks and + // the submission path queue it through the runtime's state. + if !state.completion.populate(completion_work) { return Err(EBUSY); } - let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; - let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; - let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), - core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; - if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } - if crate::m3_board::expose_render_node(soc) { - *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); - dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); - } else { - dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); - } - Ok(owner) + Ok(backend) } + + /// Stops the runtime: called on unbind, while the device's resources are still live, and + /// again (doing nothing) when the backend is dropped. pub(crate) fn stop(&self) { self.health.mark_failed(); - drop(self.registration.lock().take()); drop(self.shared.lock().take()); } } -impl Drop for Registered { fn drop(&mut self) { self.stop(); } } -struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, - scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, - soc: &'static crate::m3_soc::Soc } -impl DrmGpu for Backend { + +impl Drop for Backend { + fn drop(&mut self) { + self.stop(); + } +} + +impl gpu::Gpu for Backend { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } fn ids(&self) -> &gpu::SequenceIDs { &self.ids } fn is_crashed(&self) -> bool { !self.health.healthy() } - fn update_globals(&self) {} - - fn supports_vm_status(&self) -> bool { true } - fn supports_scheduled_queues(&self)->bool {true} - fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } - fn params(&self) -> Result { + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { let soc = self.soc; - let masks = crate::m3_board::core_masks(soc, self.core_mask); - Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, - gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, - chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, - num_cores_per_cluster: soc.cores_per_cluster, - core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, - gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, - max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) - } - fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } - fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + let masks = crate::m3::m3_board::core_masks(soc, self.core_mask); + params.gpu_generation = hw::GpuGen::G15 as u32; + params.gpu_variant = soc.gpu_variant as u32; + params.gpu_revision = soc.gpu_revision as u32; + params.chip_id = soc.chip_id; + params.num_dies = 1; + params.num_clusters_total = soc.clusters; + params.num_cores_per_cluster = soc.cores_per_cluster; + for (out, mask) in params.core_masks.iter_mut().zip(masks.iter()) { + *out = (*mask).into(); + } + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = 3; + params.gpu_hal_generation = hw::GpuHalGeneration::Legacy as u32; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// The features the 7.1.12 file layer reported for the M3 runtime: VM status (not its + /// mirror), scheduled queues, and the SoC's optional render and compute features. + fn feature_flags(&self) -> u64 { + let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; + if self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64; + } + if self.soc.features.compute_wide_visibility { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64; + } + flags + } fn base_clock_hz(&self) -> u32 { let value: u64; // SAFETY: read-only architectural counter frequency. unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; value as u32 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn tracked_submission(&self) -> bool { true } + /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). + fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { + crate::m3::m3_client::admit_vm(dev, params_size, flags) + } fn new_vm(&self, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64) -> Result> { - Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 9ae509ad959c4..0d2835174c677 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -use crate::m3_resources as agx_resources; +use crate::m3::m3_resources as agx_resources; /// A GPU firmware image the runtime can identify. #[derive(Debug)] @@ -30,7 +30,7 @@ pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ KnownImage { name: "J514S RTKit-2419.140.12", uuid: None, - stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + stkg_sha256: Some(crate::m3::m3_firmware::TEXT_SHA256), initdata_magic: Some(G15_V14_8_3_INITDATA), }, KnownImage { @@ -140,20 +140,6 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { version: "RTKit-2419.140.12.release", }; -/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 -/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither -/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); -/// an armed start searches the loaded text for the tag. The loaded image's version string is -/// not known ("unknown" stands for it). -pub(crate) const T6031_LAYOUT: Layout = Layout { - text_size: 0x64000, - data_size: 0x14_0000, - vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], - entropy_tag_offset: 0, - entropy_tag: *b"GKTS\x08\x00\x00\x00", - version: "unknown", -}; - pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; @@ -194,11 +180,10 @@ impl Firmware { #[cfg(not(test))] pub(crate) fn identify_loaded( - pdev: &kernel::platform::Device, - soc: &'static crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &'static crate::m3::m3_soc::Soc, resources: agx_resources::Resources, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, + experiment: Option<&crate::m3::t8122_start::Experiment>, ) -> kernel::error::Result { use kernel::{ bindings, c_str, @@ -220,21 +205,6 @@ pub(crate) fn identify_loaded( let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; let mut canonical = KVec::new(); canonical.extend_from_slice(bytes, GFP_KERNEL)?; - // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is - // a placeholder, so the T8122 image table must not see this image. - if let Some(experiment) = t6031 { - experiment.normalize_text(&mut canonical)?; - let mut digest = [0u8; 32]; - // SAFETY: the initialized private copy and distinct digest live for the synchronous - // SHA-256 call. Normalization never writes the loaded firmware. - unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; - experiment.admit_image(pdev.as_ref(), bytes, &digest)?; - return Ok(Firmware { - resources, - initdata_magic: experiment.initdata_version(), - layout, - }); - } if !normalize_boot_entropy(layout, &mut canonical) { return Err(ENODEV); } @@ -247,7 +217,7 @@ pub(crate) fn identify_loaded( Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], None => soc.images, }; - let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; let initdata_magic = match experiment { Some(experiment) => experiment.initdata_version(), None => image.initdata_magic.ok_or(ENODEV)?, @@ -393,21 +363,6 @@ mod tests { } } - #[test] - fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { - let layout = &T6031_LAYOUT; - assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); - assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); - assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); - let mut regions = [Region { base: 0, size: 0 }; 6]; - regions[4].size = layout.text_size; - regions[5].size = layout.data_size; - let resources = Resources { regions, firmware_vas: layout.vas }; - assert!(Firmware::matching_layout(layout, &resources)); - assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); - assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); - } - #[test] fn entropy_normalization_changes_only_the_eight_boot_bytes() { for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs index cbfb771fd0873..5e8018a802e70 100644 --- a/drivers/gpu/drm/asahi/m3/m3_memory.rs +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -8,6 +8,20 @@ enum Backing { Paged(gem::ObjectRef), } +/// Invalidates context 0's non-global translations when dropped. A job's command aliases live in +/// the context-0 lower tree, which has no bind slot on the adopted UAT (7.1.12 gave it the fixed +/// slot 0), so unmapping them invalidates nothing by itself. Declared first in a constructor and +/// last in the job, it drops after every alias. +pub(crate) struct LowerTlbi; + +impl Drop for LowerTlbi { + fn drop(&mut self) { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } +} + pub(crate) struct Buffer { mapping: mmu::KernelMapping, gpu_mapping: Option, @@ -17,18 +31,31 @@ pub(crate) struct Buffer { object: Backing, offset: usize, size: usize, - _coherent_allocation: Option, + _coherent_allocation: Option, +} + +impl Drop for Buffer { + fn drop(&mut self) { + // A GPU view may be a command alias in the context-0 lower tree, which has no bind slot + // to invalidate on unmap: drop it and invalidate context 0 before the backing below is + // freed (7.1.12 invalidated inside the unmap). + if self.gpu_mapping.take().is_some() { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } + } } impl Buffer { - pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, gpu_shared: bool) -> Result { Self::at_prot(dev,fw,gpu,address,size, if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, prot::PROT_GPU_SHARED_RW) } - pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at_prot(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, protection: prot::Prot, gpu_protection: prot::Prot) -> Result { let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; @@ -39,18 +66,18 @@ impl Buffer { // normal case here and is not reported; only a failed fallback is. // SAFETY: M3 runtime owns the bound platform device for the entire // diagnostic; all mappings retire before normal DMA allocation Drop. - let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL, kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; let pa:usize=object.dma_handle().try_into()?; - let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?} else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; - let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, - _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + _coherent_allocation:Some(crate::m3::agx_memory_stats::Allocation::coherent(allocated)?)}) } /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the /// old allocation mapped until the replacement has been fully allocated. - pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + pub(crate) fn tiler(dev:&driver::AsahiDrmDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} // Large render targets can require 128 MiB of tiler storage. Engines // and firmware address it through UAT, so physically contiguous DMA @@ -62,7 +89,7 @@ impl Buffer { mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; // Growth retains the cached GPU-only policy of the original TPC and // tilemap allocations; the firmware alias remains shared/uncached. - let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_protection=prot::Prot::from_pte(crate::m3::m3_pass_layout::Access::GpuCachedRw.pte()); let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, mmu::UAT_PGSZ as u64,gpu_protection,true)?); Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) @@ -73,7 +100,7 @@ impl Buffer { pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { let address=self.gpu_va()?&!0x3fff; match &mut self.object { - Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), } } @@ -82,7 +109,10 @@ impl Buffer { fn cpu_ptr(&mut self)->Result<*mut u8> { match &mut self.object { Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), - Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + Backing::Paged(object)=>{ + use kernel::io::IoBase; + Ok(object.vmap()?.as_view().as_ptr().cast::()) + } } } @@ -124,7 +154,7 @@ impl Buffer { if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} let ptr=self.cpu_ptr()?; // SAFETY: checked span in retained CPU storage; no aligned access is required. - unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + unsafe {crate::m3::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; Ok(()) } pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index d34e1f23840eb..8167c6c3ec05c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -139,7 +139,7 @@ unsafe extern "C" fn get_atomic_param( } fn parse_render_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_pass_layout::SLOTS as u64) } /// Runtime tuning is opt-in: zero keeps the existing boot parameter and its @@ -202,20 +202,58 @@ pub(crate) fn render_internal_timestamps()->bool { M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 } +// The runtime's boot-time module parameters. On 7.1.12 they were `module!` parameters; they +// keep their names, and exist only with the T8122 runtime built in. + +/// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16). +static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); +/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). +static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); +/// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. +static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_early_tiling", M3_EARLY_TILING, parse_u64); + +/// `asahi.m3_early_tiling` is set. +pub(crate) fn early_tiling() -> bool { + M3_EARLY_TILING.load(Ordering::Relaxed) != 0 +} + +fn parse_expose(text: &str) -> Option { + match text.trim() { + "-1" => Some(EXPOSE_AUTO), + other => parse_u64(other).filter(|v| *v <= 1), + } +} +const EXPOSE_AUTO: u64 = u64::MAX; +/// `asahi.m3_expose`: M3 render node registration: -1 = auto (on the boards where the runtime +/// is validated), 0 = never, 1 = always. +static M3_EXPOSE: AtomicU64 = AtomicU64::new(EXPOSE_AUTO); +m3_param!("m3_expose", M3_EXPOSE, parse_expose); + +/// `asahi.m3_expose`: `None` for auto, else whether to register the render node. +pub(crate) fn expose() -> Option { + match M3_EXPOSE.load(Ordering::Relaxed) { + EXPOSE_AUTO => None, + v => Some(v != 0), + } +} + /// Snapshot once per batch. A live change affects later publications, never the /// storage, ordering or retirement checks of work already on the firmware queues. pub(crate) fn render_batch_size() -> usize { let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_render_batch_size.value() as usize + M3_RENDER_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_pass_layout::SLOTS) + size.clamp(1, crate::m3::m3_pass_layout::SLOTS) } fn parse_compute_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_compute_storage::SLOTS as u64) } /// Independently tune the existing adjacent-compute path. Zero preserves the @@ -229,11 +267,11 @@ m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, pub(crate) fn compute_batch_size() -> usize { let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_compute_batch_size.value() as usize + M3_COMPUTE_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_compute_storage::SLOTS) + size.clamp(1, crate::m3::m3_compute_storage::SLOTS) } /// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. @@ -320,15 +358,6 @@ fn parse_backend(text: &str) -> Option { static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); m3_param!("m3_backend", M3_BACKEND, parse_backend); -/// The T6030 GPU backend to start. -pub(crate) enum T6030Backend { - /// The serialized M3 runtime (m3_drm / m3_runtime). - Runtime, - /// The G15 GpuManager backend (g15_probe). - Manager, - /// No backend: the probe stops without touching the GPU. - Off, -} /// `asahi.m3_initdata` values. const INITDATA_AUTO: u64 = 0; @@ -418,7 +447,7 @@ pub(crate) fn unlocked_wait() -> bool { } /// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback -/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +/// cap. Unset: the SoC's default ([`crate::m3::m3_soc::Soc::ut_engagement`]). static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; fn parse_ut_engagement(text: &str) -> Option { @@ -427,7 +456,7 @@ fn parse_ut_engagement(text: &str) -> Option { m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); /// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. -pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { +pub(crate) fn ut_engagement(soc: &crate::m3::m3_soc::Soc) -> (u32, bool) { match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), v => (v as u32, true), @@ -443,14 +472,14 @@ static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); fn parse_pipeline_depth(text: &str) -> Option { - parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|v| *v<=crate::m3::m3_pass_layout::SLOTS as u64) } /// Effective depth; changing it never releases already published pass slots. -pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { +pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { - 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), - v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + 0 => crate::m3::m3_runtime::policy::default_depth(crate::m3::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3::m3_pass_layout::SLOTS as u64) as usize, } } @@ -464,9 +493,9 @@ static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); /// The render batch GPU-time budget in ns, or `None` without one. -pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { +pub(crate) fn render_batch_budget_ns(soc: &crate::m3::m3_soc::Soc) -> Option { let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); - let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + let us=if value==u64::MAX {crate::m3::m3_runtime::policy::default_budget_us(crate::m3::t8122_start::is_t8122(soc))} else {value}; match us { 0 => None, us => Some(us.saturating_mul(1000)), @@ -486,12 +515,12 @@ fn parse_retire_mmio(text: &str) -> Option { /// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every /// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 /// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance -/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +/// state (0xe01000). Unset: the SoC's default ([`crate::m3::m3_soc::Soc::retire_mmio`]). static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); /// The `asahi.m3_retire_mmio` mask for `soc`. -pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { +pub(crate) fn retire_mmio(soc: &crate::m3::m3_soc::Soc) -> u64 { match M3_RETIRE_MMIO.load(Ordering::Relaxed) { RETIRE_MMIO_UNSET => soc.retire_mmio, mask => mask, @@ -563,7 +592,7 @@ pub(crate) fn thermal_hot_param() -> (u64, bool) { /// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with /// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another /// board. The G15 manager backend is T6030-only. -pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { +pub(crate) fn t8122_backend(pdev: &platform::Device>) -> bool { let dev = pdev.as_ref(); match M3_BACKEND.load(Ordering::Relaxed) { BACKEND_OFF => { @@ -577,7 +606,7 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { ); false } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ if crate::m3::m3_board::runtime_validated_board(&crate::m3::m3_soc::T8122) => true, _ => { dev_info!( dev, @@ -588,57 +617,27 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { } } -/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. -/// -/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook -/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; -/// `asahi.m3_backend=runtime` starts it anyway. -pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { - let dev = pdev.as_ref(); - match M3_BACKEND.load(Ordering::Relaxed) { - BACKEND_RUNTIME => { - dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); - T6030Backend::Runtime - } - BACKEND_MANAGER => { - dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); - T6030Backend::Manager - } - BACKEND_OFF => { - dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); - T6030Backend::Off - } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, - _ => { - dev_info!( - dev, - "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" - ); - T6030Backend::Off - } - } -} fn parse_t8122_number(text: &str) -> Option { - Some(crate::t8122_knobs::parse_number(text)) + Some(crate::m3::t8122_knobs::parse_number(text)) } fn parse_t8122_fender(text: &str) -> Option { - Some(crate::t8122_knobs::parse_fender(text)) + Some(crate::m3::t8122_knobs::parse_fender(text)) } fn parse_t8122_clkgen(text: &str) -> Option { - Some(crate::t8122_knobs::parse_clkgen(text)) + Some(crate::m3::t8122_knobs::parse_clkgen(text)) } fn parse_t8122_sgx_setup(text: &str) -> Option { - Some(crate::t8122_knobs::parse_sgx_setup(text)) + Some(crate::m3::t8122_knobs::parse_sgx_setup(text)) } // The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a // value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather // than fall back to a default. -use crate::t8122_knobs::UNSET as T8122_UNSET; +use crate::m3::t8122_knobs::UNSET as T8122_UNSET; /// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. /// Only with a boot loader that handed the GPU over. @@ -667,21 +666,21 @@ static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); /// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) -/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// writes the 20 words at HwDataA +0x418c ([`crate::m3::t8122_hwdata`]); 0 leaves the /// range zero, for an A/B test. Readable in sysfs. -static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); -m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::m3::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::m3::t8122_hwdata::parse, 0o444, Some(get_atomic_param)); /// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). pub(crate) fn t8122_hwdata_418c() -> bool { - crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) + crate::m3::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) } /// The T8122 start-experiment parameters of this boot, as given. -pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { +pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw { let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t8122_knobs::Raw { + crate::m3::t8122_knobs::Raw { start: get(&T8122_START), initdata_version: get(&T8122_INITDATA_VERSION), fender: get(&T8122_FENDER), @@ -692,209 +691,3 @@ pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { pstate_cap: get(&T8122_PSTATE_CAP), } } - -fn parse_t6031_number(text: &str) -> Option { - Some(crate::t6031_knobs::parse_number(text)) -} - -fn parse_t6031_fender(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fender(text)) -} - -fn parse_t6031_clkgen(text: &str) -> Option { - Some(crate::t6031_knobs::parse_clkgen(text)) -} - -fn parse_t6031_sgx_setup(text: &str) -> Option { - Some(crate::t6031_knobs::parse_sgx_setup(text)) -} - -fn parse_t6031_mcache(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mcache(text)) -} - -fn parse_t6031_aic_swint(text: &str) -> Option { - Some(crate::t6031_knobs::parse_aic_swint(text)) -} - -fn parse_t6031_slot7(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot7(text)) -} - -fn parse_t6031_gifaf(text: &str) -> Option { - Some(crate::t6031_knobs::parse_gifaf(text)) -} - -fn parse_t6031_slot(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot(text)) -} - -fn parse_t6031_ane(text: &str) -> Option { - Some(crate::t6031_knobs::parse_ane(text)) -} - -fn parse_t6031_mtr(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mtr(text)) -} - -fn parse_t6031_fw_words(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fw_words(text)) -} - -fn parse_t6031_hwdata_object(text: &str) -> Option { - Some(crate::t6031_knobs::parse_hwdata_object(text)) -} - -fn parse_t6031_rev_id(text: &str) -> Option { - Some(crate::t6031_knobs::parse_rev_id(text)) -} - -fn parse_t6031_tristate(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tristate(text)) -} - -fn parse_t6031_tag(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tag(text)) -} - -// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its -// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. -use crate::t6031_knobs::UNSET as T6031_UNSET; - -/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe -/// is the same refusal as before. -static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_start", T6031_START, parse_t6031_number); -/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. -static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); -/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. -static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); -/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. -static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); -/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. -static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); -/// `asahi.t6031_fender=rule|adt`: the Fender window size. -static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); -/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. -static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); -/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. -static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); -/// `asahi.t6031_mcache=none|x2|x4|x8`. -static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); -/// `asahi.t6031_aic_swint=page|reg`. -static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); -/// `asahi.t6031_slot7=none|rule`. -static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); -/// `asahi.t6031_gifaf=none|fender`. -static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); -/// `asahi.t6031_slot21=none` or a physical address. -static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); -/// `asahi.t6031_slot24=none` or a physical address. -static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); -/// `asahi.t6031_slot28=none` or a physical address. -static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); -/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. -static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); -/// `asahi.t6031_io_drop`: a mask of default slots to leave out. -static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); -/// `asahi.t6031_mtr=t6030|t8122|adt|none`. -static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); -/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. -static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); -/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. -static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); -/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). -static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); -/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). -static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); -/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. -static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); -/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. -static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); -/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. -static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); -/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. -static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); -/// `asahi.t6031_fw_words=none|t8122`. -static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); -/// `asahi.t6031_hwdata_object=aligned|fixed`. -static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); -/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. -static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); -/// `asahi.t6031_rev_id=auto` or 1 to 6. -static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); -/// `asahi.t6031_csafr=auto|on|off`. -static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); -/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. -static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); - -/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. -pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { - let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t6031_knobs::Raw { - start: get(&T6031_START), - image_uuid: crate::t6031_knobs::UuidParam::Unset, - image_hash: get(&T6031_IMAGE_HASH), - tag: get(&T6031_TAG), - initdata_version: get(&T6031_INITDATA_VERSION), - pstate_cap: get(&T6031_PSTATE_CAP), - fender: get(&T6031_FENDER), - clkgen: get(&T6031_CLKGEN), - sgx_setup: get(&T6031_SGX_SETUP), - mcache: get(&T6031_MCACHE), - aic_swint: get(&T6031_AIC_SWINT), - slot7: get(&T6031_SLOT7), - gifaf: get(&T6031_GIFAF), - slot21: get(&T6031_SLOT21), - slot24: get(&T6031_SLOT24), - slot28: get(&T6031_SLOT28), - ane: get(&T6031_ANE), - io_drop: get(&T6031_IO_DROP), - mtr: get(&T6031_MTR), - mtr_fast_die: get(&T6031_MTR_FAST_DIE), - mtr_alarm: get(&T6031_MTR_ALARM), - unit_mask_a: get(&T6031_UNIT_MASK_A), - unit_mask_b: get(&T6031_UNIT_MASK_B), - hwb_454: get(&T6031_HWB_454), - hwb_b20: get(&T6031_HWB_B20), - hwb_17b8: get(&T6031_HWB_17B8), - hwb_1818: get(&T6031_HWB_1818), - fw_words: get(&T6031_FW_WORDS), - hwdata_object: get(&T6031_HWDATA_OBJECT), - gpu_core: get(&T6031_GPU_CORE), - rev_id: get(&T6031_REV_ID), - csafr: get(&T6031_CSAFR), - power_cap_mw: get(&T6031_POWER_CAP_MW), - } -} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 273d2701c3f14..94c2172e33279 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -2,12 +2,12 @@ //! Private storage for one retired/unpublished G15 render pass. A Pass retains //! all its buffers and mappings until Render retires the shared queues. use kernel::prelude::*; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_pass_layout as layout}; use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, slot:usize,clusters:u32,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { @@ -28,7 +28,7 @@ impl Pass { pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} /// Caller must prove retirement, and must not clear a prepared sibling. /// Dependency commands are rewritten completely by the sync encoder. - pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + pub(crate) fn reset(&mut self,geometry:&crate::m3::agx_render::Geometry)->Result { // Tiler arenas grow to retain the largest prior target. The firmware // descriptors use the current geometry's strides and layer count, so // only that prefix is addressable by this pass. Clear it on every use, diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs index 979c1a37bb202..ab50ab7eca8a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -3,7 +3,7 @@ //! GPU mappings and counters outlive this initialization and remain owned by //! Render/Compute. Callers must never invoke this on a live or replayed pool. use kernel::prelude::*; -use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +use crate::{m3::m3_memory::Buffer, m3::m3_pool_layout as wire, m3::m3_queue_layout::FirmwareVa}; pub(crate) struct Owners { pub(crate) backing: core::ops::Range, pub(crate) pages: usize, diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index c2c67f5c3aa5a..e7a02982e96f1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_sync_layout as sync; -use crate::m3_state_layout as state; -use crate::m3_scene_layout as scene; -use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; -use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; -use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_sync_layout as sync; +use crate::m3::m3_state_layout as state; +use crate::m3::m3_scene_layout as scene; +use crate::{m3::m3_pass::{Pass,ViewOwner},m3::m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3::m3_fragment_command as fragment, m3::m3_tiler_command as tiler, m3::m3_render_sequence as sequence, m3::m3_init_layout::Region}; +use crate::{m3::m3_parameter_layout as parameter, m3::m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_shared_layout as s,m3::agx_uapi}; +use crate::m3::m3_timeline as timeline; use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; @@ -38,7 +38,9 @@ pub(crate) struct Render { /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. clusters:u32, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). - registers:crate::m3_compute_layout::RegisterSet, + registers:crate::m3::m3_compute_layout::RegisterSet, + /// Last: drops after the pass aliases. + _lower_tlbi:crate::m3::m3_memory::LowerTlbi, } // Queue allocations belong to Render, not to a pass. Keep their fixed arenas // and firmware/shared mapping permissions from the qualified allocation map. @@ -51,9 +53,9 @@ use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; const MANAGER_SCENE_COUNT: u32 = 48; const TA_POOL_COUNTER: usize = 11; -const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ - crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, - crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +const UMA_POOLS: [crate::m3::m3_pool::Owners; 2] = [ + crate::m3::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, ]; const FRAGMENT_POOL_COUNTER: usize = 23; // Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. @@ -65,8 +67,9 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { @@ -91,14 +94,14 @@ impl Render { tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; Ok(job) } @@ -223,7 +226,7 @@ impl Render { // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { - use crate::m3_compute_layout::GpuVa; + use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -247,7 +250,7 @@ impl Render { user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, - vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; @@ -290,8 +293,8 @@ impl Render { Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. - fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { - use crate::agx_render::{Geometry,Utile}; + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::m3::agx_render::{Geometry,Utile}; let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; @@ -299,7 +302,7 @@ impl Render { Ok(g) } fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { - use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + use crate::{m3::m3_compute_layout::GpuVa,m3::agx_render_state::{Program,DepthStencil}}; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -355,7 +358,7 @@ impl Render { self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; } - crate::agx_memory::publish();Ok(()) + crate::m3::agx_memory::publish();Ok(()) } fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} /// GPU time of the batch of `count` passes starting at pass slot `base`. @@ -416,13 +419,13 @@ impl Render { /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). - pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, - commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3::m3_submit::Command],base:usize,overlap:bool)->Result { if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} if overlap { if !self._binding.matches(vm) {return Err(EBUSY);} for (index,&command) in commands.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; let slot=Self::at(base,index); if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 @@ -433,9 +436,9 @@ impl Render { self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; // Validate every pass before changing any queue or completion state. for &command in commands { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; Self::geometry(r,self.clusters)?; - crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; } if !self._binding.matches(vm) { let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { @@ -477,7 +480,7 @@ impl Render { // already-grown buffers are reusable on the next submission. for (index,&command) in commands.iter().enumerate() { let slot=Self::at(base,index); - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; @@ -504,7 +507,7 @@ impl Render { for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } /// Append only into caller-reserved free pass slots. With overlap, WRITE updates @@ -533,7 +536,7 @@ impl Render { // Retain the conservative stage-scope permission. Only a pass after the // first of its batch may overlap its predecessor's stages. Explicit // start timestamps keep the original full-order execution boundary. - let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + let early=predecessor.is_none() && self.batch_count>0 && crate::m3::m3_params::early_tiling() && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 && r.fragment_timestamps.start.handle==0; if early {self.early_count+=1;} @@ -585,15 +588,15 @@ impl Render { self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; self.heads[stage]=(self.heads[stage]+1)%0x500; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) } /// Emitted only after both events, stamp/queue checks, clear faults, /// idle engines and consumed firmware pipes have been checked by Runtime. /// The existing TA/fragment descriptors request completion stamp flushes. - pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn progress(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Keep summaries sparse at high throughput, but emit at least once a // second while completed batches advance. Batched draw counts can skip // multiples of 128; a count-only gate can starve the watchdog observer. @@ -654,7 +657,7 @@ impl Render { || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} } Ok(true) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter // pool can stall TA without producing an MMU fault. for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs index bfc94fdfa0ca8..7d34305250018 100644 --- a/drivers/gpu/drm/asahi/m3/m3_resources.rs +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 GPU resources admitted against the measured boot contract (J514S on T6030). -pub(crate) use crate::agx_resources::{Region, Resources}; +pub(crate) use crate::m3::agx_resources::{Region, Resources}; use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; @@ -11,13 +11,13 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result = node.get_property(c_str!("compatible"))?; let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; - if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + if !crate::m3::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { return Err(EINVAL); } let names: KVec = node.get_property(c_str!("memory-region-names"))?; @@ -44,8 +44,8 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result, - soc: &crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &crate::m3::m3_soc::Soc, ) -> kernel::error::Result { - crate::m3_board::admit(pdev, soc) + crate::m3::m3_board::admit(pdev, soc) } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index 3e8cbbfa93781..be81c4a4e9d92 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -3,18 +3,17 @@ //! RTKit callbacks for the identified J514S firmware. The firmware advertises //! its crash storage by physical address inside the reserved data segment. -use crate::agx_resources::Region; +use crate::m3::agx_resources::Region; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ c_str, impl_has_hr_timer, device::Core, sync::Completion, - io::mem::{Mem, MemFlag}, - iosys_map::IoSysMapRef, + io::{mem::{Mem, MemFlag}, IoSysMap, Region as IoRegion, SysMem}, platform, prelude::*, soc::apple::rtkit, - sync::{aref::ARef, Arc, ArcBorrow}, + sync::{aref::ARef, Arc, ArcBorrow, SetOnce}, time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, }; @@ -40,7 +39,7 @@ impl EventWait { pin_init!(EventWait { count: AtomicU64::new(0), wait_us: AtomicU64::new(100), - overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), timer_wakes: AtomicU64::new(0), @@ -139,6 +138,9 @@ pub(crate) struct State { /// The firmware sends one after a command's completion processing. pub(crate) event_messages: AtomicU64, pub(crate) events: Arc, + /// The completion worker, published once the runtime it services exists. The DRM device + /// carries no driver data for it on 7.2, so the runtime's own state owns it. + pub(crate) completion: SetOnce>, } /// Shared status contains no device references, so DRM files may retain it @@ -147,8 +149,8 @@ pub(crate) struct Health { crashed: AtomicBool, mapped: AtomicBool, failed: AtomicBool, - progress: crate::agx_host_progress::Progress, - pub(crate) timing: crate::agx_timing_stats::Stats, + progress: crate::m3::agx_host_progress::Progress, + pub(crate) timing: crate::m3::agx_timing_stats::Stats, } impl Health { @@ -181,7 +183,7 @@ impl Health { } impl State { - pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + pub(crate) fn new(dev: &platform::Device>, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { Ok(Arc::new( Self { dev: dev.into(), @@ -191,12 +193,13 @@ impl State { crashed: AtomicBool::new(false), mapped: AtomicBool::new(false), failed: AtomicBool::new(false), - progress: crate::agx_host_progress::Progress::new(), - timing: crate::agx_timing_stats::Stats::new(), + progress: crate::m3::agx_host_progress::Progress::new(), + timing: crate::m3::agx_timing_stats::Stats::new(), }, GFP_KERNEL)?, claimed: AtomicBool::new(false), event_messages: AtomicU64::new(0), events: EventWait::new()?, + completion: SetOnce::new(), }, GFP_KERNEL, )?) @@ -227,8 +230,18 @@ impl rtkit::Buffer for FirmwareBuffer { fn iova(&self) -> Result { Ok(self.physical) } - fn buf(&mut self) -> Result> { - self.mapping.iosys_map(self.offset, self.size) + fn buf(&mut self) -> Result> { + // The view of 7.1.12's `Mem::iosys_map`: the bounded span inside the retained WB + // mapping, as normal memory. + if self.offset.checked_add(self.size).ok_or(EOVERFLOW)? > self.mapping.size() { + return Err(EINVAL); + } + // SAFETY: The span lies entirely inside the retained mapping (checked above). + let base = unsafe { self.mapping.ptr().add(self.offset) }; + let region = IoRegion::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The span is mapped kernel memory for as long as `self.mapping`, which the + // returned view borrows through `&mut self`. + Ok(IoSysMap::Sys(unsafe { SysMem::new(region) })) } } @@ -244,15 +257,12 @@ impl rtkit::Operations for Operations { // Wake the serialized worker even if the firmware can no longer send // its usual completion notification. Never take the runtime lock here. state.events.record(); - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); dev_err!( state.dev.as_ref(), "M3 G15S: firmware crashed, retained crashlog bytes={}\n", crashlog.map_or(0, |b| b.len()) ); - if let Some(bytes) = crashlog { - crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); - } } fn shmem_map( @@ -275,7 +285,7 @@ impl rtkit::Operations for Operations { return Err(EBUSY); } let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; - let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + let resource = crate::m3::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; if resource.start() != state.data.base || resource.size() != state.data.size { return Err(EINVAL); } @@ -314,7 +324,7 @@ impl rtkit::Operations for Operations { ::ktime_get(), state.event_messages.load(Ordering::Acquire)); } - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); return; } dev_warn!( diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 08885f591c140..b12fc73f9f248 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -2,19 +2,19 @@ use core::{mem::ManuallyDrop, sync::atomic::Ordering}; use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, time::{Instant, Monotonic, Delta, delay::fsleep}}; -use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; +use crate::{driver, mmu, m3::m3_device::Device, m3::m3_config::Config, m3::m3_rtkit}; #[path = "m3_batch_policy.rs"] pub(crate) mod policy; -const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); +const _: () = assert!(policy::SLOTS == crate::m3::m3_pass_layout::SLOTS); -enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +enum NativeJob {Compute(crate::m3::m3_compute::Compute),Render(crate::m3::m3_render::Render)} /// Completion-worker outcome. pub(crate) enum Service {Idle,Progress,Waiting(u64)} /// The published batch: its commands in order, as (packet, command index). struct Batch { - entries: KVec<(Arc,usize)>, - index: usize, kind: usize, first: crate::m3_submit::Command, + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3::m3_submit::Command, preparation_ns: i64, previous_events: u64, expected_events: u64, start: Instant, measure: bool, doorbell_counter_ns: i64, /// Render: first pass slot and the batch's last draw number. @@ -40,16 +40,16 @@ struct GeometryTiming { } impl NativeJob { fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} - fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} } struct Inner { transport: rtkit::RtKit, state: Arc, jobs: KVec, - packets: KVec>, + packets: KVec>, /// Packets not yet (fully) published: (packet, next command index), most urgent first. - pending: KVec<(Arc,usize)>, + pending: KVec<(Arc,usize)>, /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. active: KVec, gpu_pending: bool, @@ -57,14 +57,14 @@ struct Inner { active_power_samples: u32, fault_captured: bool, timing: [[i64;7];2], - render_batches: [u64;crate::m3_pass_layout::SLOTS], + render_batches: [u64;crate::m3::m3_pass_layout::SLOTS], geometry: [GeometryTiming; 32], geometry_overflow: u64, /// Whether a T8122 job verdict line was already emitted for the current job, so the /// execute-path fallback does not add a second one. T8122 only. t8122_verdict: bool, /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. - boot_step: crate::t8122_start::BootStep, + boot_step: crate::m3::t8122_start::BootStep, /// Packets that joined a batch started by an earlier packet (debug summary). coalesced_packets: u64, coalesced_render_stamps: u64, @@ -76,6 +76,9 @@ struct Inner { render_initialized: bool, config: Config, uat: mmu::Uat, + /// The firmware's view of the host window (the shared L2 the firmware walks), kept in step + /// with the kernel VM's shadow window before the firmware is given new kernel addresses. + window: crate::g15::uat::Window, drm: driver::AsahiDevRef, device: Device, } @@ -86,10 +89,10 @@ impl Inner { self.state.health.mark_failed(); if self.fault_captured { return; } self.fault_captured = true; - #[cfg(CONFIG_DEV_COREDUMP)] - if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { - dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); - } + // 7.1.12 also published a devcoredump of the firmware objects here; 7.2's kernel crate + // has no devcoredump binding, so the first error is only logged. + dev_err!(self.drm.as_ref(), "M3: first GPU failure: {:?} (GPU work pending: {})\n", + primary, self.gpu_pending); } /// Record a batch that failed or never retired. Mark the GPU failed first, @@ -97,15 +100,12 @@ impl Inner { /// behind a device that still reports itself healthy. fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { self.capture_fault(primary); - if crate::t8122_start::is_t8122(self.device.soc()) { + if crate::m3::t8122_start::is_t8122(self.device.soc()) { let pstate=self.device.pstate_register().ok(); // The job's first GPU start timestamp: nonzero means the firmware dispatched it, // whatever the power state reads at the timeout. let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); - crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + crate::m3::t8122_start::job_failed_verdict(self.drm.as_ref().as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); self.t8122_verdict=true; } let events=self.state.event_messages.load(Ordering::Acquire); @@ -116,7 +116,7 @@ impl Inner { dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); } let _=self.config.log_recovery_state(&self.drm,events); - if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault) { if let Err(e)=self.resume_experiment(index, vm) { dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); } @@ -203,10 +203,10 @@ impl Inner { } /// Hardware completion is separate from fence signaling: cancellation can win the /// signal race while the DMA guard must survive until its final published batch retires. - fn packet_active(&self, packet:&Arc)->bool { + fn packet_active(&self, packet:&Arc)->bool { self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) } - fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { for (packet,_) in entries { let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} @@ -221,37 +221,37 @@ impl Inner { } } /// Signal every packet of `entries` (packets that continue past the batch included). - fn finish_all(entries:&[(Arc,usize)],result:Result) { + fn finish_all(entries:&[(Arc,usize)],result:Result) { for (packet,_) in entries {packet.finish(result);} } /// Choose the next batch from the front of `pending`: the front packet's next commands of /// one engine, and for render also the leading render commands of following packets of the /// same VM, up to the ordered-batch limit. - fn next_batch(&self,max:usize)->Result,usize)>> { + fn next_batch(&self,max:usize)->Result,usize)>> { let mut entries=KVec::new(); let Some((packet,next))=self.pending.first() else {return Ok(entries);}; - let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + let mixed=policy::mixed_prefix(crate::m3::m3_params::mixed_batch(), self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), - packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3::m3_submit::Command::Render{..}))); if mixed>0 { for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} return Ok(entries); } - let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let render=matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}); let limit=if render { // The first render draw initializes the shared buffer manager (InitBM) and opens // both queues. Publish it alone: appending later draws first advances the shared // manager and event counters it starts from, and on J613 the second pass's // fragment then fails with firmware error event 4 (draw=2 batch=2). let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); - if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + if started {crate::m3::m3_params::render_batch_size().min(max).max(1)} else {1} } else { - crate::m3_params::compute_batch_size() - .clamp(1,crate::m3_compute_storage::SLOTS) + crate::m3::m3_params::compute_batch_size() + .clamp(1,crate::m3::m3_compute_storage::SLOTS) }; - let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); let per_pass=self.pass_cost.get(packet.vm.id()); - let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { // Later packets join only render batches, only from their first command, and only @@ -272,19 +272,19 @@ impl Inner { /// Whether the front pending work may be published behind the in-flight batches, and if so /// in which pass slot it starts and how many passes it may use. fn pipeline_slot(&mut self)->Result> { - let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3::m3_pass_layout::SLOTS)));}; if !self.render_initialized {return Ok(None);} let Some((packet,next))=self.pending.first() else {return Ok(None);}; - if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + if !matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}) || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); - let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + let Some(room)=policy::pipeline_room(crate::m3::m3_params::pipeline_depth(self.device.soc()), self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; // Yield at the next retirement when more urgent work is already waiting. if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), last.entries[0].0.urgency) {return Ok(None);} if !policy::within_budget(self.pass_cost.get(packet.vm.id()), - crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + crate::m3::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} if !policy::overlap_ready(self.render_initialized,true,true,true, self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} Ok(Some(room)) @@ -325,11 +325,8 @@ impl Inner { } Err((error,published))=>{ if published { - if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { - crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { - crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + if crate::m3::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::m3::t8122_start::job_setup_failed_verdict(self.drm.as_ref().as_ref(),error); self.t8122_verdict=true; } // A live ring WRITE or doorbell may have been published: keep packets @@ -355,9 +352,9 @@ impl Inner { progress } /// Prepare and publish one batch. On error, reports whether firmware may see its writes. - fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { - let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); - let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Compute(_))); if render && compute {return self.publish_mixed(entries,base,overlap);} let early=|e:Error|(e,false); let (packet,first_index)=(&entries[0].0,entries[0].1); @@ -378,19 +375,19 @@ impl Inner { if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} } let found=self.jobs.iter().position(|job| matches!((job,control), - (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | - (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..}))); let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), - crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { match (&mut self.jobs[index],control) { - (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, _=>return Err(early(ENOTSUPP)), } } @@ -398,11 +395,11 @@ impl Inner { let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { if slot!=0 { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; j.append(c)?; } let timestamps=p.timestamp_addresses(*next); - let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + let coalesce=crate::m3::m3_compute_storage::coalesce_stamp_flush( p.wide_visibility[*next],timestamps,slot+1==batch_count); j.prepare_completion(coalesce)?; j.set_user_timestamps(timestamps)?; @@ -417,8 +414,8 @@ impl Inner { } } let mut coalesced_stamps=0; - let coalesce_render=crate::m3_params::render_stamp_coalesce(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let coalesce_render=crate::m3::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); let chip=self.device.soc().chip_id; if let NativeJob::Render(j)=&mut self.jobs[index] { let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -426,7 +423,7 @@ impl Inner { j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*next); let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, overlap,slot,batch_count,timestamps); @@ -448,6 +445,19 @@ impl Inner { } } let preparation_ns=prepare.elapsed().as_nanos(); + // The batch's kernel objects may have created leaf tables in the host window: link + // them into the firmware's shared L2 before it is told about the batch. A failure + // means the window no longer matches what the firmware may walk: stop scheduling and + // retain the batch, as for a published error. + if let Err(error)=self.window.sync_all(&self.uat) { + return Err((error,true)); + } + // The command aliases live in the context-0 lower tree, which has no bind slot on the + // adopted UAT (7.1.12 gave it the fixed slot 0), so remapping one invalidates nothing: + // drop context 0's non-global entries before the firmware fetches the new ones. + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; @@ -470,7 +480,7 @@ impl Inner { }; let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; - Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3::m3_submit::Command::Compute(_))), first:control, preparation_ns, previous_events, expected_events, start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) @@ -478,7 +488,7 @@ impl Inner { /// Prepare both idle queues from one packet. Cross-engine changes wait on /// the preceding command's firmware stamp; all commands remain GPU ordered. /// Producer storage and DMA guards survive every partial publication error. - fn publish_mixed(&mut self,entries:&[(Arc,usize)], + fn publish_mixed(&mut self,entries:&[(Arc,usize)], base:usize,overlap:bool)->core::result::Result { let early=|e:Error|(e,false); if overlap || base!=0 || self.device.soc().chip_id!=0x6030 @@ -486,7 +496,7 @@ impl Inner { let packet=&entries[0].0; if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} let prepare=Instant::::now(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -497,8 +507,8 @@ impl Inner { for (p,i) in entries { owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; match p.commands[*i] { - c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, - crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + c@crate::m3::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, } } let renders=render_commands.len(); @@ -514,7 +524,7 @@ impl Inner { let prepared=(||->Result { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { - let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, self.config.stats_region()?,control,self.device.soc().registers)?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { @@ -528,12 +538,12 @@ impl Inner { let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; render_prepared=true; - let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut previous:Option<(bool,crate::m3::m3_sync_layout::CompletionDependency)>=None; let mut compute_slot=0; let mut dependencies=0; for (p,i) in entries { match p.commands[*i] { - crate::m3_submit::Command::Render{command,usc}=>{ + crate::m3::m3_submit::Command::Render{command,usc}=>{ let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*i); @@ -543,7 +553,7 @@ impl Inner { dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; if compute_slot!=0 {j.append(c)?;} if let Some((true,dependency))=previous { @@ -628,14 +638,14 @@ impl Inner { let mut compute_slot=0; for (packet,next) in batch.entries.iter() { match packet.commands[*next] { - crate::m3_submit::Command::Render{command:r,..}=>{ + crate::m3::m3_submit::Command::Render{command:r,..}=>{ let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); render_slot+=1; }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(compute_slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", @@ -718,10 +728,10 @@ impl Inner { } } } - let power_enabled=crate::m3_params::active_power(); + let power_enabled=crate::m3::m3_params::active_power(); if power_enabled { let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); - if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + if crate::m3::m3_active_power::due(power_enabled,self.device.soc().chip_id, stamped,self.active_power_samples,since_last) { self.active_power_last=Some(Instant::::now()); self.active_power_samples+=1; @@ -760,23 +770,20 @@ impl Inner { (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) }; if stamped && evented { - let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc()); match self.device.check_idle_parts( - retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, - retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_FAULTS!=0) { Ok(())=>match self.config.pipes_idle() { Ok(true)=>{ if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let tick=physical_counter(); if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} } - if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if retire_mmio&crate::m3::m3_params::RETIRE_MMIO_PSTATE!=0 { if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { - if crate::t8122_start::is_t8122(self.device.soc()) { - crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + if crate::m3::t8122_start::is_t8122(self.device.soc()) { + crate::m3::t8122_start::cap_violated_verdict(self.drm.as_ref().as_ref(),e); self.t8122_verdict=true; } self.fail_active(e);return Err(e); @@ -862,8 +869,8 @@ impl Inner { if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let observed_tick=physical_counter(); let (cs,span)=match (&mut self.jobs[index],control) { - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), - (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, _=>return Err(EINVAL), }; let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; @@ -874,14 +881,14 @@ impl Inner { if kind==0 {gpu_ns} else {0},stages[1]); if let NativeJob::Compute(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); } } if let NativeJob::Render(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; let s=j.slot_stage_ns(batch.base,batch_count,slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); } @@ -893,7 +900,7 @@ impl Inner { preparation_ns, active_ns, gpu_ns, stages); let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); if batch.measure { - if let crate::m3_submit::Command::Render {command:r,..}=control { + if let crate::m3::m3_submit::Command::Render {command:r,..}=control { if batch_count>1 {self.geometry_overflow+=1;} else { if let Some(g)=self.geometry.iter_mut().find(|g| g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { @@ -917,9 +924,9 @@ impl Inner { } if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { match control { - crate::m3_submit::Command::Render {command:r,..}=> + crate::m3::m3_submit::Command::Render {command:r,..}=> dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), - crate::m3_submit::Command::Compute(_)=> + crate::m3::m3_submit::Command::Compute(_)=> dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), } } @@ -927,14 +934,12 @@ impl Inner { t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; for i in 0..3 {t[4+i]+=stages[i] as i64;} // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. - if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + if t[0]==1 && crate::m3::t8122_start::is_t8122(self.device.soc()) { let span=match &mut self.jobs[index] { NativeJob::Compute(j)=>j.batch_gpu_span().ok(), NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), }; - crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); - } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + crate::m3::t8122_start::job_completed_verdict(self.drm.as_ref().as_ref(),kind,gpu_ns,span); } if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], @@ -971,26 +976,23 @@ impl Inner { pub(crate) struct Runtime { inner: ManuallyDrop> } impl Runtime { #[inline(never)] - pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does // not borrow `device`, which is moved into Inner below. - let t8122 = crate::t8122_start::is_t8122(device.soc()); - let t6031 = crate::t6031_start::is_t6031(device.soc()); + let t8122 = crate::m3::t8122_start::is_t8122(device.soc()); let coproc_refused = |e: &Error| { - if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t8122 { crate::m3::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } }; device.require_stopped(pdev).inspect_err(coproc_refused)?; // Reserve before starting ASC. ENOMEM must not drop firmware owners // while the coprocessor may still access them. Writing the completed // graph into this allocation is infallible. let owner = KBox::::new_uninit(GFP_KERNEL)?; - let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let drm: driver::AsahiDevRef = drm.into(); let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; - if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) { return Err(ENODEV); } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. @@ -1004,18 +1006,22 @@ impl Runtime { Pin::new(&mut transport).start_endpoint(ep)?; } started=true; - // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). - // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU - // job has been published. Its running PPL handoff owns the peer lock. - let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + // The firmware's UAT, adopted as the G15 backend adopts it on 7.2: the handoff and + // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as + // every context's upper half, and the host window (root entry 2) through the + // shared L2. The RTKit is awake and no initdata or GPU job has been published. + let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; let config=Config::new(&drm,&uat,device.firmware(),&contents)?; - Ok((uat,config)) + // Link every kernel table the configuration created before the firmware sees an + // address in it, then drop any translation cached before they existed. + window.sync_all(&uat)?; + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok((uat,window,config)) })(); - let (uat,config)=match prepared { + let (uat,window,config)=match prepared { Ok(v)=>v, Err(e)=>{ - crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); - if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; @@ -1025,34 +1031,43 @@ impl Runtime { return Err(e); } }; - Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, - jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS], geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, - boot_step:crate::t8122_start::BootStep::Publish })) }) + boot_step:crate::m3::t8122_start::BootStep::Publish })) }) } pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + /// The RTKit callbacks' state, which owns the completion worker. + pub(crate) fn rtkit_state(&self) -> Arc { self.inner.state.clone() } pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { use crate::util::RangeExt; let reserved_range=0x70_0000_0000..0x80_0000_0000; if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - let vm=self.inner.uat.new_vm(id, range)?; + // Status and job lifetime tracking: deferred unmaps while accepted work runs + // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). + let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; vm.install_driver_mappings(KVec::new(),reserved)?; Ok(vm) } - pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + let mapping=bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::m3::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false)?; + // A job of an overlapping batch can reach the firmware without a new publication: + // link a new leaf table now, not only at the next batch. + let inner: &mut Inner=&mut self.inner; + inner.window.sync_all(&inner.uat)?; + Ok(mapping) } /// Queue a packet and return its fence at once. The scheduler can hand over further /// ready jobs up to its credit limit while the GPU runs. Only the completion worker /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full /// retirement. A burst can therefore share a firmware round trip across packets. - pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + pub(crate) fn submit(shared:&crate::m3::m3_drm::Shared,packet:Arc)->Result { crate::debug::update_debug_flags(); let mut guard=shared.lock(); let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; @@ -1069,14 +1084,14 @@ impl Runtime { if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} inner.pending.reserve(1,GFP_KERNEL)?; inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; - let drm=inner.drm.clone(); + let state=inner.state.clone(); drop(guard); - crate::m3_completion::queue(&drm); + crate::m3::m3_completion::queue(&state); Ok(()) } /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active /// commands remain owned until retirement; error signaling cannot release a DMA guard. - pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + pub(crate) fn withdraw(shared:&crate::m3::m3_drm::Shared,packet:&Arc)->bool { let mut guard=shared.lock(); let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; let inner: &mut Inner=&mut runtime.inner; @@ -1120,18 +1135,17 @@ impl Runtime { Err(_)=>{inner.fail_pending(EIO);Service::Idle}, } } - pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { - let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); - let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); - self.inner.boot_step = crate::t8122_start::BootStep::Publish; + pub(crate) fn boot(&mut self, pdev: &platform::Device>) -> Result { + let is_t8122 = crate::m3::t8122_start::is_t8122(self.inner.device.soc()); + self.inner.boot_step = crate::m3::t8122_start::BootStep::Publish; let result = self.boot_inner(pdev); if let Err(error) = result { // On T8122, log the firmware readiness words on every boot failure, so the verdict's // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. - let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + let logged = self.inner.boot_step == crate::m3::t8122_start::BootStep::AwaitReady && error == ETIMEDOUT; - if (is_t8122 || is_t6031) && !logged { + if is_t8122 && !logged { let inner: &mut Inner = &mut *self.inner; let _ = inner.config.log_ready(&inner.drm); } @@ -1139,18 +1153,13 @@ impl Runtime { } if is_t8122 { let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, - self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); - } - if is_t6031 { - let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + crate::m3::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); } result } - fn boot_inner(&mut self, pdev: &platform::Device) -> Result { - use crate::t8122_start::BootStep; + fn boot_inner(&mut self, pdev: &platform::Device>) -> Result { + use crate::m3::t8122_start::BootStep; let root = self.inner.config.root(); dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); self.inner.boot_step = BootStep::Publish; @@ -1174,7 +1183,7 @@ impl Runtime { inner.config.drain(&inner.drm)?; inner.config.log_ready(&inner.drm)?; if inner.device.soc().chip_id==0x6030 && - (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + (crate::m3::m3_params::active_power() || crate::m3::m3_params::globals_debug_off()) { inner.config.log_globals_debug(&inner.drm)?; } inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; @@ -1244,3 +1253,22 @@ pub(crate) fn physical_counter() -> u64 { if frequency == 0 { return 0; } (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency } + +/// Adopts the running firmware's UAT on `soc`: the `mmu::Uat` of the admitted reserved regions +/// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the +/// G15 backend does it on 7.2 (`g15::uat`). +fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, + resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + let cfg = soc.hwcfg.ok_or(ENODEV)?; + let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; + let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; + let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; + let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; + let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; + window.bootstrap(drm.as_ref().as_ref(), &uat)?; + // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be + // reserved System RAM mapped cacheable. + window.clean_tables(); + Ok((uat, window)) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index f19717d536f72..7be649415cccc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -2,48 +2,230 @@ //! The per-SoC facts of the M3 runtime backend. //! -//! Each table describes the configuration of one GPU. This compile-only subset holds only the -//! facts the M3 boundary carries; the register windows, memory layout, firmware images and -//! admission of the full table stay with the M3 runtime port. Nothing here is read by a probe. - -use super::m3_compute_layout::RegisterSet; - -/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor -/// device-tree values: configuration words where the manager layout keeps 0 or its validated -/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values -/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. -#[derive(Debug)] -pub(crate) struct G15RuntimeHwDataB { - pub(crate) unk_454: u32, - pub(crate) unk_464: u32, - pub(crate) unk_a7c: u64, - pub(crate) unk_a98: u32, - pub(crate) unk_abc: u64, - pub(crate) unk_ae4: u32, - pub(crate) unk_b20: u32, - pub(crate) unk_b24: u64, - /// +0xb40, the "UAT enabled" word of the manager layout. - pub(crate) unk_554: u32, - pub(crate) unk_17b8: u32, - /// Unit enable masks, +0x17c0/+0x17c8. - pub(crate) unit_mask_a: u64, - pub(crate) unit_mask_b: u32, - pub(crate) unk_1808: u32, - pub(crate) unk_1818: u32, +//! Each table describes the register windows, memory layout and configuration of one GPU. +//! Missing configuration is `None`; outstanding runtime requirements are listed in `unported`. +//! Admission rejects either before reading firmware, mapping GPU registers or starting the +//! GPU coprocessor ([`Soc::require_complete`]). + +use kernel::{c_str, device, prelude::*}; + +use crate::{ + m3::hw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_compute_layout::RegisterSet, + m3::m3_board::KnownImage, + m3::m3_firmware::Layout, + m3::m3_init_storage::IoMap, // +}; + +/// The SGX identification words the runtime admits, each compared under its mask. +pub(crate) struct IdWords { + /// SGX+0xd04000: family [31:24], variant [23:16], revision [15:8]. + pub(crate) version: u32, + pub(crate) version_mask: u32, + /// SGX+0xd04010: dies [19:16], clusters per die [15:8]. + pub(crate) counts: u32, + pub(crate) counts_mask: u32, } +/// The register windows the runtime admits, as CPU physical addresses (device-tree `reg` values +/// translated through `arm-io` `ranges`). +pub(crate) struct Windows { + /// Base of the GPU coprocessor window (`asc`, ADT `gfx-asc` reg[0]). + pub(crate) asc: u64, + /// Base of the GPU window (`sgx`, ADT `sgx` reg[0]). + pub(crate) sgx: u64, + /// Base of the GPU coprocessor mailbox, a 16 KiB window inside `asc`. + pub(crate) mailbox: u64, +} + +/// The performance-state table the runtime accepts: checked on the generated InitData before +/// any GPU register is touched, and again on the uploaded HwDataB. +#[derive(Copy, Clone, Debug)] +pub(crate) enum PstateTable { + /// A fixed table: `states` states above the off state, the highest at `top_mhz` MHz. + Fixed { states: u32, top_mhz: u32 }, + /// The table of the device tree's operating points: one state per distinct voltage above + /// the off state, the highest at the highest frequency + /// (`t8122_admission::opp_table_shape`). + DeviceTree, +} + +/// Optional userspace features advertised for a SoC (`DRM_ASAHI_GET_PARAMS`). The runtime +/// implements them the same way on every SoC; a SoC advertises one once it has been validated +/// there. Soft faults are never advertised: the runtime does not apply `asahi.fault_control`. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Features { + /// `DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY`. + pub(crate) compute_wide_visibility: bool, + /// `DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY`, while `asahi.m3_early_tiling` is set. + pub(crate) fragment_dependency: bool, +} + +/// The MTR temperature-sensor masks of the runtime's HwDataA, where the SoC table sets them +/// instead of the hardware configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct MtrMasks { + /// The fast-die sensor mask, in both HwDataA copies (+0x8ac and +0x1288). + pub(crate) fast_die: u64, + /// The mask the firmware matches an MTR alarm against (HwDataA +0x1a98). An alarm from a + /// sensor outside it, or any alarm while it is 0, is fatal to the firmware. + pub(crate) alarm: u64, +} + +/// Where the boot loader exports a SoC's decoded GPU leakage fuse values, in `/chosen`, and the +/// boot loader switch that makes them this boot's leakage. +#[derive(Copy, Clone, Debug)] +pub(crate) struct LeakFuse { + /// Two cells, value 1 then value 2, encoded as `apple,core-leak-coef` is. + pub(crate) values: &'static CStr, + /// The switch: the boot loader uses the values exactly when it holds the string "1" (the + /// two bytes `1\0`), the form it copies from its configuration line. + pub(crate) switch: &'static CStr, +} + +/// Where the runtime places the HwData object (`m3_init_storage::HARDWARE_DATA`), when it is not +/// the fixed allocation's address, and how the firmware maps it. +#[derive(Copy, Clone, Debug)] +pub(crate) struct HwDataObject { + /// Firmware VA of the object. + pub(crate) address: u64, + /// Map the object cacheable for the firmware (its atomic updates need cacheable memory) + /// instead of uncached. + pub(crate) cached: bool, +} + +/// One firmware IO mapping of the runtime's InitData: HwDataB slot, physical address, total +/// size, element size, writable. +pub(crate) type IoMapping = (usize, u64, u32, u32, bool); + +/// The M3 runtime's facts for one SoC. pub(crate) struct Soc { /// SoC name, for logs. pub(crate) name: &'static str, + /// GPU name, for logs. + pub(crate) gpu_name: &'static str, + pub(crate) chip_id: u32, + pub(crate) gpu_variant: hw::GpuVariant, + pub(crate) gpu_revision: hw::GpuRevision, + pub(crate) gpu_revision_id: hw::GpuRevisionID, + /// Root node compatible of the SoC. + pub(crate) board: &'static str, + /// GPU node compatible. + pub(crate) gpu: &'static str, + /// Boards the runtime starts on by default and registers a render node on. On a T8122 it + /// starts on no other board, whatever `asahi.m3_backend` says (`m3_params::t8122_backend`). + pub(crate) validated_boards: &'static [&'static [u8]], + /// GPU clusters, and core slots per cluster. + pub(crate) clusters: u32, + pub(crate) cores_per_cluster: u32, + /// Accepted compatible lists of the GPU coprocessor mailbox. + pub(crate) mailbox_compatibles: &'static [&'static [u8]], + /// The mailbox interrupts: send-empty, send-not-empty, recv-empty, recv-not-empty, as the + /// mailbox node's `interrupts` cells (three per interrupt, or four with the die cell of a + /// multi-die interrupt controller). + pub(crate) mailbox_interrupts: &'static [u32], + /// The register windows of the GPU node and its mailbox. + pub(crate) windows: Windows, + /// The identification words the runtime admits. + pub(crate) id: IdWords, + /// GPU firmware images the runtime can identify. + pub(crate) images: &'static [KnownImage], + /// The layout of the loaded firmware the runtime accepts. + pub(crate) firmware: Option<&'static Layout>, + /// The hardware configuration of the InitData. + pub(crate) hwcfg: Option<&'static hw::HwConfig>, + /// The firmware IO mappings of the InitData. + pub(crate) io_mappings: Option<&'static [IoMapping]>, + /// The runtime's mapping of each of those IO mappings: the CPU physical block and the + /// firmware VA it is mapped at. One per entry of `io_mappings`, covering it. + pub(crate) iomaps: Option<&'static [IoMap]>, + /// The performance-state table the runtime accepts. + pub(crate) pstates: PstateTable, + /// The SGX write (offset, value) made after the identity checks, before the firmware starts. + pub(crate) sgx_setup: Option<(usize, u32)>, /// The runtime InitData's HwDataB configuration words and unit masks. pub(crate) hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// The optional userspace features advertised. + pub(crate) features: Features, + /// Whether the power-management coefficients, the leakage coefficients and the operating + /// points are this machine's, added by the boot loader from its ADT, rather than static + /// device-tree values. Admission then requires every one of them (see + /// `m3_board::check_boot_loader_power`), with no driver default standing in except the + /// idle-off standby timer, which an ADT may lack. + pub(crate) power_from_boot_loader: bool, + /// Code the runtime runs after admission that still holds another SoC's values. + pub(crate) unported: &'static [&'static str], + /// The GPU registers read after every job by default (`asahi.m3_retire_mmio` bits). + pub(crate) retire_mmio: u64, + /// The MTR sensor masks, when they are not the hardware configuration's. + pub(crate) mtr_masks: Option, + /// The HwData object's placement and mapping, when they are not the fixed allocation's. + pub(crate) hwdata_object: Option, /// The register lists the GPU's firmware expects in compute and render commands. pub(crate) registers: RegisterSet, + /// Globals +0x7d0, which gates the firmware's frequency-feedback cap: the firmware measures + /// the GPU clock against the requested performance state and, while this is 1, caps the + /// state on a shortfall (`asahi.m3_ut_engagement` overrides it). + pub(crate) ut_engagement: u32, + /// The highest GPU power target, in mW, while the boot loader's power model is a stand-in: + /// the operating points' powers are scaled down together so that the highest is this. + pub(crate) power_target_cap_mw: Option, /// Words of the generated HwData object (offset into the object, value) that this SoC's /// firmware takes with other values than the shared builder writes. pub(crate) hwdata_words: &'static [(usize, u32)], /// The same for the Globals object. pub(crate) globals_words: &'static [(usize, u32)], + /// The boot loader's decoded GPU leakage fuse, when it exports one for this SoC. + pub(crate) leak_fuse: Option, + /// Whether each operating point may give each cluster its own voltage (the per-cluster, + /// binned voltage tables of T6031). Otherwise every cluster of a point must have the same. + pub(crate) per_cluster_voltages: bool, +} + +impl Soc { + /// Reject incomplete configuration before accessing the GPU. + pub(crate) fn require_complete(&self, dev: &device::Device) -> Result { + let mut missing = 0u32; + let mut note = |what: &str| { + dev_info!(dev, "M3 {}: missing {}\n", self.gpu_name, what); + missing += 1; + }; + if !self.images.iter().any(|image| image.initdata_magic.is_some()) { + note("the identity of the loaded GPU firmware image and its InitData version"); + } + if self.firmware.is_none() { + note("the layout of the loaded GPU firmware (segment sizes and VAs)"); + } + if self.hwcfg.is_none() { + note("the hardware configuration of the InitData (HwConfig)"); + } + if self.io_mappings.is_none() { + note("the firmware IO mappings of the InitData"); + } + if self.iomaps.is_none() { + note("the runtime's mapping of the firmware IO mappings (IO maps)"); + } + if self.sgx_setup.is_none() { + note("the SGX setup write made before the firmware starts"); + } + if self.hwdata_b.is_none() { + note("the runtime HwDataB configuration words and unit masks"); + } + for &what in self.unported { + note(what); + } + if missing == 0 { + return Ok(()); + } + dev_err!( + dev, + "M3 {}: {} configuration requirements missing; GPU startup disabled\n", + self.gpu_name, + missing + ); + Err(ENODEV) + } } /// The HwDataB configuration words and unit masks of the T6030 runtime InitData, as checked on @@ -77,24 +259,115 @@ pub(crate) static T8122_HWDATA_B: G15RuntimeHwDataB = G15RuntimeHwDataB { }; /// Every SoC the M3 runtime has a table for. -pub(crate) static SOCS: [&Soc; 2] = [&T6030, &T8122]; - -/// T6030 (M3 Pro, G15S): one die, two clusters of ten core slots. -pub(crate) static T6030: Soc = Soc { - name: "T6030", - hwdata_b: Some(&T6030_HWDATA_B), - registers: RegisterSet::G15S, - hwdata_words: &[], - globals_words: &[], -}; +pub(crate) static SOCS: [&Soc; 1] = [&T8122]; + +/// The table of the SoC with chip id `chip_id`, if the M3 runtime has one. +pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> { + SOCS.iter().copied().find(|soc| soc.chip_id == chip_id) +} /// T8122 (M3, G15G): one die, one cluster of ten core slots (eight or ten of them active). +/// +/// The compatibles, the mailbox and its interrupts are the T8122 device tree's. The identity is +/// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20, +/// core slots in the first core-mask word only), with the die count of the AGX3 identification +/// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration +/// comes from the boot loader. The rest of the runtime configuration is complete, but until the +/// runtime has passed on an M3 MacBook Air the SoC is refused, before the GPU is accessed, unless +/// `asahi.t8122_start=1` arms the start (`t8122_start`), which uses this table's values and adds a +/// performance-state cap. pub(crate) static T8122: Soc = Soc { name: "T8122", + gpu_name: "G15G", + chip_id: 0x8122, + gpu_variant: hw::GpuVariant::G, + gpu_revision: hw::GpuRevision::C0, + gpu_revision_id: hw::GpuRevisionID::C0, + board: "apple,t8122", + gpu: "apple,agx-t8122", + // The M3 MacBook Airs (13" J613, 15" J615). Allowed, not validated: the runtime has not run + // on either, and starts only behind a boot loader that hands the GPU over. + validated_boards: &[b"apple,j613", b"apple,j615"], + clusters: 1, + cores_per_cluster: 10, + mailbox_compatibles: &[ + b"apple,t8122-agx-asc-mailbox\0", + b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0", + ], + mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4], + // The J613 ADT: sgx reg[0] (child 0x80000000 + arm-io 0x210000000) and gfx-asc reg[0]; the + // mailbox at asc + 0x8000 (t8122-gpu.dtsi). The same addresses as on T6030. + windows: Windows { + asc: 0x2_9240_0000, + sgx: 0x2_9000_0000, + mailbox: 0x2_9240_8000, + }, + id: IdWords { + version: 0x0702_2000, + version_mask: 0xffff_ff00, + // One die, one cluster per die; the other fields are not known. + counts: 0x0001_0100, + counts_mask: 0x000f_ff00, + }, + images: &crate::m3::m3_board::KNOWN_IMAGES_T8122, + firmware: Some(&crate::m3::m3_firmware::T8122_LAYOUT), + hwcfg: Some(&hw::t8122::HWCONFIG_T8122), + io_mappings: Some(&crate::m3::m3_adt_config::T8122_IO_MAPPINGS), + iomaps: Some(&crate::m3::m3_adt_config::T8122_IOMAPS), + // The boot loader's ladder from this machine's ADT (J613: eight voltages, up to 1338 MHz). + pstates: PstateTable::DeviceTree, + // The same SGX setup write as on T6030. + sgx_setup: Some((0xd14000, 0x70001)), hwdata_b: Some(&T8122_HWDATA_B), + // Neither is validated on G15G yet; userspace keeps its default ordering and visibility. + features: Features { + compute_wide_visibility: false, + fragment_dependency: false, + }, + power_from_boot_loader: true, + unported: &[ + "a pass on an M3 MacBook Air: until then the GPU starts only with asahi.t8122_start=1", + ], + // The performance state only. On the single-cluster G15G, reading the engine-busy or the + // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s; + // the performance-state read alone does not, and a per-frame readback saw no job retired + // before its writes were visible. + retire_mmio: 4, + // The fast-die controller's sensors (0x4248, both copies), and every sensor the MTR block + // enables as the alarm mask (sensors 3, 6, 8, 9, 11 and 14: 0x4b48, the upper half of the + // block's configuration word 0x4b480003). With the second fast-die copy at 0, or an alarm + // mask without sensor 8, the firmware stopped on its first MTR alarm. + mtr_masks: Some(MtrMasks { + fast_die: 0x4248, + alarm: 0x4b48, + }), + // The firmware's MTR alarm handler does a 64-bit atomic update of HwDataA +0x4350. The fixed + // allocation ends the 0x8a04-byte object at its page end, leaving HwDataA (+0x4580) only + // 4-byte aligned, and the update took an alignment fault. Starting the object 0x8a80 bytes + // before the same page end puts HwDataA on a 16-byte boundary. Aligned, the update then + // faulted as an unsupported atomic on the uncached mapping: map it cacheable. + hwdata_object: Some(HwDataObject { + address: T8122_HWDATA_ADDRESS, + cached: true, + }), registers: RegisterSet::G15G, + // On a T8122 with the 14.8.3 system firmware the shader clock runs at 3/4 of every requested + // state, so with the cap engaged the GPU stays at state 2 (462 MHz effective) under any load. + // With 0 it reaches the requested states (state 8: about 1000 MHz effective). + ut_engagement: 0, + // The boot loader's T8122 power model is a stand-in that overstates the power (its highest + // operating point is about 30 W). Until each Mac's fused leakage gives a real one, scale it + // to a 22 W target, about the GPU power budget of an M3 MacBook Air. + power_target_cap_mw: Some(22_000), hwdata_words: &T8122_HWDATA_WORDS, globals_words: &T8122_GLOBALS_WORDS, + // Exported on a J613 on every boot; used only with the boot loader's switch, which also makes + // value 1 its apple,core-leak-coef and the base of its operating points' powers. + leak_fuse: Some(LeakFuse { + values: c_str!("asahi,t8122-gpu-leak-fuse"), + switch: c_str!("asahi,t8122-gpu-fuse-leakage"), + }), + per_cluster_voltages: false, }; /// The T8122 HwData words that differ from the shared builder's (offsets into the object; @@ -104,7 +377,7 @@ pub(crate) static T8122: Soc = Soc { /// +0x1804, +0x1814 and +0x1860 and all-ones at +0x1848); /// - HwDataA +0x11e0: 30 (T6030: 40), a word of the shader-engine controller block; /// - HwDataA +0x1290: 125, and HwDataA +0x424c: 24000000, the 24 MHz reference clock. -static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ +pub(crate) static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ (0xa30, 0), (0xa34, 4), (0x17e0, 1), @@ -126,4 +399,18 @@ static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ /// The T8122 Globals words that differ from the shared builder's: +0x9bc, the CDM backoff /// timeout (4) with the three bytes after it 0 (T6030: 1, 0, 0). -static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; +pub(crate) static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; + +/// The T8122 HwData object's firmware VA: 0x8a80 bytes before the end of the fixed allocation's +/// last page. +pub(crate) const T8122_HWDATA_ADDRESS: u64 = 0xffff_fc20_4070_4000 - 0x8a80; + +// The fixed HwData allocation (`m3_init_storage::allocation`): 0x8a04 bytes ending at the page +// end 0xfffffc2040704000. The moved object stays inside the same pages, and HwDataA is 16-byte +// aligned. +const _: () = { + let (fixed, size) = (0xffff_fc20_406f_b5fc_u64, 0x8a04_u64); + assert!(T8122_HWDATA_ADDRESS & !0x3fff == fixed & !0x3fff); + assert!(T8122_HWDATA_ADDRESS + size <= (fixed + size + 0x3fff) & !0x3fff); + assert!((T8122_HWDATA_ADDRESS + crate::m3::m3_adt_config::HWDATA_A as u64) % 16 == 0); +}; diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 9bd08d0257b0c..326e514808382 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; -use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use crate::{file,m3::agx_uapi,mmu,queue,m3::m3_drm::Shared}; use core::sync::atomic::{AtomicBool, Ordering}; #[derive(Default)] pub(crate) struct Completion; @@ -32,14 +32,14 @@ impl Destination { #[derive(Copy,Clone)] pub(crate) enum Command { - Compute(crate::agx_compute::Control), + Compute(crate::m3::agx_compute::Control), Render{command:agx_uapi::UapiRenderCommand,usc:u64}, } pub(crate) struct Packet { pub(crate) vm:mmu::Vm, pub(crate) commands:KVec, timestamps:KVec<[Option;4]>, - pub(crate) attachments:KVec, + pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, @@ -47,7 +47,7 @@ pub(crate) struct Packet { pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. pub(crate) owner:Owner, - vm_job:Pin>>>, + vm_job:Pin>>>, } /// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: /// a renamed task leaves a stale name, which is harmless. @@ -57,8 +57,8 @@ impl Owner { fn current()->Self { let task=kernel::current!(); let leader=task.group_leader(); - Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), - thread:crate::m3_client::task_comm(task)} + Self {pid:leader.pid(),process:crate::m3::m3_client::task_comm(leader), + thread:crate::m3::m3_client::task_comm(task)} } fn name(comm:&[u8;16])->&str { let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); @@ -84,18 +84,18 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. // On failure the runtime quarantines this guard with the packet until // firmware has provably stopped; signalling an error is not retirement. - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); - let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; self.completion.signal(); if profile { - let after=crate::m3_runtime::physical_counter(); + let after=crate::m3::m3_runtime::physical_counter(); let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); @@ -113,7 +113,7 @@ impl sched::JobImpl for Job { fn run(job:&mut Self)->Result> { // Queue the packet and return its fence; the completion worker publishes and retires // it, so the scheduler can hand over further ready jobs meanwhile. - if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + if let Err(e)=crate::m3::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { pr_err!("M3 scheduler: submission failed {:?}\n",e); // submit() failed before this packet could be published. job.packet.release_guard();job.packet.finish(Err(e)); @@ -122,9 +122,9 @@ impl sched::JobImpl for Job { } fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { // Retain the publishing branch's explicit diagnostic override. - if !crate::m3_params::timeout_nohang() { + if !crate::m3::m3_params::timeout_nohang() { if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; } let guard=job.shared.lock(); @@ -138,12 +138,12 @@ impl sched::JobImpl for Job { // device loss or corrupting an already signaled fence. if runtime.health().healthy() { return sched::Status::NoHang; } drop(guard); - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(EIO)); sched::Status::NoDevice } fn cancel(job:&mut Self) { - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ECANCELED)); } } @@ -178,9 +178,9 @@ impl Queue { } } impl queue::Queue for Queue { - fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { - let vm_job=self.vm.retain_m3_job()?; + let vm_job=self.vm.retain_job()?; let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { @@ -195,7 +195,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.vertex_timestamps.end)?, Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; - flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -217,10 +217,10 @@ impl queue::Queue for Queue { pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); EINVAL })?; - crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + crate::m3::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; - commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, @@ -238,8 +238,8 @@ impl queue::Queue for Queue { } fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) - -> Result { - use crate::agx_attachments::{Attachments, CAPACITY}; + -> Result { + use crate::m3::agx_attachments::{Attachments, CAPACITY}; use agx_uapi::{GpuAddressSpace, GpuAccess}; if list.as_slice().len()>CAPACITY {return Err(E2BIG);} let mut ranges = [(0, 0); CAPACITY]; @@ -262,7 +262,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; // Early tiling (render flag bit 5) only where it is advertised. - let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + let stage_scope=if early_tiling && crate::m3::m3_params::early_tiling() {1<<5} else {0}; if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) || r.sampler_count != 0 || r.sampler_heap != 0 || r.vertex_helper != empty || r.fragment_helper != empty @@ -279,7 +279,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { return Err(EINVAL); } - crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs index 59a72187c9699..e30f8fe011332 100644 --- a/drivers/gpu/drm/asahi/m3/m3_thermal.rs +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -47,7 +47,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use kernel::{ - bindings, c_str, device, + c_str, device, error::to_result, new_spinlock, prelude::*, @@ -58,9 +58,9 @@ use kernel::{ }; use crate::{ - m3_adt_config::PstatePolicy, - m3_params::{self, ThermalMode}, - m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, + m3::m3_adt_config::PstatePolicy, + m3::m3_params::{self, ThermalMode}, + m3::m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, }; /// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors @@ -78,7 +78,7 @@ fn required_zones() -> u32 { let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { return J516_REQUIRED_ZONES; }; - crate::m3_thermal_policy::required_zones(&compatible) + crate::m3::m3_thermal_policy::required_zones(&compatible) } /// Sensor polling period while the GPU is busy and while it is idle, in ms. @@ -122,13 +122,18 @@ impl_has_delayed_work! { impl HasDelayedWork for Sensor { self.work } } +extern "C" { + // include/linux/thermal.h; not in 7.2's generated bindings. + fn thermal_zone_get_temp_by_name(name: *const core::ffi::c_char, temp: *mut i32) -> core::ffi::c_int; +} + /// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. fn read_zone(name: &CStr) -> Result { let mut temp: i32 = 0; // SAFETY: name is NUL-terminated and temp is a valid output pointer. The // thermal core acquires a device reference under its registry lock and // keeps it through sampling, so zone unregistration cannot free it here. - to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + to_result(unsafe { thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; Ok(temp) } diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 3d900a80fef38..1d10657c02f66 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -43,6 +43,96 @@ pub(crate) fn refuse_probe(pdev: &platform::Device>, chip: Chip) -> Err ENODEV } +/// Whether the M3 runtime may start on this T8122 (M3, G15G), whose GPU the boot loader handed +/// over: its identity gate passes (only DT properties are read) and the backend selection allows +/// the board. The runtime then admits the boot loader's description, and refuses the SoC unless +/// `asahi.t8122_start=1` arms the start ([`t8122_start`]). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) fn t8122_admitted(pdev: &platform::Device>) -> Result<&'static m3_soc::Soc> { + let identity = t8122_admission::read_identity(pdev).map_err(|error| { + dev_info!( + pdev.as_ref(), + "G15G: T8122 identity or ABI rejected ({:?}); startup refused\n", + error + ); + ENODEV + })?; + dev_info!( + pdev.as_ref(), + "G15G: T8122 revision 2.0 identity admitted ({} active / {} slots)\n", + identity.active_cores, + identity.core_slots + ); + if !m3_params::t8122_backend(pdev) { + return Err(ENODEV); + } + Ok(&m3_soc::T8122) +} + +// The T8122 runtime, ported from the 7.1.12 release-20 kernel (c46d7dcdb871). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_attachments; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_status; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_uapi; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod g15_raw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod hw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod identity; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod initdata; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_adt_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_board; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_client; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_completion; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_device; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_drm; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_firmware; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_params; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pass; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pool; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_render; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_rtkit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_runtime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_submit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_thermal; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_knobs; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_start; + mod m3_compute_layout; mod m3_compute_sequence; @@ -103,6 +193,8 @@ mod m3_fragment_command; mod t8122_admission; -mod m3_soc; +// The runtime's SoC tables; they name the runtime's own modules. +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_soc; mod t8122_hwdata; diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs index f9c3ff08006d5..b48e561f677ef 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -172,7 +172,7 @@ fn given(param: u64, default: u64) -> Option { /// Resolve the parameters. The first parameter with a value it does not accept is the refusal. pub(crate) fn resolve(raw: &Raw) -> Result { let refuse = |name, accepts| Refusal { name, accepts }; - let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + let initdata_version = given(raw.initdata_version, crate::m3::m3_firmware::G15_V14_8_3_INITDATA) .filter(|v| *v != 0) .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; let fender = match given(raw.fender, FENDER_RULE as u64) { diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index cf0ee658e8da5..ca9f21b56dbb6 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -27,13 +27,13 @@ use core::mem::offset_of; use kernel::{device, prelude::*}; use crate::{ - fw::initdata::raw, - initdata::G15RuntimeHwDataB, - m3_adt_config::T8122_IO_MAPPINGS, - m3_init_storage::{self as storage, IoMap}, - m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + m3::g15_raw::raw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_adt_config::T8122_IO_MAPPINGS, + m3::m3_init_storage::{self as storage, IoMap}, + m3::m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, pgtable::{prot, Prot}, - t8122_knobs::{ + m3::t8122_knobs::{ self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, FENDER_RULE, }, @@ -140,7 +140,7 @@ const _: () = { assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the // opt-in itself; if another is added, this fails the build until the start accounts for it. - assert!(crate::m3_soc::T8122.unported.len() == 1); + assert!(crate::m3::m3_soc::T8122.unported.len() == 1); }; /// The values of one armed boot. @@ -149,7 +149,7 @@ pub(crate) struct Experiment(Values); /// Whether `soc` is the T8122 table. pub(crate) fn is_t8122(soc: &Soc) -> bool { - core::ptr::eq(soc, &crate::m3_soc::T8122) + core::ptr::eq(soc, &crate::m3::m3_soc::T8122) } impl Experiment { @@ -182,7 +182,7 @@ impl Experiment { /// they are the T8122 table's own IO maps. pub(crate) fn iomaps(&self) -> &'static [IoMap] { if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { - return &crate::m3_adt_config::T8122_IOMAPS; + return &crate::m3::m3_adt_config::T8122_IOMAPS; } &IOMAPS[self.variant()][..self.mapping_count()] } @@ -245,7 +245,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> if !is_t8122(soc) { return Ok(None); } - let raw = crate::m3_params::t8122_params(); + let raw = crate::m3::m3_params::t8122_params(); match knobs::start(&raw) { Start::On => {} Start::Off => { @@ -354,7 +354,7 @@ pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, sta /// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { - if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + if is_t8122(soc) && knobs::start(&crate::m3::m3_params::t8122_params()) == Start::On { dev_err!( dev, "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index e868e30baca4c..dc828a460202d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -117,6 +117,8 @@ mod bind; mod context; mod shared; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3; mod residency; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs new file mode 100644 index 0000000000000..a63101a813426 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The VM and binding helpers of the M3 T8122 runtime (`crate::m3`), which 7.1.12's `mmu.rs` +//! provided for it. + +use super::*; + +impl Vm { + /// Whether `other` is the same DRM VM. + pub(crate) fn same(&self, other: &Vm) -> bool { + self.id == other.id + } + + /// Copy the GPU-readable bytes this VM maps at `iova`, through the point of coherency + /// ([`Vm::read_user`]). M3 compute validates the client control stream before publishing it. + pub(crate) fn read_bytes(&self, iova: u64, out: &mut [u8]) -> Result { + if out.is_empty() { + return Err(EINVAL); + } + self.read_user(iova, out) + } + + /// Snapshot the first `out.len()` TTBAT entries as `(ttb0, ttb1)` pairs. The TTBAT is + /// ordinary memory, so this reads no GPU register and is safe with the cores gated. + pub(crate) fn context_roots(&self, out: &mut [(u64, u64)]) -> Result { + let inner = lock_vm(&self.gpuvm, None, false)?; + let shared = inner.uat_inner.lock(); + let ttbs = shared.ttbs(); + for (slot, entry) in out.iter_mut().enumerate().take(UAT_NUM_CTX) { + *entry = ( + ttbs[slot].ttb0.load(Ordering::Acquire), + ttbs[slot].ttb1.load(Ordering::Acquire), + ); + } + Ok(()) + } + + /// Map `size` bytes of MMIO or DMA memory at `phys` into this Vm, at `iova`, or at a free + /// address in `range` when `iova` is `None`. Unlike [`Vm::map_io`] this logs nothing on + /// success: the runtime maps job views on every submission. + fn map_io_m3_inner( + &self, + iova: Option, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + if size == 0 || (iova.unwrap_or(0) as usize | phys | size) & UAT_PGMSK != 0 { + pr_err!( + "MMU: M3 KernelMapping {:#x}:{:#x} -> {:#x?} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + // SAFETY: As in `map_io`, the exec lock is only used for this mapping. + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + let inner = vm_exec.unique_gpuvm.data(); + let payload = KernelMappingInner { + owner: self.gpuvm.clone(), + uat_inner: inner.uat_inner.clone(), + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }; + let node = match iova { + Some(iova) => inner.mm.reserve_node(payload, iova, size as u64, 0)?, + None => inner.mm.insert_node_in_range( + payload, + size as u64, + UAT_PGSZ as u64, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?, + }; + let iova = node.start(); + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; + let mutation = inner.mapping_mutation(); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } + // Drop the exec lock before the node, as in `map_io`. + drop(mutation); + let _ = inner; + ret?; + Ok(KernelMapping(node)) + } + + /// [`Vm::map_io`] at `iova`, without the per-mapping log line. + pub(crate) fn map_io_m3( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(Some(iova), 0..0, phys, size, prot) + } + + /// [`Vm::map_io`] at a free address in `range`. + pub(crate) fn map_io_in_range( + &self, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(None, range, phys, size, prot) + } +} + +impl VmBind { + /// The bound `Vm` itself, for diagnostics that need to walk its tables. + pub(crate) fn vm(&self) -> &Vm { + &self.0 + } + + /// Whether this retained binding belongs to the supplied DRM VM. + pub(crate) fn matches(&self, vm: &Vm) -> bool { + self.0.id == vm.id + } + + /// Physical base of the bound VM's page-table root. + pub(crate) fn root(&self) -> u64 { + self.0.ttb() + } + + /// The bound VM's driver-assigned id. + pub(crate) fn vm_id(&self) -> u64 { + self.0.id + } +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 5f7e572a3a462..87ec3bd73658b 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -318,6 +318,18 @@ impl RtKit { }) } + /// Returns whether the coprocessor is running (not crashed, IOP and AP power states on). + pub fn is_running(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_running(self.rtk) } + } + + /// Returns whether the coprocessor has reported a crash. + pub fn is_crashed(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_crashed(self.rtk) } + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index 35ff049ff0783..095a51627edfe 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -90,6 +90,28 @@ impl Completion { self.inner.get() } + /// Publish one completion token, including from interrupt context. + #[inline] + pub fn complete(&self) { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::complete(self.as_raw()) }; + } + + /// Consume one pending token without blocking. + #[inline] + pub fn try_wait_for_completion(&self) -> bool { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::try_wait_for_completion(self.as_raw()) } + } + + /// Wait for a token for at most `timeout` jiffies; return whether one arrived. + #[inline] + pub fn wait_for_completion_timeout(&self, timeout: crate::time::Jiffies) -> bool { + let timeout = timeout.min(core::ffi::c_long::MAX as _); + // SAFETY: The completion remains live while this method waits. + unsafe { bindings::wait_for_completion_timeout(self.as_raw(), timeout as _) != 0 } + } + /// Signal all tasks waiting on this completion. /// /// This method wakes up all tasks waiting on this completion; after this operation the From e16c5bbae3f8f6cd101fc4890e040c50f1aeeee8 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:52 -0400 Subject: [PATCH 004/392] drm/asahi: m3: map each VM's dummy page where the firmware expects it 7.1.12 mapped each VM's dummy page at the backend's unknown_page(), which the M3 runtime returned as 0x2ff_ffff_8000, the address its InitData gives the firmware (unk_30). On 7.2 file.rs maps it at uat_geometry().unk_page() instead, which is 0x3ff_ffff_8000 with the runtime's 42-bit geometry, so the firmware's unknown page was unmapped in every T8122 VM. Add a gpu::Gpu::unknown_page() hook, defaulting to the geometry's page so no other GPU changes, and have the M3 backend return RUNTIME_UNKNOWN_PAGE, which also ends its user range, so the two can't drift again. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/gpu.rs | 4 ++++ drivers/gpu/drm/asahi/m3/initdata.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 9 ++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 242f6cdcd3f83..f2f055931d29a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -549,7 +549,7 @@ impl File { } let dummy_mapping = dummy_obj.map_at( &vm, - gpu.uat_geometry().unk_page(), + gpu.unknown_page(), mmu::PROT_GPU_SHARED_RW, true, )?; diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b9283efb470a9..a06b5da5ed6b1 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -344,6 +344,10 @@ pub(crate) trait Gpu: Send + Sync { fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } + /// GPU VA of each VM's dummy page, which must match the address the firmware was given. + fn unknown_page(&self) -> u64 { + self.uat_geometry().unk_page() + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index 2348d075a731c..ab0e1643468f7 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -122,7 +122,7 @@ impl G15Options { /// Start of the USC (shader code) window in the runtime backend's GPU VA layout. const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; /// The runtime backend's unknown-page address, the end of its 42-bit user VA range. -const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; +pub(crate) const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; /// Maximum number of entries of a firmware performance-state table, including the off state. const PSTATE_TABLE_ENTRIES: usize = 16; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 2d93823e67d01..dc8f1a074e809 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -232,7 +232,14 @@ impl gpu::Gpu for Backend { value as u32 } fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } - fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn user_range(&self) -> Range { + mmu::UAT_PGSZ as u64..crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } + /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the + /// 42-bit geometry's own unk_page() is a different address. + fn unknown_page(&self) -> u64 { + crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } fn tracked_submission(&self) -> bool { true } /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { From eae5bfa6b7d34549e157bb3ec77276c3ca6e7868 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 005/392] drm/asahi: m3: quarantine a failed job's VM before signalling its fence Packet::finish signalled an error fence while the packet could still hold its VM job guard, and nothing on the M3 path quarantined it, so after a timeout userspace could unmap and close memory that work the firmware had not provably stopped was using. Unmap invalidates the TLB, so this ended in a GPU fault rather than freed memory, as on 7.1.12, but 7.2's shared job lifetime can block it. As G15 does, quarantine a guard that is still held before the error fence becomes visible: mapping changes wait until the runtime drops the packet. Guards already released (work never published, or retired) are unaffected. Fix the comment that said this already happened. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_submit.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 326e514808382..ca16e18f10fbf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -84,12 +84,19 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} + if let Err(e)=result { + // A guard still held here belongs to work the firmware may still run: block + // mapping changes before the error fence is visible (as G15 does), so userspace + // can't unmap memory under it. It stays with the packet until the runtime drops + // the packet, after the firmware has provably stopped. + if let Some(guard)=self.vm_job.lock().as_ref() {guard.quarantine();} + if let Some(status) = self.vm.status() { status.record(e); } + self.completion.set_error(e); + } // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. - // On failure the runtime quarantines this guard with the packet until - // firmware has provably stopped; signalling an error is not retirement. + // Signalling an error is not retirement. crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; From c9dcc611f3f1646e49a5451a9004a7744b7d02f0 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 006/392] drm/asahi: m3: keep the adopted UAT until ASC stops on a failed start Once adopt_uat() points TTBAT context 0 at the kernel-owned lower root, a failure in Config::new() or the window sync dropped the UAT and its window inside the start closure, before stop_asc(), and the branch that retains everything when ASC fails to stop could not retain tables that were already freed. Keep both outside the closure: drop them only after ASC has stopped, and leak them with the rest when it does not. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index b12fc73f9f248..28bbea3c86889 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -997,6 +997,9 @@ impl Runtime { } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; + // The adopted UAT and its window live outside the closure: once TTBAT context 0 names + // the kernel-owned lower root, a failure must not free those tables before ASC stops. + let mut adopted=None; let prepared=(|| -> Result<_> { device.start_asc(pdev)?; Pin::new(&mut transport).wake()?; @@ -1010,23 +1013,28 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; - let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. - window.sync_all(&uat)?; + window.sync_all(uat)?; crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); - Ok((uat,window,config)) + Ok(config) })(); - let (uat,window,config)=match prepared { - Ok(v)=>v, - Err(e)=>{ + let (uat,window,config)=match (prepared,adopted.take()) { + (Ok(config),Some((uat,window)))=>(uat,window,config), + (prepared,adopted)=>{ + let e=prepared.err().unwrap_or(EINVAL); crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(adopted); core::mem::forget(drm);core::mem::forget(device); + } else { + // ASC has stopped: the firmware no longer walks the adopted tables. + drop(adopted); } return Err(e); } From 34b754f9d5cf09417d043ed1c77aafdf03041ad2 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:54 -0400 Subject: [PATCH 007/392] drm/asahi: m3: don't let the compute stream debug read fail a job Compute::new() read 48 bytes of the client's CDM stream for a debug message, whatever the stream's length, and failed the job if the read did. A short stream ending at a mapped page boundary with the next page unmapped passed the stream checks but then failed its first submission, even with debug logging off (as on 7.1.12). Read no further than the stream, and only log when the read succeeds. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index eab8bf71f7bcd..34aa0c0d2cd3f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -96,9 +96,12 @@ impl Compute { objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); // The M3 compute stream uses absolute IOTO addresses and does not // include the USC_EXEC_BASE_CP register. + // Diagnostic only: read no further than the stream, and never fail the job over it. let mut stream=[0u8;48]; - vm.read_bytes(control.base,&mut stream)?; - crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + let len=core::cmp::min(control.end.saturating_sub(control.base),stream.len() as u64) as usize; + if vm.read_bytes(control.base,&mut stream[..len]).is_ok() { + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",&stream[..len]); + } agx_memory::publish(); // Finish the write-combining page-table stores before broadcasting // the invalidation. From 1694a19aa4e7eb37fc2ff58b9e7c3dae20e6a6e5 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:24:02 -0500 Subject: [PATCH 008/392] Prepare the original context for the carried M3 timestamp dependency The following original G17 sync commit includes the M3 timestamp adapter. Restore current Aurora integration in the separate adaptation afterwards. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/asahi_core.rs | 100 +- drivers/gpu/drm/asahi/driver.rs | 259 +-- drivers/gpu/drm/asahi/file.rs | 273 +-- drivers/gpu/drm/asahi/g15/command.rs | 40 +- drivers/gpu/drm/asahi/g15/mod.rs | 69 +- drivers/gpu/drm/asahi/g15/uat.rs | 14 +- drivers/gpu/drm/asahi/g17/buffer.rs | 80 +- drivers/gpu/drm/asahi/g17/buffer/target.rs | 137 +- drivers/gpu/drm/asahi/g17/capacity.rs | 93 - drivers/gpu/drm/asahi/g17/command.rs | 50 +- drivers/gpu/drm/asahi/g17/completion.rs | 330 +--- drivers/gpu/drm/asahi/g17/context.rs | 245 +-- drivers/gpu/drm/asahi/g17/context/identity.rs | 265 +-- drivers/gpu/drm/asahi/g17/data_segment.rs | 56 - drivers/gpu/drm/asahi/g17/dependency.rs | 132 -- drivers/gpu/drm/asahi/g17/event.rs | 16 +- drivers/gpu/drm/asahi/g17/feed.rs | 98 +- drivers/gpu/drm/asahi/g17/freelist.rs | 178 +- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 64 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 85 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 157 +- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 619 ------ drivers/gpu/drm/asahi/g17/fw/kick.rs | 30 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/queue.rs | 137 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 377 +--- drivers/gpu/drm/asahi/g17/fw/variant.rs | 140 -- drivers/gpu/drm/asahi/g17/initdata.rs | 489 ++--- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 230 --- drivers/gpu/drm/asahi/g17/job.rs | 143 +- drivers/gpu/drm/asahi/g17/kick.rs | 459 +---- drivers/gpu/drm/asahi/g17/mod.rs | 838 ++------ drivers/gpu/drm/asahi/g17/object.rs | 208 +- drivers/gpu/drm/asahi/g17/power.rs | 205 +- drivers/gpu/drm/asahi/g17/preparation.rs | 249 +-- drivers/gpu/drm/asahi/g17/queue/admission.rs | 821 +------- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1238 ++---------- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 32 +- .../drm/asahi/g17/queue/compute/retirement.rs | 623 +----- drivers/gpu/drm/asahi/g17/queue/mod.rs | 209 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 1087 ++--------- .../gpu/drm/asahi/g17/queue/render/memory.rs | 140 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 44 +- .../drm/asahi/g17/queue/render/publication.rs | 42 +- .../drm/asahi/g17/queue/render/retirement.rs | 189 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/recovery.rs | 229 +-- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 131 -- drivers/gpu/drm/asahi/g17/recovery/slot.rs | 8 +- drivers/gpu/drm/asahi/g17/replay.rs | 170 -- drivers/gpu/drm/asahi/g17/runtime.rs | 1684 ++--------------- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 196 +- .../drm/asahi/g17/runtime/compute_control.rs | 31 +- .../drm/asahi/g17/runtime/compute_replay.rs | 26 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 164 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 460 +---- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 13 +- .../drm/asahi/g17/runtime/render/observe.rs | 118 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 46 +- .../drm/asahi/g17/runtime/render/prepare.rs | 87 +- .../drm/asahi/g17/runtime/render/replay.rs | 148 +- .../drm/asahi/g17/runtime/render/settle.rs | 30 +- .../drm/asahi/g17/runtime/render/submit.rs | 572 +----- .../drm/asahi/g17/runtime/render/teardown.rs | 61 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 25 +- drivers/gpu/drm/asahi/g17/teardown.rs | 163 -- drivers/gpu/drm/asahi/g17/timeout.rs | 326 +--- drivers/gpu/drm/asahi/g17/validation.rs | 179 +- drivers/gpu/drm/asahi/gem.rs | 101 +- drivers/gpu/drm/asahi/gpu.rs | 123 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 - drivers/gpu/drm/asahi/hw/t6040.rs | 216 --- drivers/gpu/drm/asahi/hw/t8140.rs | 41 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 48 +- drivers/gpu/drm/asahi/memacct.rs | 549 ------ drivers/gpu/drm/asahi/mmu.rs | 1259 +++--------- drivers/gpu/drm/asahi/mmu/bind.rs | 17 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 79 +- drivers/gpu/drm/asahi/mmu/ppl.rs | 71 - drivers/gpu/drm/asahi/mmu/residency.rs | 71 +- drivers/gpu/drm/asahi/mmu/shared.rs | 4 +- drivers/gpu/drm/asahi/object.rs | 42 +- drivers/gpu/drm/asahi/pgtable.rs | 449 ++--- drivers/gpu/drm/asahi/queue/common.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 84 +- drivers/gpu/drm/asahi/regs.rs | 75 +- drivers/gpu/drm/asahi/sysfs.c | 196 +- drivers/gpu/drm/asahi/sysfs.h | 23 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 96 +- drivers/gpu/drm/asahi/timestamp.rs | 220 --- drivers/gpu/drm/asahi/util.rs | 11 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 46 +- include/drm/drm_gem_shmem_helper.h | 15 +- include/uapi/drm/asahi_drm.h | 127 +- rust/bindings/bindings_helper.h | 24 - rust/helpers/mm.c | 12 - rust/kernel/drm/device.rs | 27 +- rust/kernel/drm/file.rs | 86 - rust/kernel/drm/gem/shmem.rs | 487 ++--- rust/kernel/mm.rs | 53 - 102 files changed, 2606 insertions(+), 18228 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs delete mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs delete mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs delete mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs delete mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs delete mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs delete mode 100644 drivers/gpu/drm/asahi/g17/replay.rs delete mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs delete mode 100644 drivers/gpu/drm/asahi/memacct.rs delete mode 100644 drivers/gpu/drm/asahi/mmu/ppl.rs delete mode 100644 drivers/gpu/drm/asahi/timestamp.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..0fbe6e728773d 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,7 +4,6 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; -mod binding; mod buffer; mod channel; mod cleanup; @@ -14,20 +13,19 @@ mod event; mod file; mod float; mod fw; -mod fwid; +mod gem; +mod g17; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; -mod g17; -mod gem; - -#[cfg(CONFIG_DRM_ASAHI_G16)] -mod g16; +// Without the G15 backend, only the UAT adoption the T8122 runtime shares with it. +#[cfg(all(CONFIG_DRM_ASAHI_UAT_ADOPTION, not(CONFIG_DRM_ASAHI_G15)))] +#[path = "g15/adopt.rs"] +mod g15; mod gpu; mod hw; mod initdata; -mod m3; mod mem; -mod memacct; +mod m3; mod microseq; mod mmu; mod object; @@ -36,10 +34,7 @@ mod queue; mod regs; mod slotalloc; mod stats; -mod stats_raw; mod sysfs_exports; -mod timestamp; -mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -51,33 +46,12 @@ use kernel::prelude::*; type G15Registration = kernel::driver::Registration>>; -#[cfg(CONFIG_DRM_ASAHI_G16)] -type G16Registration = kernel::driver::Registration>; -#[cfg(not(CONFIG_DRM_ASAHI_G16))] -type G16Registration = (); - -fn register_g16(module: &'static kernel::ThisModule) -> Result>>> { - #[cfg(CONFIG_DRM_ASAHI_G16)] - { - Ok(Some(KBox::pin_init( - kernel::driver::Registration::new(g16::DRIVER_NAME, module), - GFP_KERNEL, - )?)) - } - #[cfg(not(CONFIG_DRM_ASAHI_G16))] - { - let _ = module; - Ok(None) - } -} - #[pin_data] struct DriverModule { #[cfg(CONFIG_DRM_ASAHI_G15)] #[pin] _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. - _g16: Option>>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -87,18 +61,15 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { - // SAFETY: Module init runs once, before either adapter is registered below. - unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), + _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, ), - _g16: register_g16(module)?, // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), @@ -108,10 +79,9 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "aurora", + name: "asahi", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", - alias: ["asahi"], params: { debug_flags: u64 { default: 0, @@ -173,6 +143,22 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", + }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before a PB reclaim, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: PB blocks posted above which an idle engine reclaims the PB", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", @@ -186,41 +172,5 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, - mem_limit_pct: u32 { - default: 67, - description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", - }, - client_mem_limit_pct: u32 { - default: 50, - description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", - }, - oom_attribution: u8 { - default: 1, - description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", - }, - g16_deadline_ms: u32 { - default: 5000, - description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", - }, - g16_compute_deadline_ms: u32 { - default: 5000, - description: "G16: compute completion-stamp deadline, ms (0 permits unbounded compute; firmware recovery stays active)", - }, - g16_repoll_us: u32 { - default: 15, - description: "G16: repoll interval while publication waits for record consumption, us", - }, - g16_resnap_us: u32 { - default: 100, - description: "G16: completion repoll interval, us", - }, - g16_recovery: u32 { - default: 1, - description: "G16: answer qualified firmware recovery handshakes", - }, - g16_stream_gate: u32 { - default: 1, - description: "G16: validate CDM/VDM stream records", - }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 6227bb4bf815b..43fa17412815d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,13 +45,11 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { - binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) g16: Option>, + /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -85,18 +83,12 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata -/// -/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h -/// and only changes if that ABI ever breaks. The minor version goes up with -/// each new ioctl, flag or feature bit, so userspace can tell what a kernel -/// accepts without probing. The patch level marks fixes userspace may need to -/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 1, + major: 0, minor: 0, patchlevel: 0, - name: c"aurora", - desc: c"Aurora GPU for Apple AGX", + name: c"asahi", + desc: c"Apple AGX Graphics", }; /// DRM Driver implementation for `AsahiDriver`. @@ -111,6 +103,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -150,82 +143,68 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on -// its entries, so the entries that the native G15 backend owns when it is -// built are passed in by configuration. -macro_rules! asahi_of_device_table { - ($($g15_owned:tt)*) => { - kernel::of_device_table!( - OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - $($g15_owned)* - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] - ); - }; -} - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -asahi_of_device_table!( - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), +// OF Device ID table. +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] ); -#[cfg(CONFIG_DRM_ASAHI_G15)] -asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -233,16 +212,19 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + fn unbind<'bound>( + _pdev: &'bound platform::Device>, + this: Pin<&Self::Data<'bound>>, + ) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { + // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } - this.close_binding(); } /// Device probe function. @@ -251,7 +233,6 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); - crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -263,7 +244,6 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; - // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -280,9 +260,6 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -314,20 +291,12 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); - let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - // The DRM core created this card's debugfs directory when the device was registered; - // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and - // waits for its readers, before the registration data that owns the snapshot is dropped. - if let Some(snapshot) = debug_stats.as_ref() { - crate::stats_raw::register(reg.device(), snapshot); - } - // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -335,17 +304,14 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && module_parameters::stats_export.value() != 0, + stats.is_some() && *module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev, raw_stats, stats, @@ -365,9 +331,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -377,13 +340,10 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -402,7 +362,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -417,9 +376,6 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { - binding, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -439,9 +395,6 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -457,75 +410,20 @@ impl AsahiPlatformDriver { return Err(error); } }; - let stats = gpu::Gpu::stats_snapshot(&*gpu); + // G17's firmware channel format has no supported stats decoder yet. let raw_dev = pdev.as_ref().as_raw(); - let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); - let raw_stats = match crate::sysfs_exports::register_g17( - raw_dev, - snapshot, - module_parameters::stats_export.value() != 0, - ) { + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; - if let Err(error) = - crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) - { - dev_warn!( - pdev.as_ref(), - "idle_off_delay_ms not registered: {:?}\n", - error - ); - } Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, - raw_dev, - raw_stats, - stats, - }) - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl AsahiPlatformDriver { - pub(crate) fn probe_g16<'bound>( - pdev: &'bound platform::Device>, - cfg: &'static crate::g16::Config, - ) -> Result> { - // SAFETY: the qualified UAT output width constrains this device's DMA. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - let resources = regs::Resources::new(pdev)?; - let res = regs::G16Resources::new(pdev)?; - let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; - let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiDrmRegistrationData { - gpu: data_gpu, - resources, - }); - // SAFETY: registration is stored in bound platform data and never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; - Ok(AsahiPlatformDriverData { - binding, - _device: reg.device().into(), - _reg: reg, - g17: None, - m3: None, - g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -533,16 +431,9 @@ impl AsahiPlatformDriver { } } -impl AsahiPlatformDriverData<'_> { - pub(crate) fn close_binding(&self) { - self.binding.close(); - } -} - impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - self.binding.close(); - // `self.raw_dev` was set in `probe` from the platform + // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..f2f055931d29a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,6 @@ use crate::{ AsahiDrmDevice, // }, gem, - gpu, - memacct, mmu, queue, util::{ @@ -45,7 +43,6 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; -use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -71,7 +68,6 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, - gpu: Arc, } impl Drop for Vm { @@ -93,7 +89,6 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); - self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -127,23 +122,10 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, - pub(crate) cumulative: Option, - pub(crate) timeline: bool, } impl SyncItem { - fn parse_one( - file: &DrmFile, - mut data: uapi::drm_asahi_sync, - out: bool, - bundled: bool, - ) -> Result { - if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { - return Err(EINVAL); - } - data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; - } + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -164,8 +146,6 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, - cumulative: None, - timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -188,10 +168,12 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: None, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, timeline_value: data.timeline_value, - cumulative: None, - timeline: true, }) } _ => { @@ -206,52 +188,28 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, - allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; - let mut anchor = None; - // Read each record once so validation and use see the same retained bytes. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !allow_cumulative || i < in_count || anchor.is_some() { - return Err(EINVAL); - } - anchor = Some(i as usize); - } - vec.push( - Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, - GFP_KERNEL, - )?; - } - if let Some(anchor) = anchor { - let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], - anchor - in_count as usize, - )?; - vec[anchor].cumulative = Some(bundle); - } else { - for sync in &mut vec[in_count as usize..] { - if sync.timeline { - sync.chain_fence = Some(dma_fence::FenceChain::new()?); - } - } + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; } Ok(vec) @@ -260,7 +218,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -275,10 +233,6 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, - /// GPU memory this client created and holds. - mem: Arc, - /// Completed engine time, retained by queues after handle destruction. - usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, @@ -301,35 +255,12 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); - let mem = memacct::Client::new()?; - let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) - }) - } - - /// The objects this client created, which `memacct` attributes to it, not every object it - /// has a handle to: an imported or shared buffer is counted once, by its creator. - fn memory_stats(&self) -> Option { - Some(drm::file::MemoryStats { - private: self.mem.committed(), - resident: Some(self.mem.resident()), - ..Default::default() + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) }) } - fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { - let names = [ - kernel::c_str!("vertex"), - kernel::c_str!("fragment"), - kernel::c_str!("compute"), - ]; - for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { - emit(name, ns); - } - } - fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -339,41 +270,25 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new( - id: u64, - user_range: Range, - mem: Arc, - usage: Arc, - ) -> impl PinInit { - // SAFETY: The initializer supplies aligned, pinned storage for Self. - // XArray::new is infallible; each pinned array is initialized in place, - // then every remaining field is written before returning success. + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - pin_init::raw_try_init( - raw_vms, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; let raw_queues = addr_of_mut!((*slot).queues); - pin_init::raw_try_init( - raw_queues, - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ), - )?; + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; let raw_objects = addr_of_mut!((*slot).objects); - pin_init::raw_try_init( - raw_objects, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).mem).write(mem); - addr_of_mut!((*slot).usage).write(usage); addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) @@ -426,7 +341,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as usize), + UserPtr::from_addr(data.pointer as _), core::mem::size_of_val(&words), ) .writer() @@ -445,11 +360,8 @@ impl File { return Err(ENODEV); } - let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, - memory_limit: memory_limits.device, - client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -469,13 +381,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // Write the retained result to this user slice once. + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; file.inner() @@ -490,18 +402,15 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as usize); + let user = UserPtr::from_addr(data.pointer as _); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size).writer().write(&[ - vm_id, - 0, - status.reported_error() as u32, - 0, - ])?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } @@ -512,7 +421,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -583,22 +492,7 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; - // G16 registers a retained backend owner before returning this VM. Any later - // allocation can fail before file::Vm exists to run its normal close path. - let failed_vm = ScopeGuard::new_with_data( - (vm.clone(), user_range.clone(), kernel_range.clone()), - |(vm, user_range, kernel_range)| { - if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { - vm.close_idle_reclaim(); - if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { - pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); - } - vm.bo_deferred_cleanup(); - gpu.vm_closed(&vm); - } - }, - ); + let vm = gpu.new_vm(kernel_range.clone())?; let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -674,12 +568,10 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, - gpu: gpu.clone(), }, GFP_KERNEL, )?)?; - drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -688,7 +580,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -697,6 +589,7 @@ impl File { Err(ENOENT) } else { drop(vm); + reg_data.gpu.vm_closed(); Ok(0) } } @@ -739,48 +632,11 @@ impl File { None }; - let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER - .admit(&file.inner().mem, size as u64) - .inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; - // Large G17 user objects are populated in full by the first VM_BIND. - // Do not multiply direct reclaim/compaction latency by their folio - // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = - gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); - let gem = gem::new_object( - device, - size, - data.flags, - resv_obj, - reclaim_cpu, - pages_no_reclaim, - commit, - )?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; + if reg_data.gpu.prepare_new_objects() { - #[cfg(CONFIG_DRM_ASAHI_G15)] - if reg_data.gpu.as_any().is::() { - gem::prepare_new_object_g15(&gem)?; - } else { - let registered = device.registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let registered = device.registration_guard().ok_or(ENODEV)?; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - { - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } + gem::prepare_new_object(&gem)?; } - let handle = gem.create_handle(file)?; data.handle = handle; @@ -857,17 +713,11 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 - || data.num_binds > limit - || data.userptr == 0 - || data.stride % 8 != 0 - || data.stride as usize > kernel::page::PAGE_SIZE - { - cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); + if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 + || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { return Err(EINVAL); } } - if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -878,7 +728,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -896,7 +746,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1344,33 +1194,11 @@ impl File { return Err(EINVAL); } - // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and - // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority is not supported on this GPU\n" - ); - return Err(EINVAL); - } - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - // SAFETY: capable() reads the current ioctl task's credentials. - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } - && !file.is_current_master() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" - ); - return Err(EACCES); - } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1391,7 +1219,6 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, - file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1433,9 +1260,6 @@ impl File { } let gpu = ®_data.gpu; - if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { - return Err(E2BIG); - } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1468,7 +1292,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as usize), + UserPtr::from_addr(data.cmdbuf as _), data.cmdbuf_size as usize, ) .reader() @@ -1480,13 +1304,8 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = SyncItem::parse_array( - file, - data.syncs, - data.in_sync_count, - data.out_sync_count, - gpu.as_any().is::(), - )?; + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; mod_dev_dbg!( device, @@ -1550,7 +1369,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = u64::from(gpu.base_clock_hz()); + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 66e52e6981197..45b65d1ddc026 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,11 +146,6 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } -struct CheckedContext<'a> { - vm: &'a mut mmu::VmReadGuard, - usc_base: u64, -} - /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -366,7 +361,6 @@ impl Timestamps<'_> { .ok_or(ENOENT)? .clone(); let file::Object::TimestampBuffer(mapping) = object; - let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); @@ -382,13 +376,7 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover( - vm: &mut mmu::VmReadGuard, - start: u64, - end: u64, - read: bool, - write: bool, -) -> Result { +pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -418,11 +406,7 @@ pub(crate) fn cover( } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments( - vm: &mut mmu::VmReadGuard, - list: &[uapi::drm_asahi_attachment], - write: bool, -) -> Result { +fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { for attachment in list { let end = attachment .pointer @@ -482,8 +466,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = &mut *context.vm; +fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -567,11 +551,11 @@ fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_rend /// Per-command compute admission. fn check_compute( - context: &mut CheckedContext<'_>, + context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = &mut *context.vm; + let vm = context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -665,12 +649,6 @@ impl Prepared<'_> { counts, timestamps, } = self; - // On error, unlock before the last timestamp mapping can drop and unmap. - let mut access = context.vm.read_lock()?; - let mut context = CheckedContext { - vm: &mut access, - usc_base: context.usc_base, - }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -679,12 +657,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(&mut context, &cmd)?; + check_render(context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -699,7 +677,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(&mut context, &cmd, &compute)?; + check_compute(context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index ae0512de0b156..95f04114ba874 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for -//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting -//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait +//! for the native display and PMP supplier before booting firmware. T6031 retains explicit +//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -144,23 +143,22 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &&hw::t6030::CONFIG, + &hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &&hw::t6031::CONFIG, + &hw::t6031::CONFIG, ), ]); -// Probe admission requires the native display before firmware starts (T6030 -// also its PMP supplier), so both may be loaded at device creation. +// T6030 probe admission requires the native display and PMP supplier before +// firmware starts. T6031 retains explicit loading until it has an equivalent +// display-order admission gate. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - @none, - [ - of::DeviceId::new(c_str!("apple,agx-t6030")), - of::DeviceId::new(c_str!("apple,agx-t6031")), - ] + MODULE_OF_TABLE_G15, + (), + [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -179,12 +177,6 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { - if let Some(driver) = &this._driver { - driver.close_binding(); - } - } - /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -219,6 +211,7 @@ impl platform::Driver for Driver<'_> { }, }) } + } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -381,7 +374,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and the client aliases of every user VM. + /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -446,7 +439,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -459,18 +452,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain()?; + gpu.retain(); Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once; `boot::start` already took it before touching the - /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) -> Result { - boot::glue::retain_module() + pub(crate) fn retain(&self) { + boot::glue::retain_module(); } } @@ -545,21 +537,7 @@ impl Firmware { vms, }, ); - let result = firmware.init_graphs().and_then(|_| { - if cfg.platform != hw::Platform::AdtWithPmp { - return Ok(()); - } - // A report before firmware startup is ACKed without enabling bandwidth scaling. - // Reassert it after setup, including the backend's final device control. - // SAFETY: This session retains its acknowledged T6030 AGX power vote. - let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; - if ret == 0 { - Ok(()) - } else { - Err(Error::from_errno(ret)) - } - }); - if let Err(e) = result { + if let Err(e) = firmware.init_graphs() { let Firmware { session, init, @@ -654,7 +632,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self, _vm: &mmu::Vm) { + fn vm_closed(&self) { vm::closed(self) } @@ -669,10 +647,9 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, + queue::Queue::new(self, vm, priority, usc_exec_base)?, GFP_KERNEL, )?) } @@ -681,7 +658,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) + ) -> Result { + timestamp::map(self, bo, range) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 0f91f7c9f0610..7ab3a1919eaec 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,10 +25,8 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement. Failed published jobs retain their slots until -//! their last reference drops (until reboot for jobs retained after a fail-stop). -//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` +//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -38,12 +36,14 @@ use kernel::{ bindings, device, io::Io, prelude::*, - sync::Arc, time::{delay::fsleep, Delta, Instant, Monotonic}, }; +#[cfg(CONFIG_DRM_ASAHI_G15)] +use kernel::sync::Arc; use super::hw; use super::regs::{mb, rmb, wmb, Mapping}; +#[cfg(CONFIG_DRM_ASAHI_G15)] use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{mmu, pgtable}; @@ -69,6 +69,7 @@ const fn indices(va: u64) -> [usize; 3] { } /// Creates the adopted UAT over the ADT regions (step 2 above). +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn adopt( dev: &AsahiDevice, cfg: &'static super::Config, @@ -98,7 +99,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(None, pa, count, f) } + unsafe { pgtable::with_root_entries(pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of @@ -201,6 +202,7 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index fd6460db2d1fb..d9b0c887deea0 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 + .ok_or(ENOSPC)? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; + let slot = selected.ok_or(ENOSPC)?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,8 +556,6 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, - /// A tiling pass initialized this buffer manager (25G83 InitBM published). - initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -659,40 +657,14 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), - initialized: false, }) } - - /// Whether a published tiling pass initialized this manager (25G83 InitBM). - pub(crate) fn initialized(&self) -> bool { - self.initialized - } - - pub(crate) fn mark_initialized(&mut self) { - self.initialized = true; - } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } - /// Read-only diagnostic while firmware is halted. In particular this - /// does not reset the high-water counter as automatic_target() does. - pub(crate) fn report_recovery(&self) { - let read = |offset| -> Result { - Ok(self.graph.word(offset)?.load(Ordering::Relaxed)) - }; - pr_warn!("G17: recovery tvb id={} blocks={} cursors={:?} pages={:?} capacity={:?} producer={:?} consumer={:?} max_pages={:?} reset={:?} initialized={}\n", - self.buffer_id, self.blocks(), self.cursors(), - read(core::mem::offset_of!(fw::State, pages)), - read(core::mem::offset_of!(fw::State, block_capacity)), - read(core::mem::offset_of!(fw::State, producer)), - read(core::mem::offset_of!(fw::State, consumer)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), - self.initialized); - } pub(crate) fn state_va(&self) -> u64 { self.graph.gpu_va() } @@ -1147,51 +1119,3 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } - -impl BufferIds { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", - self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, - self.entries.iter().filter(|entry| entry.owner).count(), - self.entries.iter().filter(|entry| entry.references != 0).count()) - } -} -impl MetricsIds { - pub(crate) fn used(&self) -> u32 { - self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() - - (128 - fw::METRICS_SLOTS as u32) - } -} - -#[kunit_tests(asahi_g17_buffer_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn failed_build_and_fault_retirement_return_parameter_buffers() { - let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); - for _ in 0..8 { - let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); - for _ in 0..BUFFER_IDS { - clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); - } - assert_eq!(ids.allocate_render(), Err(ENOSPC)); - for (index, id) in clients.into_iter().enumerate() { - if index % 2 == 0 { - // Failed constructor never installed the graph owner. - ids.release(id).unwrap(); - } else { - ids.reserve_owner(id).unwrap(); - ids.release(id).unwrap(); - ids.acquire_owner(id).unwrap(); - // Close cannot return a still-owned failed command. - ids.unreserve_owner(id).unwrap(); - assert!(!ids.entries[id as usize].free()); - ids.release(id).unwrap(); - } - } - assert!(ids.entries.iter().all(|entry| entry.free())); - assert_eq!(ids.returned_count, BUFFER_IDS); - } - } -} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bd96ecc9d460e..ed0f3c71a4c93 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::{RegisterPrograms, RenderPass}, + fw::render::RenderPass, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,17 +25,6 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } -/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at -/// the default parameter-buffer sizes. Sizes are -/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. -const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; -const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, GTP6_SIZE, 0x80]; -/// Room for the tiler's records across a partial render. macOS uses 0x1c00, so this is a superset, -/// and the size every partial-render test ran with. -const GTP_SLOTS: usize = 64; -const GTP6_SIZE: usize = GTP_SLOTS - * (((cfg::TVB_MAX_BLOCKS * cfg::TVB_BLOCK_SIZE).div_ceil(0x8000) + 0x17ff) / 24 * 8 + 8); - /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -44,18 +33,9 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, - /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an - /// empty one; the TPC is entry 0. - gtp: [Option; 8], - /// Bytes of the TPC object the command uses. - tpc_end: usize, } impl Layout { - pub(crate) fn new( - pass: &RenderPass, - clusters: u32, - programs: RegisterPrograms, - ) -> Result { + pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -110,51 +90,16 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); - let mut gtp = [None; 8]; - let mut tpc_end = tpc_size; - if programs == RegisterPrograms::V25G83 { - let sizes = [ - tpc_size, - tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, - if layers > 1 { 0x4000 } else { 0 }, - 0, - GTP_FIXED_SIZES[0], - GTP_FIXED_SIZES[1], - GTP_FIXED_SIZES[2], - GTP_FIXED_SIZES[3], - ]; - let mut position = 0; - for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { - position = aligned(position, alignment)?; - if size != 0 { - *offset = Some(position); - } - position = position.checked_add(size).ok_or(EOVERFLOW)?; - } - tpc_end = position; - } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, - gtp, - tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object - // also contains GTP sub-buffers, notably 64 times the tile map. Applying - // the cache limit to their combined extent rejects valid large targets. - // Keep the original cache and tile-record limits; they also bound the - // derived auxiliary extent before any backing allocation or replacement. - if aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN) > cfg::TPC_MAX - || self.record_stride()? > cfg::TARGET_RECORD_MAX - { - return Err(ERANGE); - } - Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -203,7 +148,7 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if stride > cfg::TARGET_RECORD_MAX { + if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { return Err(ERANGE); } let records = Object::new(dev, vm, stride * DEPTH)?; @@ -224,21 +169,6 @@ impl Target { self.tpc.address } - /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty - /// or absent one. The TPC object holds them after the TPC. - pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { - if layout.tpc_end > self.tpc.backing.size() { - return Err(ERANGE); - } - let mut addresses = [0; 8]; - for (address, offset) in addresses.iter_mut().zip(layout.gtp) { - if let Some(offset) = offset { - *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; - } - } - Ok(addresses) - } - /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { @@ -287,6 +217,9 @@ impl Target { let stride = layout.record_stride()?; let mut grew = false; if tpc > self.tpc.backing.size() { + if tpc > cfg::TPC_MAX { + return Err(ERANGE); + } Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; grew = true; } @@ -362,59 +295,3 @@ impl Target { } } } - - -#[kernel::macros::kunit_tests(asahi_g17_target_capacity)] -mod tests { - use super::*; - - fn pass(size: u32, utile: u8) -> RenderPass { - // SAFETY: RenderPass and its nested surfaces/programs contain only - // integers and bools, for which the all-zero representation is valid. - let mut pass: RenderPass = unsafe { core::mem::zeroed() }; - pass.width = size; - pass.height = size; - pass.layers = 1; - pass.utile_width = utile; - pass.utile_height = utile; - pass - } - - #[test] - fn t6040_auxiliary_tiler_storage_is_not_tail_pointer_cache() { - // CTS point_size_10000 renders to 10240 squared. Its GTP sub-buffer - // is required by the existing 25G83 program, not a larger cache. - for utile in [16, 32] { - let layout = Layout::new(&pass(10240, utile), 2, RegisterPrograms::V25G83).unwrap(); - assert!(layout.tpc_size < cfg::TPC_MAX); - assert!(layout.record_stride().unwrap() < cfg::TARGET_RECORD_MAX); - assert!(layout.tpc_end > cfg::TPC_MAX); - assert!(layout.tpc_capacity().unwrap() >= layout.tpc_end); - assert_eq!(layout.gtp[0], Some(0)); - assert!(layout.gtp[1].unwrap() >= layout.tpc_size); - } - } - - #[test] - fn core_and_tile_record_limits_still_reject_oversized_targets() { - let mut layout = Layout::new(&pass(10240, 32), 2, RegisterPrograms::V25G83).unwrap(); - layout.tpc_size = cfg::TPC_MAX + 1; - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - let mut layered = pass(16384, 16); - layered.layers = 2; - let layout = Layout::new(&layered, 2, RegisterPrograms::V25G83).unwrap(); - assert!(layout.record_stride().unwrap() > cfg::TARGET_RECORD_MAX); - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - } - - #[test] - fn neo_keeps_the_same_cache_extent_and_limit() { - let mut layout = Layout::new(&pass(10240, 32), 1, RegisterPrograms::G17P).unwrap(); - assert!(layout.gtp.iter().all(Option::is_none)); - assert_eq!(layout.tpc_capacity().unwrap(), - aligned(layout.tpc_size, mmu::UAT_PGSZ).unwrap().max(cfg::TPC_MIN)); - layout.tpc_size = cfg::TPC_MAX + 1; - layout.tpc_end = layout.tpc_size; - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs deleted file mode 100644 index 57406e5487a37..0000000000000 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Name capacity refusals at their source, before a common submit or fence -//! failure handler loses their provenance. Counters are device-lifetime -//! independent diagnostics, never admission or ownership state. - -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::prelude::*; - -#[derive(Clone, Copy, Debug)] -pub(super) enum Site { - ComputeReserve, - PhysicalQids, - RenderSlots, - LogicalQueues, - ComputeSlots, - QosIds, - ComputeItems, - RetiredRenderBindings, - RenderPoolIds, - RenderWorkSlots, - ParameterBuffers, - MetricsIds, - LowerVa, - UpperVa, - AliasVa, - ExecutionContexts, -} - -const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; -static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; - -/// First and power-of-two refusals per site keep a pressure storm bounded in -/// the log. This does not change whether the caller retries or rejects work. -#[track_caller] -pub(super) fn refused(site: Site) -> Error { - let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); - if count.is_power_of_two() { - let at = core::panic::Location::caller(); - pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, - at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); - } - ENOSPC -} - -/// Read-only diagnostics hold the live device session while sampling its pools. -/// They never wake firmware, reclaim a resource, or change admission state. -pub(super) struct View(pub(super) kernel::sync::Arc); - -impl kernel::debugfs::Writer for View { - fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), - u8::from(self.0.crashed.load(Ordering::Acquire)))?; - let uat = { - let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; - writeln!(f, "online=1")?; - let (converted, published) = super::sync_input::counters(); - writeln!(f, "fw_inputs converted={} published={} native={} pending={}", - converted, published, super::sync_input::native_count(), - super::sync_input::pending_count())?; - writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; - firmware.queues.write_capacity(f)?; - firmware.pool.write_capacity(f)?; - writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; - firmware.uat.clone() - }; - // Mapping destruction can issue firmware flushes while holding VM - // locks. Never take a VM/TTBAT lock with the device mutex held here. - // The retained UAT owns valid allocator metadata even during shutdown. - writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; - let mut upper = uat.geometry().kernel_range(); - upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; - for (name, vm, range) in [ - ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), - ("dynamic_upper_bytes", uat.kernel_vm(), upper), - ] { - match vm.capacity(range.clone()) { - Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, - Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, - } - } - for (site, count) in [ - Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, - Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, - Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, - Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, - ].iter().zip(REFUSALS.iter()) { - writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index c65f7de6ac915..3dfa760969224 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,8 +12,6 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; -const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); - const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -123,7 +121,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -133,7 +131,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if u32::from(header.cmd_type) + let payload = if header.cmd_type as u32 == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -240,7 +238,7 @@ impl UscWindow { } require_range( space, - self.base + u64::from(tagged_offset & !7), + self.base + (tagged_offset & !7) as u64, 4, Access::Read, ) @@ -308,14 +306,12 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { - long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { - long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -327,19 +323,6 @@ pub(crate) enum Validated { }, } -impl Validated { - pub(crate) fn long_job(&self) -> bool { - match self { - Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, - } - } - pub(crate) fn allow_long_job(&mut self) { - match self { - Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, - } - } -} - impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -351,7 +334,6 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, - variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -425,7 +407,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -458,11 +440,6 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { - if (stage.intersection.enabled() || stage.scratch.enabled()) - && !variant.ray_traversal - { - return Err(refuse_ray_traversal()); - } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -470,8 +447,6 @@ impl Payload { } } Ok(Validated::Render { - long_job: super::timeout::select(false, - ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -479,8 +454,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; - if cmd.cdm_ctrl_stream_base & 3 != 0 + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -495,19 +470,14 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; - let scratch = ScratchRequest::new(cmd.scs_layout)?; - if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { - return Err(refuse_ray_traversal()); - } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { - long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch, + scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -519,9 +489,3 @@ impl Payload { } } } - -/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). -fn refuse_ray_traversal() -> Error { - kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); - Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) -} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 90f6d77780bff..a3888f98231db 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,12 +13,7 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{ - driver::AsahiDrmDevice, - file, - hw::t8140::{dynamic, resources as cfg}, - mmu, -}; +use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -32,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // The buffer keeps its GPU alias and its CPU mapping alive. - buffer: Arc, - offset: usize, + // Keep the existing GPU alias alive along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, } impl Destination { @@ -50,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(buffer) = object; + let file::Object::TimestampBuffer(mapping) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { return Err(ERANGE); } - let offset = buffer.timestamp_offset(offset)?; - Ok(Some(Self { buffer, offset })) + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { _mapping: mapping, cpu })) } fn write(&self, ticks: u64) { @@ -66,14 +61,13 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.buffer.write_timestamp(self.offset, ns); + self.cpu.write(ns); barrier(Ordering::Release); } } -/// Timestamp destinations resolved and retained before any command is enqueued. Dropping -/// one releases only a buffer reference; the buffer defers its own mapping release. -pub(crate) struct Destinations([Option; 4]); +/// Timestamp mappings resolved and retained before any command is enqueued. +pub(crate) struct Destinations(Option; 4]>>); impl Destinations { pub(crate) fn resolve( @@ -88,15 +82,20 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - Ok(Self(destinations)) - } - - pub(crate) fn has_outputs(&self) -> bool { - self.0.iter().any(Option::is_some) + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) } fn iter(&self) -> core::slice::Iter<'_, Option> { - self.0.iter() + let destinations: &[Option] = + self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() } } @@ -145,8 +144,7 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a write-combined page of its own per -/// compute command. +/// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -168,84 +166,6 @@ impl ComputeTimestamps { } } -/// Compute timestamp pages of one context, kept mapped for its next commands. -/// -/// A fresh page costs a GEM object, its write-combined kernel mapping and a -/// GPU mapping on the submission path, and their teardown when the command's -/// completion is dropped. A page comes back here only from that drop, which -/// follows exact retirement or a kill witness, so the firmware no longer -/// writes it; both words are cleared before the page is handed out again. -#[pin_data] -pub(crate) struct ComputeTimestampCache { - #[pin] - pages: Mutex>, -} - -impl ComputeTimestampCache { - /// Pages retained per context; deeper pipelines allocate the rest. - const RETAINED: usize = 8; - - pub(crate) fn new() -> Result> { - let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - pages <- new_mutex!(pages, "G17 compute timestamp pages"), - }), - GFP_KERNEL, - ) - } - - fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { - let reused = self.pages.lock().pop(); - let page = match reused { - Some(page) => { - page.object.dword(0)?.store(0, Ordering::Relaxed); - page.object.dword(8)?.store(0, Ordering::Relaxed); - barrier(Ordering::SeqCst); - page - } - None => ComputeTimestamps::new(dev, vm)?, - }; - Ok(PooledTimestamps { - page: Some(page), - cache: self.clone(), - }) - } - - fn put(&self, page: ComputeTimestamps) { - let rejected = { - let mut pages = self.pages.lock(); - pages.push_within_capacity(page).err() - }; - // Teardown of a page beyond the retained count runs unlocked. - drop(rejected); - } -} - -/// A compute timestamp page lent to one command, returned to its context's -/// cache when the command's completion is dropped. -struct PooledTimestamps { - page: Option, - cache: Arc, -} - -impl PooledTimestamps { - fn page(&self) -> Result<&ComputeTimestamps> { - self.page.as_ref().ok_or(EINVAL) - } -} - -impl Drop for PooledTimestamps { - fn drop(&mut self) { - if let Some(page) = self.page.take() { - self.cache.put(page); - } - } -} - -/// Republications of one command whose earlier publications firmware dropped unexecuted. -pub(crate) const MAX_REPLAYS: u8 = 3; - #[pin_data] pub(crate) struct Completion { status: Arc, @@ -253,10 +173,8 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, - usage: Arc, - sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -267,21 +185,11 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, - /// A recovery blamed this command. Unlike `class`, which a later - /// classification can change, this never clears. - blamed: AtomicBool, - /// A firmware restart listed this command as in flight. Until a replay - /// republishes it from the start, it never reports success. - interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { - #[expect( - clippy::too_many_arguments, - reason = "Completion construction takes distinct queue, context, and lifetime owners." - )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -292,28 +200,23 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, - sources: Option>, - usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute + && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { - fence, - member, - destinations, - }; + let output = Output { fence, member, destinations }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { HostOutput::Immediate(output) }; let compute = if compute { - Some(context.compute_timestamps().take(dev, context.vm())?) + Some(ComputeTimestamps::new(dev, context.vm())?) } else { None }; @@ -324,8 +227,6 @@ impl Completion { terminal, compute, feed, - usage, - sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -334,8 +235,6 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), - blamed: AtomicBool::new(false), - interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -347,9 +246,7 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal - .as_ref() - .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -373,62 +270,22 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } - pub(super) fn sources(&self) -> Option<&Arc> { - self.sources.as_ref() - } - pub(super) fn is_input_source(&self, source: &Arc) -> bool { - self.output.output().member.is_source(source) - } - pub(super) fn set_source_ready(&self, ready: Fence) { - self.output.output().member.set_source_ready(ready); - } - pub(super) fn needs_source_ready(&self) -> bool { - self.output.output().member.needs_source_ready() - } - pub(super) fn note_input_error(&self, error: Error) -> bool { - self.output.output().member.note_error(error) - } - /// Called under the device mutex as a successful retirement is queued. - pub(super) fn note_source_retired(&self) { - if self.interrupted() { - // complete() fails it; consumers must not take its output as valid. - self.note_input_error(ECANCELED); - return; - } - match self.check_source_inputs() { - Ok(()) => self.output.output().member.note_retired(), - Err(error) => { - self.note_input_error(error); - } - } - } - pub(super) fn check_source_inputs(&self) -> Result { - if let Some(sources) = &self.sources { - sources.check()?; - } - Ok(()) - } - pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output - .output() - .destinations - .iter() - .any(Option::is_some) + self.output.output().destinations.iter().any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -448,14 +305,11 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < MAX_REPLAYS).then_some(count + 1) + (count < 3).then_some(count + 1) }) .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); - // Callers replay only work proven not to have started; the new - // publication runs from the beginning and reports its own result. - self.interrupted.store(false, Ordering::Release); } accepted } @@ -481,7 +335,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -530,49 +384,6 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } - pub(crate) fn guilty(&self) -> bool { - self.class.load(Ordering::Acquire) == 2 - } - - /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, - /// so it is never published again. - pub(crate) fn note_blamed(&self) { - self.blamed.store(true, Ordering::Release); - } - - pub(crate) fn blamed(&self) -> bool { - self.blamed.load(Ordering::Acquire) - } - - /// A firmware restart listed this command in its recovery table: it was in - /// flight, so it may have run partly. Its stamps and completion record may - /// still look like success. Unless a replay republishes it from the start, - /// it fails and the failure is always reported to the client. - pub(crate) fn note_interrupted(&self) { - self.interrupted.store(true, Ordering::Release); - } - - pub(crate) fn interrupted(&self) -> bool { - self.interrupted.load(Ordering::Acquire) - } - - /// Whether this failure makes the VM unusable. Spared work, and interrupted - /// work no recovery found guilty, fail without poisoning it. - pub(crate) fn poisons(&self, error: Error) -> bool { - let interrupted = self.interrupted(); - (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) - } - - /// ECANCELED stays silent only for work that was never interrupted (closed - /// or cancelled by its owner); every other failure reaches the client. - fn report(&self, error: Error) { - if self.poisons(error) { - self.status.record(error); - } else if error != ECANCELED || self.interrupted() { - self.status.report_failure(error); - } - } - /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -590,12 +401,6 @@ impl Completion { Ok(()) } - /// An ioctl publication that met contention returns its lease before the - /// scheduler publishes the command. Nothing referred to it yet. - pub(crate) fn detach_render_lease(&self) -> Option { - self.render.lock().take() - } - pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -603,26 +408,16 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } - /// Called under the device mutex once the render's kicks are committed. - pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { - if let Some(lease) = self.render.lock().as_ref() { - lease.note_publication(fragment); - } - } - /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(self.spared()); + lease.quarantine(); } } /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { - if let Some(sources) = &self.sources { - sources.release(); - } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -631,15 +426,7 @@ impl Completion { drop(lease); } - /// Whether the command still holds its VM pin: firmware may still own or replay it. - pub(crate) fn retains_vm(&self) -> bool { - self.vm_job.lock().is_some() - } - fn release_healthy(&self) { - if let Some(sources) = &self.sources { - sources.release(); - } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -647,40 +434,19 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { - if result.is_ok() { - if let Err(error) = self.check_source_inputs() { - result = Err(error); - } - } + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { if self.signalled.swap(true, Ordering::AcqRel) { return; } - if result.is_ok() && self.interrupted() { - result = Err(ECANCELED); - } - if let Ok(ticks) = &result { - use crate::usage::Engine; - let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; - if self.compute.is_some() { - self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); - } else { - self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); - self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); - } - } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - // Compute reports its dispatch envelope in the first pair, a - // render its fragment pass in the second. - if self.compute.is_some() { - feed.note_pass(ticks[0], ticks[1]); - } else { - feed.note_pass(ticks[2], ticks[3]); - } + feed.note_render_pass(ticks[2], ticks[3]); } if let Err(error) = result { - // Spared work cancelled after another VM's recovery is not this client's device loss. - self.report(error); + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { + self.status.record(error); + } } self.release_healthy(); self.signal_terminal(result); @@ -694,9 +460,7 @@ impl Completion { return; } self.status.record_if_device_loss(error); - // As in complete(): a cancellation is not this client's device loss. A guilty VM - // already carries its own sticky error. - if self.accepted.load(Ordering::Acquire) && error != ECANCELED { + if self.accepted.load(Ordering::Acquire) { self.status.report_failure(error); } self.release_healthy(); @@ -715,11 +479,9 @@ impl Completion { guard.quarantine(); } if self.spared() { - if error != ECANCELED || self.interrupted() { - self.status.report_failure(error); - } - } else { - self.report(error); + self.status.report_failure(error); + } else if error != ECANCELED { + self.status.record(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 8a6d9b572dec0..6bc62a7d44bee 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,9 +14,8 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; -use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; -use super::job::Engine; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use kernel::{ new_mutex, prelude::*, @@ -40,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(super::capacity::refused(super::capacity::Site::QosIds)); + return Err(ENOSPC); } *used |= 1u128 << id; Ok(QosLease { @@ -82,19 +81,6 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } - /// Whether the firmware completed every kick counted so far on node `word`. - fn drained(&self, word: u32) -> Result { - let slot = (word & 63) as usize; - if self.words.get(slot) != Some(&Some(word)) { - return Err(EINVAL); - } - let completed = self - .object - .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? - .load(Ordering::Relaxed); - fence(Ordering::Acquire); - Ok(completed >= self.counts[slot]) - } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -128,39 +114,18 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, - /// A kill of this context was published, through a compute queue or on its render pairs' - /// behalf. A context is killed at most once. - kill_published: AtomicBool, - /// Advisory only: publication still checks the physical binding under the - /// device mutex. False skips an inline attempt known to need admission. - compute_binding_hint: AtomicBool, - /// Mixed-engine contexts keep the original inline publication path. - render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, - /// Signalled when a work state returns; publishers that found none wait on it. - state_returns: Arc, - /// The domain refused compute publishers wait on; signalled with `state_returns` when the - /// admission turn moves on. - compute_returns: Arc, - /// The order in which this context's ioctls take work-state nodes. - order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, - /// Timestamp pages of retired compute commands, mapped in this context's VM. - compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, } impl Context { - #[expect( - clippy::too_many_arguments, - reason = "A context takes its allocators, identities and both admission domains." - )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -168,8 +133,6 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, - state_returns: Arc, - compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -218,53 +181,25 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; - let order = - Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, scheduler, scheduler_generation, published: AtomicBool::new(false), - kill_published: AtomicBool::new(false), - compute_binding_hint: AtomicBool::new(false), - render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), - state_returns, - compute_returns, - order, work, work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), - compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } - pub(crate) fn compute_binding_hint(&self) -> bool { - self.render_seen.load(Ordering::Acquire) - || self.compute_binding_hint.load(Ordering::Relaxed) - } - /// This logical queue has submitted render work, so its compute may have a render prefix. - pub(crate) fn render_seen(&self) -> bool { - self.render_seen.load(Ordering::Acquire) - } - pub(crate) fn note_render(&self) { - if !self.render_seen.load(Ordering::Relaxed) { - self.render_seen.store(true, Ordering::Release); - } - } - pub(crate) fn note_compute_binding(&self, owned: bool) { - self.compute_binding_hint.store(owned, Ordering::Relaxed); - } - pub(crate) fn compute_timestamps(&self) -> &Arc { - &self.compute_timestamps - } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -312,28 +247,6 @@ impl Context { _work: self.work.clone(), } } - /// Called once the firmware consumed this context's scheduler release. If its work list - /// still links a node, the firmware keeps referencing that node: on Neo a render command - /// that ran behind a failed compute command is never unlinked, and a later context given - /// the same work storage (the object cache hands it out again) made the firmware recover - /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether - /// it was linked. - pub(crate) fn retain_linked_work(&self, pool: &Arc) -> bool { - // An unreadable head counts as linked: retaining is the safe side. - let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]); - if linked { - self.work.lock().object.retain_until_stop(pool); - } - linked - } - - /// The firmware may still walk this context's work list into memory it names: a node is - /// linked and the scheduler page shows no kill. Unreadable state counts as linked. - pub(crate) fn firmware_may_link(&self) -> bool { - let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed); - must_retain(self.work_head().ok(), self.work_head_va(), killed) - } - pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } @@ -348,16 +261,6 @@ impl Context { Ok(head) } - /// Record that a kill of this context was published (see `kill_published`). - pub(crate) fn note_kill_published(&self) { - self.kill_published.store(true, Ordering::Release); - } - - /// A kill of this context was published. - pub(crate) fn kill_published(&self) -> bool { - self.kill_published.load(Ordering::Acquire) - } - pub(crate) fn is_published(&self) -> bool { self.published.load(Ordering::Acquire) } @@ -424,60 +327,41 @@ impl Context { pub(crate) struct WorkStateLease { context: Arc, word: AtomicU32, - /// Index of the engine whose admission took the node, valid once `word` is set. - engine: AtomicU8, - /// The ioctl's place in its context's submission order. - ordinal: u64, } impl WorkStateLease { - pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { - let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), - engine: AtomicU8::new(0), - ordinal, } } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many - /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now - /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. - pub(crate) fn acquire(&self, engine: Engine) -> Result { - let current = self.word(); - if current != FREE { - return Ok(current); - } - // The work-storage lock serializes the decision with this context's other - // admissions, and with the other engine of this ioctl taking its node. - let mut work = self.context.work.lock(); + pub(crate) fn acquire(&self) -> Result { let current = self.word(); if current != FREE { return Ok(current); } - let mut order = self.context.order.lock(); - if !order.admits(self.ordinal) { - return Err(EBUSY); - } - let engine = engine.index(); - let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; - if let Err(error) = work.initialize(candidate) { - self.context.release_state(candidate, engine); + let Some(candidate) = self.context.states.allocate() else { + let current = self.word(); + return if current != FREE { + Ok(current) + } else { + Err(EBUSY) + }; + }; + if let Err(error) = self.context.work.lock().initialize(candidate) { + self.context.states.release(candidate); return Err(error); } - self.engine.store(engine as u8, Ordering::Relaxed); - self.word.store(candidate, Ordering::Release); - let wake = order.admit(self.ordinal); - drop(order); - drop(work); - if wake { - self.context.pass_turn(); + match self + .word + .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(candidate), + Err(installed) => { + self.context.states.release(candidate); + Ok(installed) + } } - Ok(candidate) - } - /// The domain a publisher samples before acquire() and waits on after EBUSY. - pub(crate) fn returns(&self) -> &Arc { - &self.context.state_returns } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) @@ -489,16 +373,6 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } - /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores - /// the completion stamps of the node's commands only then, so a command whose record is - /// present still waits for its stamp while other commands of its ioctl run. - pub(crate) fn drained(&self) -> Result { - let word = self.word(); - if word == FREE { - return Err(EINVAL); - } - self.context.work.lock().drained(word) - } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work @@ -513,8 +387,7 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or_else(|| - super::capacity::refused(super::capacity::Site::RenderWorkSlots)) + self.context.work.lock().render.get(pair).ok_or(ENOSPC) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -529,75 +402,7 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); - } else if self.context.order.lock().leave(self.ordinal) { - self.context.pass_turn(); - } - } -} - -impl Context { - /// Every return bumps the domain: a publisher that sampled it before its - /// failed acquire() then never sleeps through this node. - fn release_state(&self, word: u32, engine: usize) { - if self.states.release(word, engine) { - self.state_returns.notify(); - } - } - /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a - /// refusal means one that sampled before its refused acquire() never sleeps through it. - fn pass_turn(&self) { - self.state_returns.notify(); - self.compute_returns.notify(); - } -} - -impl QosIds { - pub(crate) fn used(&self) -> u32 { - (*self.used.lock() & !3u128).count_ones() - } -} - -/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or -/// an unreadable head) and no acknowledged kill. -fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool { - killed == 0 && head.is_none_or(|head| head != [0, head_va]) -} - -#[kunit_tests(asahi_g17_qos_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn linked_unkilled_work_is_retained() { - let va = 0xfffffc20_0000_4400; - assert!(!must_retain(Some([0, va]), va, 0)); - assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0)); - assert!(must_retain(None, va, 0)); - assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1)); - assert!(!must_retain(None, va, 1)); - } - - #[test] - fn failed_client_construction_returns_every_qos_id() { - let ids = QosIds::new().unwrap(); - for _ in 0..8 { - let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); - for _ in 0..126 { - clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); - } - assert_eq!(ids.used(), 126); - assert!(matches!(ids.acquire(), Err(ENOSPC))); - drop(clients); - assert_eq!(ids.used(), 0); - assert_eq!(*ids.used.lock(), 3); + self.context.states.release(word); } } } - -impl Context { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { - writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", - scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7586f8474e602..6c48367e2d686 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -4,50 +4,23 @@ //! the complete sequence distinguishes its owner until exact retirement. use core::sync::atomic::{AtomicU32, Ordering}; -use kernel::prelude::*; pub(super) const FREE: u32 = u32::MAX; pub(super) const STATE_COUNT: usize = 63; pub(super) const NODE_STRIDE: usize = 0x100; -/// Engines whose admissions take work-state nodes: render (0) and compute (1). -pub(super) const ENGINES: usize = 2; - -/// Whether an engine that holds `held` nodes may take one of `free` nodes. -/// -/// All engines of one ioctl share its node, and the node returns only once every command of -/// the ioctl has retired. So render admission can keep taking nodes for newer ioctls while -/// their compute commands still wait in the compute queue behind an older ioctl that has no -/// node yet; once render held every node, nothing could retire and both queues stalled. An -/// engine therefore never takes the last free node while it already holds one. The other -/// engine can then always take a node, and an engine refused here waits only for nodes it -/// took itself, for ioctls ahead of its waiting command. Each engine admits a context's -/// ioctls in submission order, so those waits cannot form a cycle. -pub(super) const fn may_take(free: usize, held: u32) -> bool { - free > 1 || (free == 1 && held == 0) -} - pub(super) struct WorkStates { next: AtomicU32, live: [AtomicU32; 64], - /// Nodes taken by each engine's admissions and not yet released. - held: [AtomicU32; ENGINES], } impl WorkStates { pub(super) const fn new() -> Self { Self { next: AtomicU32::new(0), live: [const { AtomicU32::new(FREE) }; 64], - held: [const { AtomicU32::new(0) }; ENGINES], } } - /// Takes a node for `engine`, or none when [`may_take`] refuses. Callers serialize - /// allocations of one context; releases may run concurrently. - pub(super) fn allocate(&self, engine: usize) -> Option { - let held = self.held.get(engine)?; - if !may_take(STATE_COUNT.saturating_sub(self.used()), held.load(Ordering::Acquire)) { - return None; - } + pub(super) fn allocate(&self) -> Option { for _ in 0..64 { let word = self.next.fetch_add(1, Ordering::Relaxed); let index = (word & 63) as usize; @@ -56,119 +29,17 @@ impl WorkStates { .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) .is_ok() { - held.fetch_add(1, Ordering::AcqRel); return Some(word); } } None } - pub(super) fn release(&self, word: u32, engine: usize) -> bool { + pub(super) fn release(&self, word: u32) -> bool { let index = (word & 63) as usize; - let released = index != STATE_COUNT + index != STATE_COUNT && self.live[index] .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) - .is_ok(); - if released { - if let Some(held) = self.held.get(engine) { - held.fetch_sub(1, Ordering::AcqRel); - } - } - released - } -} - -/// Ordinals past the turn remembered for ioctls that took a node or left early. -pub(super) const ORDER_WINDOW: u64 = 1024; - -/// How far past the oldest waiting ioctl an ioctl may take its node. -pub(super) const RUN_AHEAD: u64 = 8; - -/// The order in which a context's ioctls take work-state nodes. -/// -/// Each engine admits a context's ioctls in submission order, but the engines advance -/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still -/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render -/// ioctls in flight on one queue) the firmware then stops completing all of the context's work -/// within seconds. Native macOS schedules a queue's command buffers in commit order across -/// engines, at most 36 between scheduling and completion, and the same load runs there without -/// failure. -/// -/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have -/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; -/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler -/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that -/// without letting one engine get far ahead of the other. -pub(super) struct Order { - /// The next ordinal to hand out. - next: u64, - /// The oldest ordinal that has neither taken a node nor left. - turn: u64, - /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. - done: [u64; (ORDER_WINDOW / 64) as usize], - /// Set once an ioctl left too far past the turn to remember; admission is unordered after. - unordered: bool, - /// An ioctl was refused since the turn last moved: its publisher waits for the next move. - refused: bool, -} -impl Order { - pub(super) const fn new() -> Self { - Self { - next: 0, - turn: 0, - done: [0; (ORDER_WINDOW / 64) as usize], - unordered: false, - refused: false, - } - } - /// The ordinal of a new ioctl, in submission order. - pub(super) fn assign(&mut self) -> u64 { - let ordinal = self.next; - self.next += 1; - ordinal - } - /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the - /// turn next moves, so that move wakes the refused publisher. - pub(super) fn admits(&mut self, ordinal: u64) -> bool { - let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; - self.refused |= !admits; - admits - } - /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. - pub(super) fn admit(&mut self, ordinal: u64) -> bool { - self.finish(ordinal) - } - /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). - /// Returns whether a refused ioctl may now be admitted. - pub(super) fn leave(&mut self, ordinal: u64) -> bool { - self.finish(ordinal) - } - fn finish(&mut self, ordinal: u64) -> bool { - if self.unordered || ordinal < self.turn { - return false; - } - if ordinal == self.turn { - self.pass(); - return core::mem::take(&mut self.refused); - } - if ordinal - self.turn >= ORDER_WINDOW { - self.unordered = true; - return core::mem::take(&mut self.refused); - } - let bit = ordinal % ORDER_WINDOW; - self.done[(bit / 64) as usize] |= 1 << (bit % 64); - false - } - /// Moves the turn past the current ordinal and every following one already done. - fn pass(&mut self) { - loop { - self.turn += 1; - let bit = self.turn % ORDER_WINDOW; - let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); - if self.done[word] & mask == 0 { - break; - } - self.done[word] &= !mask; - } + .is_ok() } } @@ -198,131 +69,3 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } - -impl WorkStates { - pub(super) fn used(&self) -> usize { - self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() - } -} -impl RenderSlots { - pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } -} - -#[kunit_tests(asahi_g17_work_states)] -mod tests { - use super::*; - - #[test] - fn last_node_stays_with_an_engine_holding_none() { - assert!(!may_take(0, 0)); - assert!(may_take(1, 0)); - assert!(!may_take(1, 3)); - assert!(may_take(2, 3)); - } - - #[test] - fn render_cannot_starve_compute_of_a_node() { - let states = WorkStates::new(); - let mut render = KVec::new(); - while let Some(word) = states.allocate(0) { - render.push(word, GFP_KERNEL).unwrap(); - } - // Render stops one short; compute takes the last node, then nothing is left. - assert_eq!(render.len(), STATE_COUNT - 1); - let compute = states.allocate(1).unwrap(); - assert_eq!(states.used(), STATE_COUNT); - assert!(states.allocate(0).is_none()); - assert!(states.allocate(1).is_none()); - // A lone free node goes to the engine holding none. - assert!(states.release(compute, 1)); - assert!(states.allocate(0).is_none()); - let compute = states.allocate(1).unwrap(); - // With both engines holding nodes, a lone free node waits for one of their returns. - assert!(states.release(render.pop().unwrap(), 0)); - assert!(states.allocate(0).is_none()); - assert!(states.allocate(1).is_none()); - // Two free nodes: render may take one again, never the last. - assert!(states.release(render.pop().unwrap(), 0)); - assert!(states.allocate(0).is_some()); - assert!(states.allocate(0).is_none()); - assert!(states.release(compute, 1)); - assert!(states.allocate(1).is_some()); - } - - #[test] - fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { - let mut order = Order::new(); - let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); - let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); - // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. - for &ordinal in &ordinals[..=RUN_AHEAD as usize] { - assert!(order.admits(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // Ioctls ahead of the turn taking their nodes do not move it. - for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { - assert!(!order.admit(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // The oldest one moves the turn past everything already admitted, and wakes the refused. - assert!(order.admit(oldest)); - assert!(order.admits(last)); - let next = order.assign(); - assert!(order.admits(next)); - assert!(!order.admits(next + RUN_AHEAD + 1)); - } - - #[test] - fn ioctls_leaving_without_a_node_pass_the_turn() { - let mut order = Order::new(); - let (a, b, c) = (order.assign(), order.assign(), order.assign()); - let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); - let beyond = later[RUN_AHEAD as usize - 1]; - // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. - assert!(!order.leave(b)); - assert!(!order.leave(c)); - assert!(!order.admits(beyond)); - // Once a takes its node, the turn passes b and c as well, and the refused one may retry. - assert!(order.admit(a)); - assert!(order.admits(beyond)); - // With nobody refused since, moving the turn wakes nobody; an ordinal already passed - // leaves without effect. - assert!(!order.leave(later[0])); - assert!(!order.leave(a)); - } - - #[test] - fn an_ioctl_leaving_too_far_ahead_unorders_admission() { - let mut order = Order::new(); - let first = order.assign(); - let mut last = first; - for _ in 0..ORDER_WINDOW { - last = order.assign(); - } - assert!(!order.admits(last)); - assert!(order.leave(last)); - assert!(order.admits(last - 1)); - } - - #[test] - fn single_engine_admission_wakes_nobody() { - let mut order = Order::new(); - for _ in 0..3 * RUN_AHEAD { - let ordinal = order.assign(); - assert!(order.admits(ordinal)); - assert!(!order.admit(ordinal)); - } - } - - #[test] - fn releases_return_nodes_to_their_engine() { - let states = WorkStates::new(); - let word = states.allocate(1).unwrap(); - assert!(states.release(word, 1)); - assert!(!states.release(word, 1)); - assert_eq!(states.used(), 0); - assert_eq!(states.held[1].load(Ordering::Acquire), 0); - // An engine outside the table takes nothing. - assert!(states.allocate(ENGINES).is_none()); - } -} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs deleted file mode 100644 index 02bcc66de8f51..0000000000000 --- a/drivers/gpu/drm/asahi/g17/data_segment.rs +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -// Copyright (C) 2026 The Gravity Linux Contributors -// -// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and -// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: -// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 -// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. - -//! Crash log buffers in the GPU firmware's data segment. -//! -//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it -//! by physical address inside its data segment, the `fw-data` reserved region, which the boot -//! loader keeps for the loaded firmware until reboot. - -use kernel::{ - c_str, device, - io::{ - self, - IoSysMap, - Region, // - }, - prelude::*, -}; - -/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. -pub(super) struct DataSegmentLog { - mem: io::mem::Mem, - offset: usize, - size: usize, -} - -impl DataSegmentLog { - /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which - /// must lie inside the `fw-data` region of `dev`. - pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { - let node = dev.of_node().ok_or(EINVAL)?; - let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; - let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; - if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { - return Err(EINVAL); - } - // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps - // it until reboot; only the requested subrange is exposed. - let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; - Ok(Self { mem, offset, size }) - } - - /// The CPU view of the buffer. - pub(super) fn buf(&mut self) -> Result> { - // SAFETY: map() checked that the range lies inside the mapping. - let base = unsafe { self.mem.ptr().add(self.offset) }; - let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; - // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. - Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) - } -} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5a0ed611b1dde..5abd494eeb1f5 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -9,7 +9,6 @@ use super::{ use kernel::prelude::*; /// A failed producer does not establish a fault in its consumer's VM. -#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn inherited_error(error: Error) -> Error { if error == EIO || error == ENODEV || error == ETIMEDOUT { ECANCELED @@ -18,61 +17,6 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } -/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its -/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with -/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too. -pub(crate) const PRODUCER_FAILED: Error = ECANCELED; - -/// A replay's original host inputs must still be signalled. Treat an unmet -/// input as a failed replay, never as a wait under the device mutex. -pub(crate) fn replay_status(status: i32) -> Result { - if status == 0 { Err(EBUSY) } - else if status < 0 { Err(PRODUCER_FAILED) } - else { Ok(()) } -} - -/// Whether a failed command may still run or write: it holds its VM pin until it has physically -/// retired, or was never published. -pub(crate) trait Retention { - fn retained(&self) -> bool; -} - -/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix -/// covers one is cancelled, so none runs before a failed producer the firmware may still replay. -/// Once that producer has retired nothing of it can run or write any more, so it stops counting: -/// otherwise one cancelled command cancelled every later command of its queue for good. -pub(crate) struct FailureRecord { - retained: KVec<(u64, T)>, - /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did. - untracked: Option<(u64, T)>, -} - -impl FailureRecord { - pub(crate) const fn new() -> Self { - Self { retained: KVec::new(), untracked: None } - } - - /// A finished frontier entry, at pruning. - pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) { - if status >= 0 || !entry.retained() { - return; - } - if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err() - && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first) - { - self.untracked = Some((sequence, entry.clone())); - } - } - - /// The earliest failure later prefixes must not bypass. - pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> { - self.retained.retain(|(_, entry)| entry.retained()); - Iterator::chain(self.retained.iter(), self.untracked.iter()) - .min_by_key(|(sequence, _)| *sequence) - .map(|(sequence, entry)| (*sequence, entry)) - } -} - pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], @@ -137,79 +81,3 @@ impl Frontier { self.insert(sequence, dependency) } } - -#[kunit_tests(asahi_g17_dependency)] -mod tests { - use super::*; - - #[derive(Clone)] - struct Probe(kernel::sync::Arc); - - impl Probe { - fn new(retained: bool) -> Self { - Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap()) - } - fn retire(&self) { - self.0.store(false, core::sync::atomic::Ordering::Relaxed); - } - } - - impl Retention for Probe { - fn retained(&self) -> bool { - self.0.load(core::sync::atomic::Ordering::Relaxed) - } - } - - fn earliest(record: &mut FailureRecord) -> Option { - record.earliest().map(|(sequence, _)| sequence) - } - - #[test] - fn a_cancelled_command_does_not_poison_its_queue() { - // Cancelled or failed before publication: it released its pin and can never run. - let mut record = FailureRecord::new(); - record.note(5, ECANCELED.to_errno(), &Probe::new(false)); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn a_retained_failure_cancels_until_it_retires() { - let mut record = FailureRecord::new(); - let failed = Probe::new(true); - record.note(5, EIO.to_errno(), &failed); - assert_eq!(earliest(&mut record), Some(5)); - failed.retire(); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn a_later_retained_failure_survives_an_earlier_retirement() { - let mut record = FailureRecord::new(); - let (first, second) = (Probe::new(true), Probe::new(true)); - record.note(5, EIO.to_errno(), &first); - record.note(9, ECANCELED.to_errno(), &second); - assert_eq!(earliest(&mut record), Some(5)); - first.retire(); - assert_eq!(earliest(&mut record), Some(9)); - second.retire(); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn successes_are_not_recorded() { - let mut record = FailureRecord::new(); - record.note(5, 1, &Probe::new(true)); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn failed_predecessor_cannot_be_bypassed() { - let mut frontier = Frontier::new(); - for error in [ECANCELED, EIO, ETIMEDOUT] { - assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED)); - } - assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(())); - assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(())); - assert_eq!(frontier.count, 0); - } -} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index a175e58392379..9e4e0253b7c81 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,8 +11,6 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { - /// The firmware instances of this GPU: the primary, and the secondary where one exists. - fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -36,8 +34,6 @@ pub(super) trait Queues { fn retain_render_completions(&mut self, masks: [u64; 2]); fn complete_compute(&mut self, masks: [u64; 2]) -> Result; fn context_killed(&mut self, cookie: u64); - /// The firmware failed a compute command (see [`Event::ComputeFault`]). - fn compute_fault(&mut self, qid: u8, stamp: u32); } /// Work requested after releasing the device mutex. @@ -74,12 +70,6 @@ fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { } Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, Event::ContextKilled(cookie) => host.context_killed(cookie), - Event::ComputeFault { qid, stamp } => { - crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0); - host.compute_fault(qid, stamp); - // Its queue retires or settles its quarantine on this pass. - masks[usize::from(qid) / 64] |= 1 << (qid % 64); - } Event::Unknown(_) => {} } Ok(true) @@ -91,10 +81,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for &role in host.roles() { + for role in [Role::Primary, Role::Secondary] { host.trace(role); } - for &role in host.roles() { + for role in [Role::Primary, Role::Secondary] { host.statistics(role); } let mut masks = [0; 2]; @@ -120,7 +110,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { + if !host.control_backpressured() { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index bd1228a82e26f..4cd05fde16581 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,8 +44,6 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, - /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). - bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -59,7 +57,6 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), - bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -111,26 +108,17 @@ impl Detector { Err(_) => 0, } }; - if streak < STREAK_ON { - return; - } - if idle_sum.saturating_mul(4) >= busy_sum { + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } - } else { - self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } - - fn bound(&self, now: u64) -> bool { - !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) - } } #[cfg(CONFIG_CPU_FREQ)] @@ -159,10 +147,6 @@ impl Requests { } } - /// Add a policy to the pinned requests. - /// - /// # Safety - /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -195,7 +179,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } } @@ -209,7 +193,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -227,7 +211,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } applied } @@ -251,7 +235,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } fn clear(mut self: Pin<&mut Self>) { @@ -380,13 +364,6 @@ impl Feed { } } - /// Process a cpufreq policy notification. - /// - /// # Safety - /// - /// `block` must be this pinned Feed's registered notifier, and `data` must - /// point to the live policy supplied by the cpufreq notifier core. - /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -410,35 +387,24 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as i32; + return bindings::NOTIFY_DONE as _; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as i32 + bindings::NOTIFY_OK as _ } fn now() -> u64 { Monotonic::ktime_get() as u64 } - pub(crate) fn note_pass(&self, start: u64, end: u64) { + pub(crate) fn note_render_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } } - /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps - /// under a quarter of their busy time, and the feed does not see host-refill-bound work. - /// Lingers like streaming, so a pause of more than a few milliseconds ends it. - pub(crate) fn gpu_bound(&self) -> bool { - if !cfg!(CONFIG_CPU_FREQ) { - return false; - } - let now = Self::now(); - self.detector.bound(now) && !self.detector.wanted(now) - } - /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -521,51 +487,3 @@ impl PinnedDrop for Feed { self.shutdown(); } } - -#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] -mod detector_tests { - use super::*; - - /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. - fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { - let mut start = 1000; - let mut now = 1_000_000; - for _ in 0..passes { - detector.note(start, start + busy, now); - start += busy + gap; - now += 1_000_000; - } - now - } - - #[test] - fn back_to_back_long_passes_are_gpu_bound() { - let detector = Detector::new(); - // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. - let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); - assert!(detector.bound(now)); - assert!(!detector.wanted(now)); - assert!(!detector.bound(now + LINGER_NS)); - } - - #[test] - fn refill_gaps_stream_instead() { - let detector = Detector::new(); - // 20 us passes with 60 us refill gaps: bound by host refills. - let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); - assert!(detector.wanted(now)); - assert!(!detector.bound(now)); - } - - #[test] - fn paced_or_short_runs_are_neither() { - let detector = Detector::new(); - // Each pass starts after a pause of at least the refill limit. - let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); - assert!(!detector.bound(now) && !detector.wanted(now)); - // Fewer back-to-back passes than the streak needs. - let detector = Detector::new(); - let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); - assert!(!detector.bound(now) && !detector.wanted(now)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 9015dc82e4076..c9c9ebda7f6bc 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) + Err(ENOSPC) } } impl Drop for RenderId { @@ -69,84 +69,19 @@ impl Drop for RenderId { } } -/// Free-list backing is by far the largest user of the shared lower GPU VA -/// window, and placing it populates every page before the window is searched. -/// After a refused placement, builders waiting for capacity would allocate and -/// free tens of MiB per attempt while nothing in the window changed. A refused -/// size is therefore refused again without allocating until free-list backing -/// is freed, except for one probe per interval across all callers. -static LOWER_FREED: AtomicU64 = AtomicU64::new(0); -/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. -static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); -static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); -static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); -const LOWER_PROBE_NS: u64 = 100_000_000; - -/// The window refused free-list backing and nothing was freed since. -pub(crate) fn lower_pressure() -> bool { - let refused = LOWER_REFUSED.load(Ordering::Acquire); - refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) -} - -/// Place free-list backing of `size` bytes unless the same or a smaller size -/// was refused with nothing freed since; `place` allocates and maps it. -fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { - let size = size as u64; - if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { - let now = super::now_ns(); - let at = LOWER_PROBE_AT.load(Ordering::Relaxed); - if now < at - || LOWER_PROBE_AT - .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return Err(ENOSPC); - } - } - match place() { - Err(error) if error == ENOSPC => { - LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); - LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); - LOWER_REFUSED.store( - LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), - Ordering::Release, - ); - Err(error) - } - Ok(placed) => { - if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { - LOWER_REFUSED.store(0, Ordering::Release); - } - Ok(placed) - } - Err(error) => Err(error), - } -} - pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } -impl Drop for Pages { - fn drop(&mut self) { - LOWER_FREED.fetch_add(1, Ordering::AcqRel); - } -} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; - let backing = place_lower(size, || { - alloc.lower( - size, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - ) - })?; - Self::from_backing(backing, blocks, layout) - } - fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + let mut backing = alloc.lower( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -165,7 +100,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - pub(crate) fn page_list_va(&self) -> u64 { + fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -209,17 +144,14 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; - let (backing, [global, client]) = place_lower(size, || { - alloc.lower_with_aliases( - size, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - [render_global, client], - ) - })?; - let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; + let pages = Pages::new(alloc, blocks, RunLayout::Render)?; + let address = pages.page_list_va(); + let global = pages + .backing + .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; + let client = pages + .backing + .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; Ok(Self { _client: client, _global: global, @@ -590,18 +522,6 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } - self.write_pages(&pages)?; - self.pages = Some(pages); - Ok(()) - } - /// Preserve both sets of pages on failure. The caller owns the consumed - /// release and returns old pages for destruction outside the device mutex. - pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { - self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; - core::mem::swap(&mut self.pages, pages); - Ok(()) - } - fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -620,9 +540,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); + self.pages = Some(pages); Ok(()) } - pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -670,18 +590,10 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, - /// No reference since the given time. The registration stays live so that - /// the owner's next render reuses it without a control-ring release; the - /// release becomes pending when the linger ends. - Lingering(u64), ReleaseSent(u32), Released, } -/// How long an idle registration of an open owner may stay live: the -/// firmware's effort idle hold. -pub(crate) const LINGER_NS: u64 = 16_000_000; - /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -715,14 +627,10 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; - // A lingering registration was never released: it continues like an - // overlap of the owner's previous render. let overlapping = same_owner - && match previous.phase { - LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, - LeasePhase::Lingering(_) => true, - _ => false, - }; + && previous.phase == LeasePhase::Ready + && previous.references >= 2 + && previous.commands >= 2; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -784,35 +692,16 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. - /// With `linger`, a registration left without references stays live - /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, - linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; - if self.references == 0 && linger { - self.phase = LeasePhase::Lingering(super::now_ns().max(1)); - } Ok(self.pending_release()) } - /// The time since which an idle registration lingers, if it does. - pub(crate) fn lingering(&self) -> Option { - match self.phase { - LeasePhase::Lingering(since) => Some(since), - _ => None, - } - } - /// Ends a linger; the final release becomes pending. - pub(crate) fn end_linger(&mut self) { - if matches!(self.phase, LeasePhase::Lingering(_)) { - self.phase = LeasePhase::Ready; - } - } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) @@ -853,30 +742,3 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } - -impl RenderIds { - pub(crate) fn used(&self) -> u32 { - self.used[0].load(Ordering::Acquire).count_ones() - + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() - } -} - -#[kunit_tests(asahi_g17_render_id_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn rejected_pool_construction_returns_all_render_ids() { - let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); - for _ in 0..8 { - let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); - for _ in 0..127 { - pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); - } - assert_eq!(ids.used(), 127); - assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); - drop(pools); - assert_eq!(ids.used(), 0); - } - } -} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index 9d430d50db365..ec6797637bdbe 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { + if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { + if scratch > u32::MAX as u64 || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 26c6adcf34c83..c4cf6960db296 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -136,9 +136,6 @@ pub(crate) enum Event { TvbGrow(TvbGrowRequest), FreeListGrow(FreeListCompletion), ContextKilled(u64), - /// The firmware failed a compute command of queue `qid` and stores no completion stamp for - /// it: `stamp` is the value that command would have stored. - ComputeFault { qid: u8, stamp: u32 }, Unknown(u32), } @@ -148,8 +145,6 @@ impl EventRecord { const TVB_GROW: u32 = 6; const FREE_LIST_GROW: u32 = 13; const CONTEXT_KILLED: u32 = 12; - const COMPUTE_FAULT: u32 = 7; - const COMPUTE_QIDS: u32 = 128; const DIAGNOSTIC_PAIRS_MAX: u16 = 5; const TVB_SLOTS: u32 = 127; const SUBPIPES: u32 = 64; @@ -165,16 +160,6 @@ impl EventRecord { unsafe { self.words[0] } } - /// Type 7 is a retirement witness only on the firmware whose layout and - /// visibility semantics are qualified. 25G83 must keep consuming it as an - /// unknown event until its own fault contract is established. - pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { - if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { - return Ok(Event::Unknown(Self::COMPUTE_FAULT)); - } - self.decode() - } - /// Validates the record's fields without interpreting resource ownership. /// A malformed known event is an error, unlike an unknown tag. pub(crate) fn decode(&self) -> Result { @@ -213,19 +198,6 @@ impl EventRecord { halt_count: record.halt_count, })) } - Self::COMPUTE_FAULT => { - // SAFETY: Every constructor initializes all integer words. - let words = unsafe { self.words }; - // Only compute faults are known to use this type: any other layout stays an - // unknown event, consumed and ignored as before. - if words[3] >= Self::COMPUTE_QIDS || words[4] & 0xff != 0 { - return Ok(Event::Unknown(Self::COMPUTE_FAULT)); - } - Ok(Event::ComputeFault { - qid: words[3] as u8, - stamp: words[4], - }) - } Self::CONTEXT_KILLED => { // SAFETY: Every constructor initializes all integer words. let words = unsafe { self.words }; @@ -526,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: u8::from(first), + first: first as u8, }) } } @@ -553,37 +525,3 @@ impl ContextKill { } } } - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_event_records))] -mod tests { - use super::*; - - fn record(words: &[u32]) -> EventRecord { - let mut all = [0u32; EVENT_WORDS]; - all[..words.len()].copy_from_slice(words); - EventRecord::from_words(all) - } - - #[test] - fn a_compute_fault_names_its_queue_and_stamp() { - let event = record(&[7, 1, 2, 0x39, 0x300, 0xffff_fc00, 1]).decode(); - assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - } - - #[test] - fn v25g83_does_not_accept_an_unqualified_fault_witness() { - use super::super::variant::InitGraph; - let fault = record(&[7, 1, 2, 0x39, 0x300]); - assert_eq!(fault.decode_for(InitGraph::G17P), - Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), - Ok(Event::Unknown(7))); - } - - #[test] - fn an_unrecognized_type_7_layout_stays_unknown() { - assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 9, 0x301]).decode(), Ok(Event::Unknown(7))); - } -} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index b1afdad816b6b..0633da7f43541 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u64::from(u16::MAX); - let cliques = (layout >> 24) & u64::from(u8::MAX); - let user = (layout >> 32) & u64::from(u16::MAX); - let driver = (layout >> 48) & u64::from(u16::MAX); + let depth = layout & u16::MAX as u64; + let cliques = (layout >> 24) & u8::MAX as u64; + let user = (layout >> 32) & u16::MAX as u64; + let driver = (layout >> 48) & u16::MAX as u64; if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,42 +152,13 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of -/// the regions the program names (two preemption banks of three 0x800-byte records per core, -/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. -pub(crate) struct OperandState { - r10229: u64, - r140a8: u64, - r10099: u64, - r10091: u64, - r0a5c1: u64, - r0a5c9: u64, - /// Size of the buffer. - pub(crate) size: usize, -} - -impl OperandState { - /// One cluster of six cores (A18 Pro). - pub(crate) const ONE_BY_SIX: OperandState = OperandState { - r10229: 0x12800, - r140a8: 0x13000, - r10099: 0x9405, - r10091: 0x12400, - r0a5c1: 0x0005, - r0a5c9: 0x9000, - size: 0x14000, - }; - /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. - pub(crate) const TWO_BY_TEN: OperandState = OperandState { - r10229: 0x3c800, - r140a8: 0x3d800, - r10099: 0x1e405, - r10091: 0x3c400, - r0a5c1: 0x0005, - r0a5c9: 0x1e000, - size: 0x44000, - }; -} +/// Offsets in the operand-state buffer of the regions the program names. +const OPERAND_STATE_10229: u64 = 0x12800; +const OPERAND_STATE_140A8: u64 = 0x13000; +const OPERAND_STATE_10099: u64 = 0x9405; +const OPERAND_STATE_10091: u64 = 0x12400; +const OPERAND_STATE_0A5C1: u64 = 0x0005; +const OPERAND_STATE_0A5C9: u64 = 0x9000; /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -312,8 +283,6 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, - /// Layout of the operand-state buffer. - pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -336,7 +305,6 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; - let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -379,14 +347,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), + RegisterWrite::new(0x14028, args.free_list_slot as u64), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), - RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), - RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -396,12 +364,11 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; - let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), - RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), ]) } @@ -507,9 +474,8 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Fills a descriptor cleared by `Object::initialize`. The unused firmware - /// fields and padding retain that zero initialization. - pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { + /// Writes the complete descriptor. + pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -523,6 +489,7 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; + clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 3092e599caa84..1f25bd0fc1c0d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,8 +27,6 @@ use kernel::prelude::*; use super::clear; -pub(crate) mod v25g83; - /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -221,21 +219,6 @@ impl SecondaryRoot { config_va, } } - - /// Returns the root of a single instance that carries both roles: the primary root, with - /// status B, extended by the power-state and configuration pointers. - pub(crate) fn merged( - args: &RootArgs, - status_b_va: u64, - power_state_va: u64, - config_va: u64, - ) -> Self { - Self { - root: Root::new(Role::Primary, args, status_b_va), - power_state_va, - config_va, - } - } } /// Offset of the secondary root from the primary root. Both live in one @@ -608,42 +591,31 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes of one instance; need not be a multiple of the page size. + /// Size in bytes; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, - /// Number of instances of the block. The firmware finds instance `n` at - /// `va + n * size`; it lives at `phys + n * phys_stride`. - pub(crate) instances: u32, - /// Distance between the physical addresses of consecutive instances. - pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length that - /// instance `instance` of the window must be mapped with, or `None` if its - /// physical and GPU addresses differ within a page or the mapped span - /// overflows. - pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length the + /// window must be mapped with, or `None` if its physical and GPU + /// addresses differ within a page or the mapped span overflows. + pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - let phys = self - .phys - .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; - let va = self - .va - .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; - if phys & mask != va & mask { + if self.phys & mask != self.va & mask { return None; } - let start = va & !mask; - let span = va - .checked_add(u64::from(self.size))? + let start = self.va & !mask; + let span = self + .va + .checked_add(self.size as u64)? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((phys & !mask, start, span)) + Some((self.phys & !mask, start, span)) } } @@ -657,9 +629,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Size in bytes of all instances of the window. + /// Window size in bytes. pub(crate) total_size: u32, - /// Size in bytes of one instance. + /// Window size in bytes, again. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -678,7 +650,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size * window.instances, + total_size: window.size, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -704,19 +676,14 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; -/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. -pub(crate) const OPP_CELLS: usize = PERF_LANES; - /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. - pub(crate) volt_uv: [u32; OPP_CELLS], - /// Number of valid cells in [`Self::volt_uv`]. - pub(crate) cells: usize, + /// GPU core voltage in µV. + pub(crate) volt_uv: u32, } /// Per-SoC parts of the performance tables that the operating points do not @@ -761,23 +728,20 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: [u32; OPP_CELLS], - sram_mv: [u32; OPP_CELLS], - cells: usize, + core_mv: u32, + sram_mv: u32, } impl PerfGroup { - /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes - /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster - /// `n`'s. The remaining lanes stay zero. + /// Fills the table. `lanes` is the number of voltage lanes holding each + /// state's voltages; the remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..if state.cells == 1 { lanes } else { state.cells } { - let cell = if state.cells == 1 { 0 } else { lane }; - self.core_mv[index][lane] = state.core_mv[cell]; - self.sram_mv[index][lane] = state.sram_mv[cell]; + for lane in 0..lanes { + self.core_mv[index][lane] = state.core_mv; + self.sram_mv[index][lane] = state.sram_mv; } } } @@ -791,18 +755,16 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: [0; OPP_CELLS], - sram_mv: [0; OPP_CELLS], - cells: 0, + core_mv: 0, + sram_mv: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv.map(|uv| uv / 1000); + let core_mv = opp.volt_uv / 1000; *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), - cells: opp.cells, + sram_mv: core_mv.max(sram_min_mv), }; } Ok(states) @@ -1207,35 +1169,6 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; -/// The values of region C that differ between SoCs and firmware builds. -pub(crate) struct RegionCConfig { - /// A18 Pro: 0xffff. - pub(crate) unk_54: u16, - /// A18 Pro: 1 in every entry. - pub(crate) unk_78: [u32; 3], - /// Highest GPU power, in mW (A18 Pro: 9550). - pub(crate) max_power_mw: u32, - /// A18 Pro: 0x3000_001d. - pub(crate) unk_9c: u32, - /// A18 Pro: 0x3100_0002. - pub(crate) unk_a0: u32, - /// A18 Pro: 0x7d0 (2000). - pub(crate) unk_7c4: u32, - /// A18 Pro: zero. - pub(crate) unk_9e0: [u32; 8], -} - -/// Region C of the A18 Pro firmware (26A428). -pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { - unk_54: 0xffff, - unk_78: [1; 3], - max_power_mw: 9550, - unk_9c: 0x3000_001d, - unk_a0: 0x3100_0002, - unk_7c4: 0x7d0, - unk_9e0: [0; 8], -}; - /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1264,7 +1197,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// See [`RegionCConfig::unk_54`]. + /// Required: 0xffff. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1280,10 +1213,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// See [`RegionCConfig::unk_78`]. + /// Required: 1 in every entry. pub(crate) unk_78: [u32; 3], - /// Highest GPU power, in mW. - pub(crate) max_power_mw: u32, + /// Required: 0x254e (9550). + pub(crate) unk_84: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1291,9 +1224,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// See [`RegionCConfig::unk_9c`]. + /// Required: 0x3000_001d. pub(crate) unk_9c: u32, - /// See [`RegionCConfig::unk_a0`]. + /// Required: 0x3100_0002. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1301,7 +1234,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// See [`RegionCConfig::unk_7c4`]. + /// Required: 0x7d0 (2000). pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1352,10 +1285,7 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0xc], - /// See [`RegionCConfig::unk_9e0`]. - pub(crate) unk_9e0: [u32; 8], - pub(crate) unk_a00: [u8; 0x41d], + pub(crate) unk_9d4: [u8; 0x449], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1389,28 +1319,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self, cfg: &RegionCConfig) { + pub(crate) fn init(&mut self) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = cfg.unk_54; + self.unk_54 = 0xffff; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = cfg.unk_78; - self.max_power_mw = cfg.max_power_mw; + self.unk_78 = [1; 3]; + self.unk_84 = 0x254e; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = cfg.unk_9c; - self.unk_a0 = cfg.unk_a0; - self.unk_7c4 = cfg.unk_7c4; + self.unk_9c = 0x3000_001d; + self.unk_a0 = 0x3100_0002; + self.unk_7c4 = 0x7d0; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1434,7 +1364,6 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; - self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1482,7 +1411,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -pub(crate) const FWCTL_RING: u64 = 0x40; +const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs deleted file mode 100644 index f4c3ee5db22e2..0000000000000 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ /dev/null @@ -1,619 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). -//! -//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's -//! 26A428 build and the same objects, with these differences: -//! -//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before -//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; -//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` -//! bytes), at bundle `+0x5380`, followed by the auxiliary data; -//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; -//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five -//! separately allocated views and a firmware area ([`kernel_window`]). -//! -//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this -//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. - -use super::{ - fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, - RegisterMapEntry, Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, - COMPLETION_RING_INDEX, PERF_STATES, REGISTER_MAP_SLOTS, -}; -use kernel::prelude::*; - -/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper -/// kernel window. -pub(crate) mod kernel_window { - /// The power-state object named by the root (`0x6a8` bytes). - pub(crate) const POWER_STATE: u64 = 0x17_0000; - /// The five views named by the main configuration. - pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; - /// The firmware area named twice by the main configuration (`0x88` bytes). - pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; - /// The AON temperature-source table named by the power configuration. - pub(crate) const AON_TABLE: u64 = 0xc050_0000; -} - -/// Offsets in the private cluster. -pub(crate) mod private { - /// Size of the private cluster: the state grid at `+0`, status B and status A. - pub(crate) const SIZE: usize = 0xd4000; - /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). - pub(crate) const STATUS_B: usize = 0x1_0000; - /// The primary status A, after the `0xf180` bytes of status B. - pub(crate) const STATUS_A: usize = 0x1_f180; - /// The configuration named by the root; its status follows at `+0x80`. - pub(crate) const CONFIG: usize = 0xd_3440; - /// The zeroed page named by the main configuration's `unk_18`. - pub(crate) const MAIN_HEADER: usize = 0x8000; -} - -/// Bundle offset of the [`PowerConfig25G83`]. -pub(crate) const POWER_CONFIG: usize = 0x5380; -/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. -pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; -/// Bundle offset of the QoS pointer in the [`HwData25G83`]. -pub(crate) const QOS_POINTER: usize = 0x26b8; - -/// Most states of a CS or AFR performance table. -const AUX_PERF_STATES: usize = 16; - -/// The CS or AFR performance table of the hardware data. -#[repr(C, packed)] -pub(crate) struct AuxPerfTable { - /// Highest state, twice. - pub(crate) max_state: [u32; 2], - /// Frequency of each state in MHz. - pub(crate) freq_mhz: [u32; AUX_PERF_STATES], - /// Voltage of each state in mV, in the first word of each pair. - pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], - /// SRAM voltage of each state in mV, in the first word of each pair. - pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], -} - -// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. -unsafe impl Zeroable for AuxPerfTable {} - -static_assert!(size_of::() == 0x148); - -impl AuxPerfTable { - /// Fills the table from its operating points (one voltage cell each), raising each SRAM - /// voltage to `sram_min_mv`. - fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { - if !(2..=AUX_PERF_STATES).contains(&opps.len()) { - return Err(EINVAL); - } - self.max_state = [opps.len() as u32 - 1; 2]; - for (index, opp) in opps.iter().enumerate() { - if opp.cells != 1 { - return Err(EINVAL); - } - let mv = opp.volt_uv[0] / 1000; - self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; - self.mv[index][0] = mv; - self.sram_mv[index][0] = mv.max(sram_min_mv); - } - Ok(()) - } -} - -/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. -pub(crate) struct HwDataArgs25G83<'a> { - /// Enabled GPU cores. - pub(crate) active_cores: u32, - /// Operating points of the CS clock domain. - pub(crate) cs_opps: &'a [Opp], - /// Operating points of the AFR clock domain. - pub(crate) afr_opps: &'a [Opp], -} - -/// The hardware data of the 25G83 build, at the base of the bundle. -#[repr(C, packed)] -pub(crate) struct HwData25G83 { - /// Required: 0x6f_0000_0000. - pub(crate) unk_0: u64, - /// Required: 0xffc0_0000. - pub(crate) unk_8: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_10: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_18: u64, - /// Required: 0x2ff_ffff_8000. - pub(crate) unk_20: u64, - /// Required: 0xffff_fc21_8140_0000. - pub(crate) unk_28: u64, - pub(crate) unk_30: [u8; 0x610], - /// Register windows the firmware maps. - pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], - pub(crate) unk_e88: [u8; 8], - /// Chip identifier. - pub(crate) chip_id: u32, - /// Required: 1. - pub(crate) unk_e94: u32, - /// Required: 1. - pub(crate) unk_e98: u32, - pub(crate) unk_e9c: [u8; 4], - /// Required: 1. - pub(crate) unk_ea0: u32, - /// Required: 1. - pub(crate) unk_ea4: u32, - pub(crate) unk_ea8: [u8; 0x10], - /// Required: 1. - pub(crate) unk_eb8: u32, - pub(crate) unk_ebc: [u8; 4], - /// Required: 1. - pub(crate) unk_ec0: u32, - pub(crate) unk_ec4: [u8; 4], - /// Required: 1. - pub(crate) unk_ec8: u32, - pub(crate) unk_ecc: [u8; 4], - /// Required: 0x5dc0 (24000). - pub(crate) unk_ed0: u32, - /// Required: 1. - pub(crate) unk_ed4: u32, - /// Power sample period in ms: 8. - pub(crate) unk_ed8: u32, - pub(crate) unk_edc: [u8; 4], - /// Required: 1 in every entry. - pub(crate) unk_ee0: [u32; 3], - pub(crate) unk_eec: [u8; 0x18], - /// Required: 0x1f (31). - pub(crate) unk_f04: u32, - pub(crate) unk_f08: [u8; 0x1c], - /// Required: 0x0700_2004. - pub(crate) unk_f24: u32, - /// Required: 6. - pub(crate) unk_f28: u32, - pub(crate) unk_f2c: [u8; 8], - /// Required: 1. - pub(crate) unk_f34: u32, - /// Required: 1. - pub(crate) unk_f38: u32, - pub(crate) unk_f3c: [u8; 0x10], - /// Required: 0x31 (49). - pub(crate) unk_f4c: u32, - pub(crate) unk_f50: [u8; 0x38], - /// Enabled GPU cores. - pub(crate) unk_f88: u32, - /// Required: 0xf (15). - pub(crate) unk_f8c: u32, - pub(crate) unk_f90: [u8; 0x28], - /// Required: 0x1c (28). - pub(crate) unk_fb8: u32, - /// Required: 4. - pub(crate) unk_fbc: u32, - /// Enabled GPU cores. - pub(crate) unk_fc0: u32, - /// Performance table A, one voltage lane per cluster. - pub(crate) perf_a: PerfGroup, - pub(crate) unk_16c8: [u8; 0x140], - /// Frequency of each state of table B in MHz. - pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], - pub(crate) unk_1834: [u8; 0x14], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_1848: [u32; 11], - pub(crate) unk_1874: [u8; 0x54], - /// Relative performance of each state of table A, in percent. - pub(crate) perf_a_relative: [u32; PERF_STATES], - pub(crate) unk_18f4: [u8; 0x14], - /// Relative performance of each state of table B, in percent. - pub(crate) perf_b_relative: [u32; PERF_STATES], - pub(crate) unk_1934: [u8; 0x94], - /// Operating point of each state of table A. - pub(crate) perf_a_states: [u32; PERF_STATES], - pub(crate) unk_19f4: [u8; 0x14], - /// Operating point of each state of table B. - pub(crate) perf_b_states: [u32; PERF_STATES], - pub(crate) unk_1a34: [u8; 0x14], - /// CS clock-domain performance table. - pub(crate) cs_perf: AuxPerfTable, - /// AFR clock-domain performance table. - pub(crate) afr_perf: AuxPerfTable, - /// Performance table B, one voltage lane per cluster. - pub(crate) perf_b: PerfGroup, - pub(crate) unk_23dc: [u8; 0x168], - /// Required: 5. - pub(crate) unk_2544: u32, - /// Required: 6. - pub(crate) unk_2548: u32, - /// Required: 3. - pub(crate) unk_254c: u32, - /// Required: 7. - pub(crate) unk_2550: u32, - /// Required: 7. - pub(crate) unk_2554: u32, - pub(crate) unk_2558: [u8; 0x18], - /// Enabled GPU cores. - pub(crate) unk_2570: u32, - pub(crate) unk_2574: [u8; 4], - /// Required: 1. - pub(crate) unk_2578: u32, - pub(crate) unk_257c: [u8; 0x1c], - /// Required: 0, 1, 1, 1. - pub(crate) unk_2598: [u32; 4], - /// Required: 0, 1, 1, 1. - pub(crate) unk_25a8: [u32; 4], - pub(crate) unk_25b8: [u8; 4], - /// Required: 0xffff_ffff in every entry, one per work channel. - pub(crate) unk_25bc: [u32; 12], - pub(crate) unk_25ec: [u8; 8], - /// Required: 2. - pub(crate) unk_25f4: u32, - pub(crate) unk_25f8: [u8; 4], - /// Required: 1. - pub(crate) unk_25fc: u32, - pub(crate) unk_2600: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2614: u32, - pub(crate) unk_2618: [u8; 4], - /// Required: 0x100. - pub(crate) unk_261c: u32, - /// Completion-ring descriptors. - pub(crate) completion_rings: [CompletionRingDescriptor; 4], - pub(crate) unk_26a0: [u8; 0x18], - /// QoS object, zeroed. - pub(crate) qos_va: u64, - /// Required: 1. - pub(crate) unk_26c0: u32, - /// Required: 1. - pub(crate) unk_26c4: u32, - pub(crate) unk_26c8: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_26f4: u32, - pub(crate) unk_26f8: [u8; 0x16bc], -} - -static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); -static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); -static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); -static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); -static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); -static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); -static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); -static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); -static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); -static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); -static_assert!(size_of::() == 0x3db4); -static_assert!(size_of::() <= POWER_CONFIG); - -// SAFETY: `HwData25G83` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwData25G83 {} - -impl HwData25G83 { - /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. - pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { - super::clear(self); - self.unk_0 = 0x6f_0000_0000; - self.unk_8 = 0xffc0_0000; - self.unk_10 = 0x10_0000_0000; - self.unk_18 = 0x10_0000_0000; - self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = 0xffff_fc21_8140_0000; - - let mut used = [false; REGISTER_MAP_SLOTS]; - for window in args.windows { - match used.get_mut(window.slot) { - Some(used) if !*used => *used = true, - _ => return Err(EINVAL), - } - self.register_map[window.slot] = RegisterMapEntry::window(window); - } - if !args.reserved_slots.is_empty() { - return Err(EINVAL); - } - - self.chip_id = args.chip_id; - self.unk_e94 = 1; - self.unk_e98 = 1; - self.unk_ea0 = 1; - self.unk_ea4 = 1; - self.unk_eb8 = 1; - self.unk_ec0 = 1; - self.unk_ec8 = 1; - self.unk_ed0 = 0x5dc0; - self.unk_ed4 = 1; - self.unk_ed8 = 8; - self.unk_ee0 = [1; 3]; - self.unk_f04 = 0x1f; - self.unk_f24 = 0x0700_2004; - self.unk_f28 = 6; - self.unk_f34 = 1; - self.unk_f38 = 1; - self.unk_f4c = 0x31; - self.unk_f88 = ext.active_cores; - self.unk_f8c = 0xf; - self.unk_fb8 = 0x1c; - self.unk_fbc = 4; - self.unk_fc0 = ext.active_cores; - - let perf = args.perf; - let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; - let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; - if states_a[0].cells < 2 { - return Err(EINVAL); - } - self.perf_a.fill(&states_a, 1); - self.perf_b.fill(&states_b, 1); - for (index, state) in states_b.iter().enumerate() { - self.perf_b_freq_mhz[index] = state.freq_mhz; - self.perf_a_states[index] = perf.states_a[index].into(); - self.perf_b_states[index] = perf.states_b[index].into(); - } - self.perf_a_relative = perf.relative_a; - self.perf_b_relative = perf.relative_b; - self.unk_1848 = [0x3f82_8f5c; 11]; - self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; - self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; - - self.unk_2544 = 5; - self.unk_2548 = 6; - self.unk_254c = 3; - self.unk_2550 = 7; - self.unk_2554 = 7; - self.unk_2570 = ext.active_cores; - self.unk_2578 = 1; - self.unk_2598 = [0, 1, 1, 1]; - self.unk_25a8 = [0, 1, 1, 1]; - self.unk_25bc = [u32::MAX; 12]; - self.unk_25f4 = 2; - self.unk_25fc = 1; - self.unk_2614 = 1; - self.unk_261c = 0x100; - for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { - self.completion_rings[index as usize] = CompletionRingDescriptor { - low_va: ring.low_va, - high_va: ring.high_va, - entries: COMPLETION_RING_ENTRIES, - entry_size: COMPLETION_RING_ENTRY_SIZE, - index, - unk_1c: 0, - }; - } - self.qos_va = args.qos_va; - self.unk_26c0 = 1; - self.unk_26c4 = 1; - self.unk_26f4 = 1; - Ok(()) - } -} - -/// Power configuration control period, in 24 MHz clocks (8 ms). -const PERIOD_CLOCKS: u32 = 200_000; -/// Temperature sensors read by the die and MTR temperature limiters. -const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; - -/// Offset of the per-chip leakage coefficients in the power configuration. -const LEAKAGE_OFFSET: usize = 0x642c; -/// Offset in the power configuration of the GPU address of the configuration's status (the -/// configuration named by the root, `+0x80`). -const CONFIG_STATUS_POINTER: usize = 0x3b50; -/// Offset in the power configuration of the GPU address of the AON table. -const AON_POINTER: usize = 0x6318; -/// Offset in the power configuration of a random word the firmware expects. -const ENTROPY: usize = 0x63d4; - -/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets -/// from the start of the power configuration. -const POWER_CONFIG_WORDS: Words = &[ - (0x4, PERIOD_CLOCKS, 2), - (0x10, 0x4, 1), - (0x14, 0x3f800000, 1), - (0x2c, 0x1, 2), - (0x40, 0x64, 1), - (0x44, 0x1, 1), - (0x48, 0x3e8, 1), - (0x50, 0x64, 1), - (0x60, 0x4, 1), - (0x64, 0x3f800000, 1), - (0x6c, 0x1, 2), - (0x80, 0x64, 1), - (0x84, 0x1, 1), - (0x88, 0x3e8, 1), - (0x90, 0x64, 1), - (0xc0, 0x3f828f5c, 11), - (0x9b8, 0x271, 1), - (0x9c4, 0x3f7f2e9f, 1), - (0x9cc, 0x3b516154, 1), - (0x9d4, 0x3ca59586, 1), - (0x9e4, 0x40a90fdb, 1), - (0x9ec, 0xbc6d0cfe, 1), - (0x9f0, 0x64, 1), - (0x9f4, 0x3e8, 2), - (0xa38, 0x3e8, 1), - (0xa70, 0x8, 1), - (0xa7c, 0x3f000000, 1), - (0xa84, 0x3f000000, 1), - (0xa8c, 0x3e3bd7b3, 1), - (0xa98, 0x47800000, 1), - (0xa9c, 0x3ecf2e49, 1), - (0xaa8, 0x64, 1), - (0xaac, 0x3e8, 1), - (0xac8, 0x10, 1), - (0xad0, 0x5dc00, 1), - (0xb28, 0x50, 1), - (0xb30, 0x55, 1), - (0xb34, 0x32, 1), - (0xb38, 0x6, 1), - (0xb40, 0x6, 1), - (0xb44, 0x1, 1), - (0xb4c, 0x3f4ccccd, 1), - (0xb50, 0x3f7eb852, 1), - (0xb54, 0x3e4ccccd, 1), - (0xb58, 0x3ba3d70a, 1), - (0xb5c, 0x3f64c986, 2), - (0xb68, 0x42be0000, 1), - (0xb6c, 0x403ea7f0, 2), - (0xb74, 0x41179436, 1), - (0xb78, 0x64, 1), - (0xb7c, 0x3e8, 1), - (0xb80, 0x64, 1), - (0xb8c, 0x50, 1), - (0xbc0, 0x64, 1), - (0x1bec, 0xff000000, 1), - (0x1bf0, 0xff7fff7f, 255), - (0x1fec, 0x7fff7f, 1), - (0x21f0, 0x47800000, 1), - (0x21f4, 0x42c80000, 1), - (0x21f8, 0x447a0000, 1), - (0x21fc, 0x62a, 1), - (0x2210, 0x64, 1), - (0x2218, 0x42c80000, 1), - (0x221c, 0x3e8, 1), - (0x2224, 0x3f4ccccd, 1), - (0x2228, 0x3e4ccccd, 1), - (0x2268, TEMP_SENSOR_MASK, 1), - (0x2270, 0x1f40, 1), - (0x2278, 0x4, 1), - (0x228c, 0x3f800000, 1), - (0x2294, 0x414eeeee, 1), - (0x22a0, 0x47800000, 1), - (0x22a4, 0x42140000, 1), - (0x22b0, 0x64, 1), - (0x22b4, 0x3e8, 2), - (0x22c4, 0x2af8, 1), - (0x22f8, 0x3e8, 1), - (0x2728, 0xb8520000, 1), - (0x272c, 0x3f7e, 1), - (0x2730, 0xd70a0000, 1), - (0x2734, 0x3ba3, 1), - (0x273c, 0x43c8, 1), - (0x2748, 0x4780, 1), - (0x274c, 0x42c8, 1), - (0x2754, 0x384bac8, 1), - (0x2758, 0x3e80000, 2), - (0x2764, 0x80000000, 1), - (0x2768, 0x26ac453b, 1), - (0x279c, 0x3e80000, 1), - (0x27b8, 0xc80000, 1), - (0x2870, 0x3e8, 1), - (0x28c0, 0x1, 1), - (0x28e0, 0x3f800000, 1), - (0x28e4, 0x4, 1), - (0x28e8, 0x8, 1), - (0x28ec, 0x2ee00, 1), - (0x29bc, 0x7c, 1), - (0x29c8, 0x3f77bdef, 1), - (0x29d0, 0x3d042108, 1), - (0x29d8, 0x3d25bea1, 1), - (0x29e4, 0x47800000, 1), - (0x29e8, 0x3f4f2e49, 1), - (0x29f4, 0x28, 1), - (0x29f8, 0x3e8, 2), - (0x2a14, 0xf8, 1), - (0x2a1c, 0x5ad200, 1), - (0x2a3c, 0x3e8, 1), - (0x2b4c, 0x32, 1), - (0x2b50, 0x1, 1), - (0x2b58, 0x3f638e39, 1), - (0x2b5c, 0x3f2aaaab, 1), - (0x2b60, 0x3de38e39, 1), - (0x2b64, 0x3eaaaaab, 1), - (0x2b68, 0xbecccccd, 2), - (0x2b74, 0x47800000, 1), - (0x2b78, 0xc0a00000, 2), - (0x2b84, 0x64, 1), - (0x2b88, 0x3e8, 1), - (0x2b8c, 0x64, 1), - (0x2b94, 0x45fa0000, 1), - (0x2b98, 0x578, 1), - (0x2ba4, 0x48, 1), - (0x2ba8, 0x18, 1), - (0x2bac, 0x1a5e00, 1), - (0x2bb4, 0x8ca00, 1), - (0x2bc0, 0x47800000, 1), - (0x2be8, 0x1f40, 1), - (0x2bec, 0x4, 1), - (0x2bf0, 0x9c4, 1), - (0x2bf4, 0x258, 1), - (0x2bf8, 0x2, 1), - (0x2bfc, 0x4, 1), - (0x2c00, 0x32, 1), - (0x2c30, 0x47800000, 1), - (0x2c40, 0x64, 1), - (0x2c44, 0x3e8, 1), - (0x2e2c, 0x4, 1), - (0x2e38, 0x3f780000, 1), - (0x2e40, 0x3d000000, 1), - (0x2e48, 0x40a00000, 1), - (0x2e54, 0x47800000, 1), - (0x2e58, 0x41700000, 1), - (0x2e64, 0x384, 1), - (0x2e68, 0x3e8, 2), - (0x2e78, 0x3e8, 1), - (0x2eac, 0x3e8, 1), - (0x2eec, 0x64, 1), - (0x2f08, TEMP_SENSOR_MASK, 1), - (0x2f10, 0x7d, 1), - (0x2f14, 0x1, 1), - (0x3b58, TEMP_SENSOR_MASK, 1), - (0x3b60, 0x1, 1), - (0x5b64, 0x40a00000, 1), - (0x6320, u32::from_be_bytes(*b"g0TR"), 1), - (0x6324, u32::from_be_bytes(*b"g1TR"), 1), - (0x6360, 0x3, 1), - (0x6368, u32::from_be_bytes(*b"r0TR"), 1), - (0x6370, 0x1, 1), - (0x6420, 0x1, 1), -]; - -/// The power configuration and auxiliary data of the 25G83 build, at bundle -/// [`POWER_CONFIG`]. The main configuration names it. -#[repr(C)] -pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); - -// SAFETY: `PowerConfig25G83` is a byte array. -unsafe impl Zeroable for PowerConfig25G83 {} - -static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); - -impl PowerConfig25G83 { - /// Initializes the power configuration. - pub(crate) fn init( - &mut self, - config_va: u64, - aon_va: u64, - entropy: u32, - max_power_mw: u32, - leakage: &[u32; 56], - ) { - self.0.fill(0); - fill_words(&mut self.0, POWER_CONFIG_WORDS); - for (index, word) in leakage.iter().enumerate() { - let at = LEAKAGE_OFFSET + 4 * index; - self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); - } - // All copies of the power ceiling must describe the same chip. - for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { - self.0[offset..offset + 4].copy_from_slice(&max_power_mw.to_le_bytes()); - } - let power = crate::float::F32::from(max_power_mw).to_bits(); - for offset in [0x9e0, 0x2a04] { - self.0[offset..offset + 4].copy_from_slice(&power.to_le_bytes()); - } - self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] - .copy_from_slice(&(config_va + 0x80).to_le_bytes()); - self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); - self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); - } -} - -/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU -/// cluster 1 and SoC sensors. -pub(crate) const AON_TABLE: [u32; 9] = [ - u32::from_be_bytes(*b"g0TR"), - 0x006b_0001, - 0x0000_1401, - u32::from_be_bytes(*b"g1TR"), - 0x00b5_0001, - 0x0000_1401, - u32::from_be_bytes(*b"r0TR"), - 0x00c1_0001, - 0x0000_1401, -]; - -/// Nonzero words of the power-state object. The fast-die limiter reads its target -/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero -/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. -pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index ca2b6604a302b..e07fbfb485a10 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -278,20 +278,6 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } - - /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the - /// remaining writes, without first copying them elsewhere. - pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { - let count = writes.len().checked_sub(1).ok_or(EINVAL)?; - if skip > count || count > REGISTER_ARRAY_ENTRIES { - return Err(EINVAL); - } - self.entries[..skip].copy_from_slice(&writes[..skip]); - self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); - self.self_va = self_va; - self.size = ((count * size_of::()) as u32) << 16 | count as u32; - Ok(()) - } } /// Binding of one [`RegisterArray`] to a kick entry. @@ -311,7 +297,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, }) } @@ -346,7 +332,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | u64::from(self.count - 1) << Self::COUNT_SHIFT + | ((self.count - 1) as u64) << Self::COUNT_SHIFT } } @@ -458,8 +444,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | u64::from(args.qid) << Self::QID_SHIFT - | u64::from(args.priority) << Self::PRIORITY_SHIFT; + | (args.qid as u64) << Self::QID_SHIFT + | (args.priority as u64) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -492,9 +478,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: u64::from(mask) - | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT - | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT + dependency_control: mask as u64 + | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT + | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 1cb46fce2d602..b6bb6aaea860f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,9 +23,6 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; -pub(crate) mod variant; - -pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -33,7 +30,7 @@ pub(crate) use variant::Variant; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; + unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 66e7d07db48a5..48b110a4193e3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,8 +20,6 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; -use super::variant::PointerLayout; - use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -96,8 +94,6 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, - /// [`InitBufferManager`] (25G83 firmware). - InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -119,29 +115,26 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: the producer, with one aligned 32-bit store per publication +/// - publication: `producer`, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. -/// -/// Where the producer and the recovery target live depends on the firmware -/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds the producer. + /// Free ring space is measured from it, so it bounds `producer`. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and the producer are equal. + /// this cursor, `done` and `producer` are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// 26A428: item-ring position after the last work pointer the host published. - word_40: AtomicU32, + /// Item-ring position after the last work pointer the host published. + pub(crate) producer: AtomicU32, pub(crate) unk_44: [u32; 3], - /// 26A428: on recovery, the host copies the producer here before acknowledging resume. - word_50: AtomicU32, + /// On recovery, the host copies producer here before acknowledging resume. + pub(crate) recovery_target: u32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -150,62 +143,29 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); -static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); +static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - /// Offsets of the two host words a [`PointerLayout`] may name. - const WORD_40: usize = core::mem::offset_of!(Self, word_40); - const WORD_50: usize = core::mem::offset_of!(Self, word_50); - - /// Whether `offset` is one of the host words a [`PointerLayout`] may name. - const fn is_host_word(offset: usize) -> bool { - offset == Self::WORD_40 || offset == Self::WORD_50 - } - - /// Whether `layout` names host words of the block, and two different ones when it has a - /// recovery target. Every firmware variant's layout is checked at build time. - pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { - match layout.recovery_target { - None => Self::is_host_word(layout.producer), - Some(target) => { - Self::is_host_word(layout.producer) - && Self::is_host_word(target) - && target != layout.producer - } - } - } + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); + pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); /// Returns the cursors of an empty item ring. - pub(crate) fn new(layout: &PointerLayout) -> Result { - let block = Self { + pub(crate) const fn new() -> Self { + Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - word_40: AtomicU32::new(0), + producer: AtomicU32::new(0), unk_44: [0; 3], - word_50: AtomicU32::new(0), + recovery_target: u32::MAX, unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], - }; - if let Some(target) = layout.recovery_target { - block.word(target)?.store(u32::MAX, Ordering::Relaxed); - } - Ok(block) - } - - /// The host word at `offset`, which must be one a [`PointerLayout`] may name. - fn word(&self, offset: usize) -> Result<&AtomicU32> { - match offset { - Self::WORD_40 => Ok(&self.word_40), - Self::WORD_50 => Ok(&self.word_50), - _ => Err(EINVAL), } } @@ -214,14 +174,12 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and the producer, in this + /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { - let producer = self.word(layout.producer)?; + pub(crate) fn rebase(&self, base: u32) { self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - producer.store(base, Ordering::Relaxed); - Ok(()) + self.producer.store(base, Ordering::Relaxed); } } @@ -313,12 +271,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [u32::from(args.policy.priority()); 2], + priority: [args.policy.priority() as u32; 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: u32::from(args.policy.policy()), + policy: args.policy.policy() as u32, unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -349,12 +307,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [u32::from(policy.priority()); 2]; + let priority = [policy.priority() as u32; 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != u32::from(policy.policy()) { - self.policy = u32::from(policy.policy()); + if { self.policy } != policy.policy() as u32 { + self.policy = policy.policy() as u32; } } } @@ -522,7 +480,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - u32::from(binding.fragment_qid), + binding.fragment_qid as u32, ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -533,14 +491,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: u16::from(args.qid), - install: u16::from(args.install), + qid: args.qid as u16, + install: args.install as u16, unk_1c: 0, - priority: u16::from(args.policy.priority()), + priority: args.policy.priority() as u16, unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: u32::from(args.completion_seed.is_some()), + completion_seed_valid: args.completion_seed.is_some() as u32, completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -548,13 +506,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: u32::from(args.qos_update), - qos_slot: u16::from(args.qos_slot), + qos_update: args.qos_update as u32, + qos_slot: args.qos_slot as u16, unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: u16::from(args.context_update), + context_update: args.context_update as u16, unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -567,37 +525,6 @@ impl QueueConfig { } } -/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling -/// descriptor that uses a parameter buffer. The block count is read from the manager. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct InitBufferManager { - /// [`CommandTag::InitBufferManager`]. - pub(crate) tag: u32, - /// UAT context of the pass. - pub(crate) context_id: u32, - /// Buffer slot of the parameter buffer. - pub(crate) pb_slot: u32, - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// Tiling stamp value of the pass. - pub(crate) stamp_value: u32, -} - -static_assert!(size_of::() == 0x18); - -impl InitBufferManager { - pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { - Self { - tag: CommandTag::InitBufferManager as u32, - context_id, - pb_slot, - pb_va, - stamp_value, - } - } -} - /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -637,7 +564,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: u16::from(qid), + qid: qid as u16, count: 1, priority, timestamp: timestamp.get(), @@ -676,7 +603,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: u16::from(qid), + qid: qid as u16, previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index d0000282cef82..85a2385b421b8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,53 +48,10 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets -/// that follow the control words of each cluster. -#[derive(Copy, Clone, Debug)] -pub(crate) struct Deflake { - /// Offsets of the second and third sub-buffers. - pub(crate) sub: [u64; 2], - /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in - /// the third sub-buffer and the buffer is left to the GPU. - pub(crate) viewport: bool, -} - -impl Deflake { - /// One cluster (A18 Pro). - pub(crate) const ONE_CLUSTER: Deflake = Deflake { - sub: [0x20, 0x2a0], - viewport: true, - }; - /// Two clusters (M4 Pro). - pub(crate) const TWO_CLUSTERS: Deflake = Deflake { - sub: [0xa0, 0x820], - viewport: false, - }; -} - -/// The register programs of a firmware build's render descriptors. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum RegisterPrograms { - /// The A18 Pro firmware's (26A428). - G17P, - /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the - /// tiler parameter sub-buffers in the TPC object. - V25G83, -} - -/// The 25G83 tiling and main fragment arrays. -const TA_REGISTERS_25G83: usize = 83; -const FRAGMENT_REGISTERS_25G83: usize = 91; -const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; -const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; -/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. -const TA_SLOT_INDEX_25G83: u8 = 79; -/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. -const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; -/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, -/// which the compute program and both A18 Pro render programs carry for such work: first in the -/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. -const SCS_ENABLE: u32 = 0x017d9; +/// The deflake buffer holds three sub-buffers: at its start and at these +/// offsets. +const DEFLAKE_20: u64 = 0x20; +const DEFLAKE_2A0: u64 = 0x2a0; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -147,10 +104,6 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; -/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, -/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment -/// stage (the 3456x2234 panel is 108 x 70 tiles). -const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -162,14 +115,8 @@ const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; /// The partial-render phases load and store every attached depth and stencil /// surface. -const ZLS_STORE_DEPTH: u64 = 1 << 19; -const ZLS_STORE_STENCIL: u64 = 1 << 18; -const ZLS_LOAD_DEPTH: u64 = 1 << 15; -const ZLS_LOAD_STENCIL: u64 = 1 << 14; -const ZLS_COMPRESS_DEPTH: u64 = 1 << 6; -const ZLS_COMPRESS_STENCIL: u64 = 1 << 8; -/// Bits of the main word a partial render sets for itself. -const ZLS_PARTIAL_CLEAR: u64 = ZLS_COMPRESS_DEPTH | ZLS_COMPRESS_STENCIL | 0xf << 56; +const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; +const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; const STENCIL_CLEAR_BASE: u32 = 0x300; /// Value of the ZLS swizzle registers and descriptor field. const ZLS_SWIZZLE: u32 = 0x0404_0404; @@ -315,9 +262,6 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, - /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for - /// an empty one and on other GPUs. - pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -370,10 +314,6 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, - /// Register programs of the firmware build. - pub(crate) programs: RegisterPrograms, - /// Deflake-buffer layout of the GPU. - pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -417,12 +357,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = u64::from(pass.width); - let height = u64::from(pass.height); + let width = pass.width as u64; + let height = pass.height as u64; let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / u64::from(pass.utile_width); - let utiles_y = TILE_SIZE / u64::from(pass.utile_height); + let utiles_x = TILE_SIZE / pass.utile_width as u64; + let utiles_y = TILE_SIZE / pass.utile_height as u64; let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -449,7 +389,6 @@ struct PassWords { layer_mode: u64, tib_blocks: u64, partial_zls_control: u64, - partial_zls_reload: u64, aux_fb_flags: u64, bg_rsrc_spec: u64, bg_va: u64, @@ -472,7 +411,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); + let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -485,27 +424,12 @@ impl PassWords { tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; } - // The partial store keeps the main word, loads included, and stores and compresses each - // plane it has. The reload arrays add the loads, which a clearing pass doesn't have. - let mut partial_zls_control = pass.zls_control & !ZLS_PARTIAL_CLEAR; - if pass.depth.va != 0 { - partial_zls_control |= ZLS_STORE_DEPTH; - } - if pass.depth.compression_va != 0 { - partial_zls_control |= ZLS_COMPRESS_DEPTH; - } - if pass.stencil.va != 0 { - partial_zls_control |= ZLS_STORE_STENCIL; - } - if pass.stencil.compression_va != 0 { - partial_zls_control |= ZLS_COMPRESS_STENCIL; - } - let mut partial_zls_reload = partial_zls_control; + let mut partial_zls_control = pass.zls_control; if pass.depth.va != 0 { - partial_zls_reload |= ZLS_LOAD_DEPTH; + partial_zls_control |= ZLS_PARTIAL_DEPTH; } if pass.stencil.va != 0 { - partial_zls_reload |= ZLS_LOAD_STENCIL; + partial_zls_control |= ZLS_PARTIAL_STENCIL; } let mut aux_fb_flags = AUX_FB_FLAGS_BASE; @@ -517,23 +441,22 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u64::from(u32::MAX), + false => spec & u32::MAX as u64, }; - let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); + let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); Ok(Self { - utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | u64::from(pass.samples_log2), + utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | pass.samples_log2 as u64, tile_config, layer_mode, - tib_blocks: (u64::from(pass.sample_size) - * u64::from(pass.utile_width) - * u64::from(pass.utile_height) + tib_blocks: (pass.sample_size as u64 + * pass.utile_width as u64 + * pass.utile_height as u64 * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, - partial_zls_reload, aux_fb_flags, bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), bg_va: usc(pass.bg.offset)?, @@ -568,10 +491,9 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let [deflake_20, deflake_2a0] = args.deflake.sub; - let deflake = u64::from(user_offset32(r.deflake_va)?); - let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); - let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); + let deflake = user_offset32(r.deflake_va)? as u64; + let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; + let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -639,7 +561,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), + RegisterWrite::new(0x1c830, r.pb_slot as u64), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -651,7 +573,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), + RegisterWrite::new(0x14308, r.free_list_slot as u64), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -659,136 +581,6 @@ fn ta_registers( ]) } -/// Tiler segment control of a two-cluster pass, what macOS writes. -const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; -const VDM_SEGMENT_INDEX: u64 = 4; -/// Pages the tiler keeps out of a partial render's segments: without them the pages a pass -/// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. -const MIN_FREE_TVB_PAGES: u64 = 0x3c; - -/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage -/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the -/// M4 Pro has run with. Returns the array and its used entry count. -fn ta_registers_25g83( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { - let a = ta_registers(args, g, w)?; - let r = args.resources; - let gtp = |index: usize| -> Result { - match r.gtp_va[index] { - 0 => Ok(0), - va => user_offset(va), - } - }; - let find = |reg: u32, nth: usize| -> Result { - a.iter() - .filter(|write| write.reg == reg) - .nth(nth) - .copied() - .ok_or(EINVAL) - }; - let gtp1 = gtp(1)?; - let gtp2 = gtp(2)?; - let gtp6 = gtp(6)? & !127; - let scs = usize::from(args.ray_query[0].scratch.enabled()); - let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; - let mut n = 0; - let mut push = |write: RegisterWrite| { - out[n] = write; - n += 1; - }; - if scs != 0 { - push(find(SCS_ENABLE, 0)?); - } - for reg in [0x10141, 0x1c039, 0x1c9c8] { - push(find(reg, 0)?); - } - push(RegisterWrite::new(0x1c041, gtp1)); - push(RegisterWrite::new(0x1c9d0, gtp1)); - for reg in [ - 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, - 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, - ] { - push(find(reg, 0)?); - } - push(find(0x1c880, 0)?); - push(RegisterWrite::new(0x1c948, gtp(4)?)); - push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); - push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); - push(find(0x1c079, 0)?); - push(find(0x1c9d8, 0)?); - push(RegisterWrite::new(0x1c089, gtp2)); - push(RegisterWrite::new(0x1c9e0, gtp2)); - push(RegisterWrite::new(0x16c41, gtp6)); - push(RegisterWrite::new(0x1ca40, gtp6)); - push(RegisterWrite::new(0x1c9a8, MIN_FREE_TVB_PAGES)); - push(RegisterWrite::new(0x1c920, gtp(7)?)); - for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { - push(find(reg, 0)?); - } - push(RegisterWrite::new(0x10131, 0x88)); - for reg in [ - 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, - 0x1c8e0, 0x1c8e8, - ] { - push(find(reg, 0)?); - } - push(find(0x1c898, 0)?); - for reg in [ - 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, - 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, - 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, - ] { - push(find(reg, 0)?); - } - push(find(0x1c880, 1)?); - push(find(0x1c898, 1)?); - if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { - return Err(EINVAL); - } - Ok((out, n)) -} - -/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with -/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 -/// Pro has run with. Returns the array and its used entry count. -fn fragment_registers_25g83( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { - let a = fragment_registers(args, g, w); - let scs = usize::from(args.ray_query[1].scratch.enabled()); - let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; - let mut n = 0; - let mut tile_state = false; - for (index, write) in a.iter().enumerate() { - if write.reg == SCS_ENABLE && scs == 0 { - continue; - } - if index == a.len() - 1 { - *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); - n += 1; - } - *out.get_mut(n).ok_or(EINVAL)? = *write; - n += 1; - if write.reg == 0x16068 && !tile_state { - tile_state = true; - *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); - n += 1; - } - } - if n != FRAGMENT_REGISTERS_25G83 + scs - || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 - { - return Err(EINVAL); - } - Ok((out, n)) -} - fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -796,12 +588,7 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let tile_state_base = if args.programs == RegisterPrograms::V25G83 { - TILE_STATE_BASE_25G83 - } else { - TILE_STATE_BASE - }; - let mut tile_state = tile_state_base + let mut tile_state = TILE_STATE_BASE | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -822,8 +609,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), - RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), + RegisterWrite::new(0x15131, p.merge_upper_x as u64), + RegisterWrite::new(0x15139, p.merge_upper_y as u64), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -838,15 +625,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), + RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, u64::from(p.depth_clear)), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x15301, p.depth_clear as u64), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -895,11 +682,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), + RegisterWrite::new(0x1c838, r.pb_slot as u64), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), + RegisterWrite::new(0x14048, r.free_list_slot as u64), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -929,7 +716,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -955,13 +742,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_reload), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -972,16 +759,14 @@ fn partial_load_registers( [ RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), - // Only the main end-of-tile program resolves, and it has to reach tiles that only - // earlier segments drew to. Without this their resolve target is never written. - RegisterWrite::new(0x10039, w.tile_config | TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_reload), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -1055,11 +840,7 @@ pub(crate) struct TaDescriptor { /// QID of the tiling queue. pub(crate) qid: u32, pub(crate) unk_8be: u32, - /// Non-zero on the first kick of a queue pair, like the fragment descriptor's word at the - /// same position. Set on every kick, the firmware also notifies the host of each pass's - /// tiling completion, which no retirement needs. The 25G83 firmware keeps it set on every - /// kick until this is measured there. - pub(crate) first_kick: u32, + pub(crate) unk_8c2: u32, /// Completion value of the tiling kick (see /// [`KickTimestamp::completion_value`]). pub(crate) completion: u64, @@ -1108,6 +889,8 @@ unsafe impl Zeroable for TaDescriptor {} impl TaDescriptor { const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; const UNK_876: u32 = u32::MAX; + const UNK_892: u32 = 1; + const UNK_8C2: u32 = 1; const UNK_932: u8 = 0x44; const UNK_93C: u8 = 1; const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; @@ -1119,8 +902,7 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = - user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; + let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -1137,14 +919,12 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if args.programs == RegisterPrograms::V25G83 { - self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); - let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; - self.registers.set(array_va, ®isters[..count])?; - } else if scs_register { + if scs_register { self.registers.set(array_va, ®isters)?; } else { - self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; + let mut trimmed = [registers[0]; TA_REGISTERS]; + trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); + self.registers.set(array_va, &trimmed)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1156,24 +936,16 @@ impl TaDescriptor { self.sampler_heap_va = pass.sampler_heap_va; self.sampler_count = pass.sampler_count; self.sampler_max = sampler_max; - // Asks the firmware to grow the heap. macOS keeps that for memoryless passes, which can't - // partial-render, and Honeykrisp has none. 26A428 keeps it: untested there. - self.unk_892 = u32::from(args.programs != RegisterPrograms::V25G83); + self.unk_892 = Self::UNK_892; self.stamp_va = args.ta.stamp_va; self.aux_stamp_va = args.ta.aux_stamp_va; self.stamp_value_2 = stamp_value; self.qid = args.ta.qid.into(); - self.first_kick = match args.programs == RegisterPrograms::V25G83 { - true => 1, - false => u32::from(args.ordinal == 0), - }; + self.unk_8c2 = Self::UNK_8C2; self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = match args.programs == RegisterPrograms::V25G83 { - true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), - false => Self::UNK_932, - }; + self.unk_932 = Self::UNK_932; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -1197,7 +969,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = u32::from(fragment_busy); + self.fragment_busy = fragment_busy as u32; self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -1208,14 +980,11 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, - programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let checkpoint = match programs == RegisterPrograms::V25G83 { - true => TA_CHECKPOINTS_25G83[0] as u8, - false => TA_CHECKPOINTS[0] as u8, - } + u8::from(scs_register); - let array = - RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; + let array = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ARRAY_OFFSET)?, + TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), + )?; Ok([Some(array), None, None, None]) } } @@ -1402,18 +1171,13 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, - programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; - let checkpoint = match programs == RegisterPrograms::V25G83 { - true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, - false => FRAGMENT_CHECKPOINTS[0] as usize, - } + usize::from(scs_register); Ok([ - array(0, checkpoint)?, + array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1485,15 +1249,13 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if args.programs == RegisterPrograms::V25G83 { - desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); - desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; - let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; - desc.arrays[0].set(arrays[0], ®isters[..count])?; - } else if scs_register { + if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; + let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; + trimmed[FRAGMENT_SCS_ENABLE_INDEX..] + .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); + desc.arrays[0].set(arrays[0], &trimmed)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; @@ -1527,15 +1289,12 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = u32::from(args.ordinal == 0); + desc.first_kick = (args.ordinal == 0) as u32; desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { - true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), - false => Self::UNK_21CC, - }; + desc.unk_21cc = Self::UNK_21CC; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs deleted file mode 100644 index 03d5bfccafeca..0000000000000 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! G17 firmware builds. -//! -//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) -//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences -//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it -//! runs. - -/// Where a firmware build keeps the host words of a queue's pointer block -/// ([`super::queue::PointerBlock`]). -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct PointerLayout { - /// Offset of the item-ring producer. - pub(crate) producer: usize, - /// Offset of the word the host copies the producer to before it acknowledges a recovery, - /// if the build has one. - pub(crate) recovery_target: Option, -} - -/// How a firmware build acknowledges the initialization root the host hands it. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum InitAck { - /// With an `INITDATA_ACK` message on the firmware endpoint. - Message, - /// By setting word `+0x14` of the primary status A to 1; it sends no message. - Status, -} - -/// The initialization graph of a firmware build. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum InitGraph { - /// Two instances, as the A18 Pro firmware (26A428) runs. - G17P, - /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). - V25G83, -} - -/// The layouts and rules of one G17 firmware build. -pub(crate) struct Variant { - /// The initialization graph. - pub(crate) graph: InitGraph, - /// How the firmware acknowledges its initialization. - pub(crate) init_ack: InitAck, - /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor - /// is initialized. - pub(crate) power_on_done: bool, - /// First QID the host may use; the firmware keeps the lower ones for itself. - pub(crate) qid_floor: usize, - /// Host words of a queue's pointer block. - pub(crate) pointers: PointerLayout, - /// The host registers each render pair through the GPU's MMIO queue ports. - pub(crate) mmio_pair_registration: bool, - /// Render kick entries may name an MCache table. - pub(crate) mcache: bool, - /// Renders may set the few-primitives auxiliary framebuffer flag. - pub(crate) few_primitives: bool, - /// The firmware runs ray-traversal work (ray queries and intersection programs). - pub(crate) ray_traversal: bool, - /// Tiling descriptors report whether the fragment queue still had earlier kicks - /// outstanding (`fragment_busy`). - pub(crate) fragment_busy: bool, - /// Render item rings carry lead commands: InitBufferManager before the first tiling pass - /// of a buffer manager. - pub(crate) lead_commands: bool, - /// Register programs of the render descriptors. - pub(crate) programs: super::render::RegisterPrograms, -} - -/// A firmware build a SoC supports, identified by its loaded TEXT. -pub(crate) struct Build { - /// macOS version (major, minor, patch), used to order image candidates. - pub(crate) os_version: [u32; 3], - /// Name, for the log. - pub(crate) name: &'static str, - /// The admitted loaded images, measured from the `fw-text` region. - pub(crate) images: &'static [crate::fwid::FirmwareBuild], - /// Layouts and rules of the build. - pub(crate) variant: &'static Variant, -} - -/// How a SoC's firmware build is determined. -pub(crate) enum Firmware { - /// The SoC runs a single build, which is not identified (A18 Pro). - Fixed(&'static Variant), - /// One of these exact loaded images. The loader's `apple,firmware-version` is only - /// a candidate-ordering hint because multiple OS releases can share an iBoot string. - Identified(&'static [Build]), -} - -/// The A18 Pro firmware (26A428). -pub(crate) const V26A428: Variant = Variant { - graph: InitGraph::G17P, - init_ack: InitAck::Message, - power_on_done: true, - qid_floor: 0, - pointers: PointerLayout { - producer: 0x40, - recovery_target: Some(0x50), - }, - mmio_pair_registration: true, - mcache: true, - few_primitives: true, - ray_traversal: true, - fragment_busy: true, - lead_commands: false, - programs: super::render::RegisterPrograms::G17P, -}; - -/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). -pub(crate) const V25G83: Variant = Variant { - graph: InitGraph::V25G83, - init_ack: InitAck::Status, - // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts - // its initialization. - power_on_done: false, - // QIDs below 32 belong to the firmware's bootstrap protocol. - qid_floor: 32, - pointers: PointerLayout { - producer: 0x50, - recovery_target: None, - }, - // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run - // without it. - mmio_pair_registration: false, - mcache: false, - // A fragment stage with the few-primitives flag hangs. - few_primitives: false, - ray_traversal: true, - // When a tiling pass runs out of memory with this set, the firmware resumes it without - // adding memory and the tiler stops silently; macOS always writes 0. - fragment_busy: false, - lead_commands: true, - programs: super::render::RegisterPrograms::V25G83, -}; - -const _: () = { - assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); - assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); -}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 578bb7533f3c9..d0dcc85c7f09e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,8 +16,6 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; -mod v25g83; - const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -38,30 +36,6 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } -/// Records in the compute completion ring that the host scans. -pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { - let slots = abi::COMPLETION_RING_SIZE / 0x40; - if slots < 0x800 { - slots - } else { - 0x800 - } -}; - -/// Timestamp bits of a completion record's start and end words. -const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; - -/// What one completion-ring slot gave a scan. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum CompletionFold { - /// No new observation of a pending request; the scan continues. - Passed, - /// Every request now has an observation. - Done, - /// A pending request's record whose payload the firmware is still writing. - Incomplete, -} - impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -147,12 +121,20 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - core::ptr::from_ref(self.record).cast::(), + self.record as *const _ as *const u8, self.destination, core::mem::size_of_val(self.record), ); } } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.next + } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -173,15 +155,6 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, - /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). - pm_metrics_va: u64, - /// `Config::recovery_slot_keys`. - recovery_slot_keys: Option, - /// Primary status A and status B in the cluster. - status_a: usize, - status_b: usize, - /// Separately allocated views; `None` when they are packed into the bundle. - views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -201,19 +174,11 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, - /// GPU clusters, as the GPU reports them. - clusters: u32, } impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed( - &self, - offset: u64, - size: usize, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { + fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { KernelObject::new( self.dev, self.vm, @@ -225,13 +190,7 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { + fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { KernelObject::new( self.dev, self.vm, @@ -276,11 +235,6 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, - pm_metrics_va: 0, - recovery_slot_keys: None, - status_a: abi::private::PRIMARY_STATUS_A, - status_b: abi::private::STATUS_B, - views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -293,10 +247,6 @@ impl InitData { } } - pub(super) fn pm_metrics_va(&self) -> u64 { - self.pm_metrics_va - } - pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -336,12 +286,7 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks( - &self, - slot: u16, - page_list_va: u64, - blocks: usize, - ) -> Result { + pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -418,7 +363,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); + | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); } Ok(()) })?; @@ -427,12 +372,6 @@ impl InitData { Ok(()) } - pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { - self.compute_shared - .as_ref() - .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) - } - pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -549,7 +488,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - // Compute publications reserve their slots through the port. + 2 => DataMaster::Compute, _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -563,7 +502,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - self.state_offset(Role::Primary) + Self::state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -583,7 +522,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -602,6 +541,16 @@ impl InitData { Ok(values) } + /// Inner queue records must be visible before this outer publication. The returned + /// producer is the retained consumer witness for this slot. + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) + } + /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -652,7 +601,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - self.state_offset(Role::Primary) + Self::state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -700,117 +649,21 @@ impl InitData { Ok(()) } - /// Raw views of the memory publishers share without the device mutex: every work ring, - /// the completion stamps, the QoS table and the recovery state word. They stay valid for - /// this session. - pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { - use super::fw::{ - channels, - queue::{DataMaster, Policy}, - }; - let ring_of = |priority: u8, engine: DataMaster| -> Result { - let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); - let state = self.object(self.cluster)?; - let word = - |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; - Ok(super::port::RingView { - slots: self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel), - channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), - )?, - consumers: [ - word(abi::ChannelState::CONSUMER)?, - word(abi::ChannelState::UNK_10)?, - ], - producer: word(abi::ChannelState::PRODUCER)?, - }) - }; - let rings = - |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { - let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; - for (priority, view) in views.iter_mut().enumerate().skip(1) { - *view = ring_of(priority as u8, engine)?; - } - Ok(views) - }; - Ok(super::port::Views { - compute: rings(DataMaster::Compute)?, - tiling: rings(DataMaster::Tiling)?, - fragment: rings(DataMaster::Fragment)?, - stamps: self.object(self.stamps[0])?.word(0)?, - qos: self - .qos()? - .pointer(0, core::mem::size_of::())?, - recovery: self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, - mailbox, - }) - } - /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - /// Folds the compute completion records the firmware appended since `start`, in ring - /// order. `seen` holds every slot's control word as of the previous pass over it: the - /// first slot still holding it is where the firmware writes next, and the pass ends there - /// instead of reading the rest of the ring. A pass also ends once `record` has found every - /// request, or at a requested record whose payload is still being written, which the next - /// pass reads again. Returns the slot the next pass starts at. - pub(super) fn fold_compute_completions( + pub(super) fn scan_compute_completions( &self, - start: usize, - seen: &mut [u64], - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { - let object = self.object(self.completion_rings[0])?; - if seen.len() != COMPUTE_COMPLETION_SLOTS { - return Err(EINVAL); - } - fence(Ordering::Acquire); - let completion = |slot: usize| ComputeCompletion { - object, - offset: slot * 0x40, - }; - super::queue::compute::retirement::walk_completions( - start, - seen, - |slot| completion(slot).read(0), - |slot| record(completion(slot)), - ) - } - - /// Folds every slot of the compute completion ring and records its control word in - /// `seen`. For a pass that stopped before a record the firmware must already have - /// written. Returns the slot after the newest record (latest end timestamp), where the - /// firmware writes next. - pub(super) fn resync_compute_completions( - &self, - seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; - if seen.len() != COMPUTE_COMPLETION_SLOTS { - return Err(EINVAL); - } fence(Ordering::Acquire); - let mut newest = None; - for (slot, seen_control) in seen.iter_mut().enumerate() { - let completion = ComputeCompletion { + for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { + record(ComputeCompletion { object, offset: slot * 0x40, - }; - let control = completion.read(0)?; - *seen_control = control; - if control != 0 { - let end = completion.read(5)? & COMPLETION_TIME_MASK; - if newest.is_none_or(|(_, latest)| end > latest) { - newest = Some((slot, end)); - } - } - record(completion)?; + })?; } - Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) + Ok(()) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -834,10 +687,6 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. - #[expect( - clippy::too_many_arguments, - reason = "The firmware graph builder keeps its independent input objects explicit." - )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -845,12 +694,7 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, - clusters: u32, - active_cores: u32, - variant: &super::fw::Variant, ) -> Result { - self.pm_metrics_va = cfg.pm_metrics.va; - self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -859,13 +703,9 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, - clusters, }; - if variant.graph == super::fw::variant::InitGraph::V25G83 { - return self.build_25g83(&env, cfg, metrics, active_cores); - } - self.map_register_windows(&env, cfg.windows)?; - let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; + self.map_register_windows(&env)?; + let tables = self.build_descriptor_tables(&env)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -873,41 +713,31 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows( - &mut self, - env: &BuildEnv<'_>, - windows: &[abi::RegisterWindow], - ) -> Result { - for window in windows { - for instance in 0..window.instances { - let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; - } + fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { + for window in &t8140::REGISTER_WINDOWS { + let (phys, va, size) = window.mapping().ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables( - &mut self, - env: &BuildEnv<'_>, - cluster_size: usize, - ) -> Result { + fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -915,8 +745,7 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = - self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -1000,11 +829,7 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing( - env.dev, - low::RENDER_PAGE_LIST_SIZE, - WB, - )?)?; + let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -1012,18 +837,10 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing( - env.dev, - low::RENDER_RUN_LIST_SIZE, - WB, - )?)?; + let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing( - env.dev, - abi::FREE_LIST_BLOCK_SIZE, - WB, - )?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } self.render_pages = pages; self.render_runs = runs; @@ -1031,12 +848,7 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias( - first_block + block, - env.render, - low::render_block(block), - GS, - )?; + self.alias(first_block + block, env.render, low::render_block(block), GS)?; } Ok(first_block) } @@ -1073,8 +885,12 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = - self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + let ready = self.add(env.fixed( + high::COMPUTE_READY_FREE_LIST, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1123,12 +939,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { + for slot in 0..2 { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - *completion_ring = abi::CompletionRing { + completion_rings[slot] = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -1172,13 +988,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let opps = Self::opps(env.dev)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: cfg.windows, - reserved_slots: cfg.reserved_slots, + windows: &t8140::REGISTER_WINDOWS, + reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, opps: &opps, - perf: cfg.perf, + perf: &t8140::PERF, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1186,7 +1002,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(); Ok(()) })?; self.object_mut(self.cluster)? @@ -1234,8 +1050,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1291,7 +1107,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1334,7 +1150,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1364,69 +1180,37 @@ impl InitData { .load(Ordering::Relaxed)) } - /// The GPU's operating points: `count` of them, each with one voltage cell for the whole - /// GPU or one per cluster. - pub(super) fn opps( - dev: &AsahiDrmDevice, - count: usize, - clusters: u32, - ) -> Result> { - let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; - if opps.len() != count { - return Err(EINVAL); - } - let cells = opps.first().map_or(0, |opp| opp.cells); - if cells != 1 && u32::try_from(cells) != Ok(clusters) { - dev_err!( - dev.as_ref(), - "GPU operating points have {} voltage cells, expected 1 or {}\n", - cells, - clusters - ); - return Err(EINVAL); - } - Ok(opps) - } - - /// The operating points of the table that the GPU node's property `name` names; all have - /// the same number of voltage cells. - fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { + fn opps(dev: &AsahiDrmDevice) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node.parse_phandle(name, 0).ok_or(EIO)?; + let table = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { - // One cell for the whole GPU, or one per cluster. - let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; - if cells.is_empty() || cells.len() > abi::OPP_CELLS { - return Err(EINVAL); - } - let mut volt_uv = [0; abi::OPP_CELLS]; - volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv, - cells: cells.len(), + volt_uv: opp.get_property(c_str!("opp-microvolt"))?, }, GFP_KERNEL, )?; } - if opps.iter().any(|opp| opp.cells != opps[0].cells) { + if opps.len() != t8140::OPPS { return Err(EINVAL); } Ok(opps) } - fn state_offset(&self, role: Role) -> usize { + fn state_offset(role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(&self, role: Role) -> usize { + fn status_offset(role: Role) -> usize { match role { - Role::Primary => self.status_a, + Role::Primary => abi::private::PRIMARY_STATUS_A, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1434,7 +1218,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1463,17 +1247,12 @@ impl InitData { Ok(()) } - /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). - pub(super) fn init_accepted(&self) -> Result { - Ok(self - .status_word(Role::Primary, 0x14)? - .load(Ordering::Acquire) - == 1) - } - pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)? - .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) + self.object(self.cluster)?.word( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) } pub(super) fn status_read_words( @@ -1481,8 +1260,11 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)? - .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) + self.object(self.cluster)?.read_words( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1554,7 +1336,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1569,55 +1351,39 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; + let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; - // Separately allocated views live at offset 0 of their own object. - let (object, base) = match self.views { - Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), - None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), - }; - let end = offset - .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) - + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - object.read_words(base + offset) + bundle.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self - .object(self.cluster)? - .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + let mask = self.object(self.cluster)? + .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - // Progress is independently available even when this firmware's key - // address is unknown. Zero keeps ownership unknown and cannot override - // firmware blame, but preserves the progress evidence for diagnostics. - let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); - let monitor = self - .recovery_view_words::<6>( - abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, - ) - .ok() - .map(|words| words[0])?; + let key = super::recovery::slot_key(slot)?; + let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) + .ok().map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self - .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok() - .map(|words| words[0]); + sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok().map(|words| words[0]); Ok(sources) } @@ -1625,21 +1391,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1651,21 +1417,17 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of - // their own object. - let (table, offset) = match self.views { - Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), - None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), - }; + let bundle = self.object(self.bundle)?; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - table.pointer(offset, ENTRIES * STRIDE)?; + let offset = abi::bundle::VIEWS[4]; + bundle.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - table + bundle .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - table + bundle .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1673,22 +1435,17 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entry(&self, index: usize) -> Result<(u32, u32)> { - if index >= super::recovery::RECOVERY_RECORDS { - return Err(EINVAL); - } + fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let offset = self.status_b - + abi::PrimaryStatusB::RECOVERY_INFO - + index * core::mem::size_of::(); + let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); - let flags = cluster - .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? - .load(Ordering::Relaxed); - let stamp = cluster - .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? - .load(Ordering::Relaxed); - Ok((flags, stamp)) + let mut entries = super::recovery::Entries::default(); + for index in 0..256 { + let offset = base + index * core::mem::size_of::(); + let flags = cluster.word(offset)?.load(Ordering::Relaxed); + entries.include(flags); + } + Ok(entries) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs deleted file mode 100644 index 670f4b96dcb76..0000000000000 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! The initialization graph of the 25G83 firmware build. -//! -//! One coprocessor carries both roles: there is one main configuration, with the -//! secondary-only words, and one merged root. The cluster, the hardware data, the power -//! configuration, region C, the views and the firmware area follow the 25G83 layouts -//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. - -use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; -use crate::g17::{calibration::Calibration, Config, Role}; -use crate::mmu; -use abi::v25g83 as v; -use core::sync::atomic::{fence, Ordering}; -use kernel::{c_str, prelude::*}; - -/// Object indices of the 25G83-specific objects. -struct Objects { - views: [usize; 5], - firmware_area: usize, - power_state: usize, - aon: usize, -} - -impl InitData { - /// Constructs the 25G83 graph after the firmware has published its root entries. - pub(super) fn build_25g83( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - metrics: &KernelObject, - active_cores: u32, - ) -> Result { - self.status_a = v::private::STATUS_A; - self.status_b = v::private::STATUS_B; - self.map_register_windows(env, cfg.windows)?; - let tables = self.build_descriptor_tables(env, v::private::SIZE)?; - self.build_render_free_lists(env)?; - let first_block = self.build_render_pool(env)?; - self.build_compute_pools(env, first_block)?; - let bundle = self.build_bundle(env, cfg, metrics)?; - let objects = self.build_25g83_objects(env)?; - self.views = Some(objects.views); - self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; - if cfg.uat_quirks.clean_tables { - // Write back every host-built byte before the firmware may read it. - for object in self.objects.iter_mut() { - let bytes = object.bytes_mut(); - crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); - } - crate::pgtable::poc_barrier(); - } - Ok(()) - } - - /// The views, the firmware area, the power-state object and the AON table. - fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { - use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; - use CpuMap::WriteCombined as WC; - let mut views = [0; 5]; - for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { - *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; - } - let firmware_area = - self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; - let power_state = - self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; - let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; - Ok(Objects { - views, - firmware_area, - power_state, - aon, - }) - } - - /// Writes the 25G83 firmware objects using the chip calibration, then - /// stages the opening control-ring state. - fn initialize_25g83( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - active_cores: u32, - tables: &DescriptorTables, - bundle: &BundleObjects, - objects: &Objects, - ) -> Result { - let bundle_va = self.object(self.bundle)?.gpu_va(); - let cluster_va = self.object(self.cluster)?.gpu_va(); - let config_va = cluster_va + v::private::CONFIG as u64; - let status_a_va = cluster_va + v::private::STATUS_A as u64; - let status_b_va = cluster_va + v::private::STATUS_B as u64; - let aon_va = self.object(objects.aon)?.gpu_va(); - let power_state_va = self.object(objects.power_state)?.gpu_va(); - let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); - let mut view_va = [0; 5]; - for (va, index) in view_va.iter_mut().zip(objects.views) { - *va = self.object(index)?.gpu_va(); - } - - let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; - let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; - let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; - let calibration = Calibration::read(env.dev)?; - calibration.check_opps(&opps)?; - let hw_args = abi::HwDataArgs { - chip_id: cfg.chip_id, - windows: cfg.windows, - reserved_slots: cfg.reserved_slots, - opps: &opps, - perf: &calibration.perf, - completion_rings: bundle.completion_rings, - qos_va: self.object(bundle.qos)?.gpu_va(), - }; - let hw_args_25g83 = v::HwDataArgs25G83 { - active_cores, - cs_opps: &cs_opps, - afr_opps: &afr_opps, - }; - // SAFETY: Plain C entropy source. - let entropy = unsafe { kernel::bindings::get_random_u32() }; - self.object_mut(self.bundle)? - .initialize::(0, |hw| { - hw.init(&hw_args, &hw_args_25g83)?; - Ok(()) - })?; - self.object_mut(self.bundle)? - .initialize::(v::POWER_CONFIG, |power| { - power.init( - config_va, - aon_va, - entropy, - calibration.max_power_mw, - &calibration.leakage, - ); - Ok(()) - })?; - self.object_mut(self.region_c)? - .initialize::(0, |region| { - region.init(cfg.region_c); - region.max_power_mw = calibration.max_power_mw; - Ok(()) - })?; - for (index, word) in v::AON_TABLE.iter().enumerate() { - self.object_mut(objects.aon)?.write(index * 4, *word)?; - } - - // Status A and B: only the boot words and the firmware-control pointers. - self.object_mut(self.cluster)? - .write(v::private::STATUS_A, abi::StatusBlock::new())?; - self.object_mut(self.cluster)? - .write(v::private::STATUS_B, abi::StatusBlock::new())?; - // Status B keeps the A18 Pro's field offsets up to its recovery words. - let fwctl_va = self.object(tables.fwctl)?.gpu_va(); - let fwctl = - v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); - let cluster = self.object(self.cluster)?; - cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); - cluster - .dword(fwctl + 8)? - .store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); - - // One main configuration carrying both roles. - let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; - let main_va = bundle_va + main_offset as u64; - let args = abi::MainConfigArgs { - bundle_va, - main_config_va: main_va, - state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, - status_a_va, - pb_descriptors_low_va: low::PB_DESCRIPTORS, - page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, - }; - let views = abi::PrimaryViews { - zero_page_va: self.object(bundle.zero)?.gpu_va(), - pb_descriptors_va: self.object(tables.pb)?.gpu_va(), - page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), - }; - self.object_mut(self.bundle)? - .initialize::(main_offset, |main| { - main.init_primary(&args, &views); - main.view_va = view_va; - main.firmware_area_va = [firmware_area_va; 2]; - main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); - // The secondary role's words, on the single instance. - main.unk_300 = 4; - main.power_config_va = bundle_va + v::POWER_CONFIG as u64; - main.unk_32c = 0; - main.unk_344 = 0; - Ok(()) - })?; - self.object_mut(self.bundle)?.write( - main_offset + abi::MainConfig::CONTROL_RING, - abi::OpeningRecord::new(abi::Role::Primary), - )?; - - // The merged root. - let root = abi::RootArgs { - region_a_va: self.object(bundle.region_a)?.gpu_va(), - region_c_va: self.object(self.region_c)?.gpu_va(), - main_config_va: main_va, - status_a_va, - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }; - self.object_mut(self.roots)?.write( - 0, - abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), - )?; - for &(offset, value, repeat) in v::POWER_STATE_WORDS { - for index in 0..repeat { - self.object_mut(objects.power_state)? - .write(offset + 4 * index, value)?; - } - } - - // The opening control record is staged in slot 0. - for (offset, value) in [ - (0, 0), - (0x10, 0), - (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), - ] { - self.object(self.cluster)? - .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? - .store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 7e43c163ac4a7..b5a5b3759de6d 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,11 +13,8 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -/// One logical render queue owns one physical pair, whose active ring holds -/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in -/// admission until a retirement; the scheduler waits for the credit instead. -const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -146,49 +143,29 @@ impl Packet { } } - /// Replay holds the device mutex. Its original inputs were satisfied before - /// first publication; never sleep on a fence that needs event service here. - pub(crate) fn check_replay_dependencies(&self) -> Result { - for fence in &self.inputs { - super::dependency::replay_status(fence_status(fence))?; - } - for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; - Ok(()) - } - /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { - let polls = if self.command.long_job() { 5 } else { 1 }; - let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - loop { - if self.cancel_requested() || self.completion.status().get() != 0 { - return Err(ECANCELED); - } - if fence_status(fence) != 0 { break; } - if super::now_ns() >= deadline { return Err(ECANCELED); } - // SAFETY: The retained fence owns the pointer throughout the bounded wait. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::time::msecs_to_jiffies(10) as isize, - ) - }; - // An interrupted wait is not a producer failure. - if waited < 0 { - return Err(Error::from_errno(waited as i32)); - } - if waited > 0 { break; } + // SAFETY: The retained fence owns the pointer throughout the wait and read. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::task::MAX_SCHEDULE_TIMEOUT, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited == 0 { + return Err(ECANCELED); } check_status(fence)?; } for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; Ok(()) } @@ -202,13 +179,6 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } - /// Gives up a claim taken from the submitting ioctl before the job was - /// pushed: nothing became visible, and the scheduler claims it again. - pub(crate) fn abandon_publication(&self) { - let _ = self - .publication - .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); - } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -231,7 +201,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |()| 1), + result.map_or_else(|error| error.to_errno(), |_| 1), Ordering::Release, ); } @@ -258,7 +228,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 { fn check_status(fence: &Fence) -> Result { let status = fence_status(fence); if status < 0 { - Err(super::dependency::PRODUCER_FAILED) + Err(super::dependency::inherited_error(Error::from_errno(status))) } else { Ok(()) } @@ -268,15 +238,11 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. - fn input_sources(&self, _inputs: &[crate::file::SyncItem]) - -> Option> { None } - fn feed(&self) -> Option> { None } - /// Validate compute sizing without reserving a physical channel. Queued - /// commands may wait on a host signal submitted after this ioctl returns. + /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -286,16 +252,12 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - /// A fence to wait for before `publish`, signalled by the release of the capacity the - /// packet lacks. Called again once it signals. - fn prepare(&self, packet: &Arc) -> Option; + fn prepare(&self, packet: &Packet) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; - /// A long job of another queue is running beside `packet`. - fn long_job_beside(&self, _packet: &Arc) -> bool { false } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -309,8 +271,6 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, - execution_renewals: u8, - shared_renewals: u8, packet: Arc, } @@ -327,15 +287,12 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = - Some(kernel::module::this_module::()); + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, - execution_renewals: 0, - shared_renewals: 0, packet: self.packet.clone(), } } @@ -344,37 +301,18 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } - if super::timeout::renew_execution( - job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, - ) { - return true; - } - // Other work retiring means the GPU is shared, not hung. So does another long job - // running beside a long job, though neither retires until it ends. - let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) - || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); - if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { - return true; - } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred { - job.victim_renewals = job.victim_renewals.saturating_add(1); + if deferred && job.packet.engine() == Engine::Compute { + job.victim_renewals += 1; } deferred } fn prepare(job: &mut Self) -> Option { - super::trace::schedule(&job.packet, 5, 0); - let wait = job.backend.prepare(&job.packet); - super::trace::schedule(&job.packet, 6, i32::from(wait.is_some())); - wait + job.backend.prepare(&job.packet) } fn run(job: &mut Self) -> Result> { - super::trace::schedule(&job.packet, 3, 0); - let result = job.backend.publish(job.packet.clone()).map(Some); - super::trace::schedule(&job.packet, 4, - result.as_ref().err().map_or(0, |error| error.to_errno())); - result + job.backend.publish(job.packet.clone()).map(Some) } fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep @@ -402,32 +340,13 @@ pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, pub(crate) publication: Option>, - pub(crate) owner: Arc, -} - -impl super::dependency::Retention for Fences { - fn retained(&self) -> bool { - self.owner.retains_vm() - } -} - -impl Fences { - /// Scheduler::enqueue always supplies our explicit completion fence here. - /// It has no polling callback, so pending frontier checks need no lock. - pub(crate) fn status(&self) -> i32 { - unsafe extern "C" { - fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; - } - // SAFETY: completed owns the live private fence throughout this read. - unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } - } } impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; @@ -445,7 +364,6 @@ impl Scheduler { early: bool, publication: Option, ) -> Result { - super::trace::schedule(&packet, 0, dependencies.len() as i32); let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); // Only commands which actually write timestamps become host-prefix @@ -478,15 +396,10 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - let inline = early && self.backend.try_early(&packet); - super::trace::schedule(&packet, if inline { 1 } else { 2 }, 0); - if !inline { - if packet.completion.needs_source_ready() { - packet.completion.set_source_ready(ready.clone()); - } + if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() }) + Ok(Fences { ready, completed, publication: host_publication }) } } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 6499cae106309..3b78b9443a86d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,15 +6,8 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::{ - mem::ManuallyDrop, - sync::atomic::{fence, Ordering}, -}; -use kernel::{ - new_mutex, - prelude::*, - sync::{Arc, Mutex}, -}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; use super::{ fw::{kick::*, queue::DataMaster}, @@ -24,40 +17,8 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Render identities that must stay reachable for `queues` logical queues: -/// one tiling/fragment pair each, plus the two bootstrap reservations. -/// -/// Installed identities never change engine before processor stop, so a -/// compute channel permanently narrows what render can still install. Compute -/// may take a free QID only while every open logical queue, and -/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated -/// pair; logical-queue admission refuses a queue whose pair no longer fits. -pub(super) const fn render_reserve(queues: usize) -> usize { - let queues = queues + RENDER_HEADROOM; - let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; - 2 * if queues < max { queues } else { max } + 2 -} -/// Logical queues that may still open after compute growth without losing a -/// dedicated render pair. -const RENDER_HEADROOM: usize = 24; -/// The render reserve with every execution context open. -const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); -/// Compute channels that stay reachable with every execution context open. -#[cfg(CONFIG_KUNIT = "y")] -pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; -/// Upper bound on distinct physical compute channels named by one kick. -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; -/// Maximum logical queues after firmware-owned QIDs are excluded. -pub(super) const fn queue_limit(floor: usize) -> u32 { - ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 -} - -const _: () = { - assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - // Compute keeps this many channels even with every execution context open. - assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); - assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); -}; +/// QIDs kept free for render pairs when compute queues fill the table. +const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -86,40 +47,19 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - initial_render: [Option; 2], + compute_publication: u64, leased: usize, - /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). - floor: usize, } impl QueueIds { - pub(crate) fn new(floor: usize) -> impl Init { + pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - initial_render: [None; 2], + compute_publication: 0, leased: 0, - // The reserve must keep render capacity above the floor. - floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } - /// Keep the initial render reservations until another engine needs their - /// capacity. No graph receives these tokens, so an unpublished reservation - /// can be replaced without changing an installed physical channel. - pub(crate) fn reserve_initial_render(&mut self) -> Result { - if self.leased != 0 { - return Err(EINVAL); - } - for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] - .into_iter() - .enumerate() - { - let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; - self.initial_render[qid] = Some(id); - } - Ok(()) - } - /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -127,58 +67,6 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, - ) -> Result { - self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) - } - - fn render_ids(&self) -> usize { - self.entries - .iter() - .flatten() - .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) - .count() - } - - /// Whether `queues` open logical queues can each still install a dedicated - /// render pair, beside the bootstrap reservations and installed compute. - pub(crate) fn render_fits(&self, queues: usize) -> bool { - self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 - } - - /// Whether a render pair not yet built can still reserve both of its identities. - pub(crate) fn render_pair_fits(&self) -> bool { - QID_COUNT - self.floor - self.leased >= 2 - } - - /// Whether a fresh compute identity fits beside the render reserve of - /// `queues` open logical queues, without reserving it. - pub(crate) fn compute_fits(&self, queues: usize) -> bool { - let initial = self.leased == QID_COUNT - self.floor - && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); - self.compute_room(queues, usize::from(initial)) - } - - fn compute_room(&self, queues: usize, replacement: usize) -> bool { - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let render = self.render_ids(); - QID_COUNT - self.floor - self.leased + replacement - > render_reserve(queues) - .min(FULL_RENDER_RESERVE - self.floor) - .saturating_sub(render - replacement) - } - - /// As [`Self::reserve`], with the compute reserve sized for `queues` open - /// logical queues instead of every possible execution context. - pub(crate) fn reserve_for( - &mut self, - owner: u64, - engine: DataMaster, - preferred: Option, - queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -195,36 +83,16 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never available to the host. - let total = QID_COUNT - self.floor; - let initial = if engine == DataMaster::Compute && self.leased == total { - self.initial_render - .iter() - .copied() - .enumerate() - .find_map(|(slot, id)| { - id.filter(|id| self.can_cancel_unpublished(*id)) - .map(|id| (slot, id)) - }) - } else { - None - }; - // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); + // Healthy existing channels keep their QID. Never recycle a published + // identity to recover capacity after a fault; leave eight render pairs. + if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { + return Err(ENOSPC); } let index = preferred - .map(|index| usize::from(index) + self.floor) + .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| { - self.entries - .iter() - .skip(self.floor) - .position(Option::is_none) - .map(|index| index + self.floor) - }) - .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) - .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?; let id = Id { qid: index as u8, owner, @@ -234,14 +102,16 @@ impl QueueIds { id, published: false, }); - if let Some((slot, _)) = initial { - self.initial_render[slot] = None; - } else { - self.leased += 1; - } + self.leased += 1; Ok(id) } + /// Ordered under the device mutex; no atomic operation on the submission path. + pub(crate) fn next_compute_publication(&mut self) -> Result { + self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.compute_publication) + } + /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -257,13 +127,6 @@ impl QueueIds { Ok(()) } - pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { - self.entries - .get(usize::from(id.qid)) - .and_then(Option::as_ref) - .is_some_and(|entry| entry.id == id && !entry.published) - } - pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -277,117 +140,6 @@ impl QueueIds { } } -#[kunit_tests(asahi_g17_qid_accounting)] -mod accounting_tests { - use super::*; - - fn table() -> KBox { - KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() - } - - #[test] - fn render_identities_satisfy_the_compute_reserve() { - let mut ids = table(); - for owner in 1..=FULL_RENDER_RESERVE as u64 { - ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - } - for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { - ids.reserve(owner, DataMaster::Compute, None).unwrap(); - } - assert_eq!(ids.leased, QID_COUNT); - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - assert_eq!(ids.leased, QID_COUNT); - } - - #[test] - fn compute_cannot_consume_the_render_minimum() { - let mut ids = table(); - for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { - ids.reserve(owner, DataMaster::Compute, None).unwrap(); - } - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); - } - - #[test] - fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { - for floor in [0, 4, 32] { - let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); - let queues = 2; - let mut owner = 1; - while ids.compute_fits(queues) { - let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); - assert!(usize::from(id.qid()) >= floor); - ids.publish(id).unwrap(); - owner += 1; - } - assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); - assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); - assert!(ids.render_fits(queues + RENDER_HEADROOM)); - assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); - } - } - - #[test] - fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { - for floor in [0, 4, 32] { - let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); - let queues = queue_limit(floor) as usize; - for owner in 1..=COMPUTE_FLOOR as u64 { - let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); - assert!(usize::from(id.qid()) >= floor); - ids.publish(id).unwrap(); - } - assert!(!ids.compute_fits(queues)); - assert!(ids.render_fits(queues)); - assert!(!ids.render_fits(queues + 1)); - } - } - - #[test] - fn initial_reservations_yield_but_published_ids_do_not() { - let mut ids = table(); - ids.reserve_initial_render().unwrap(); - for owner in 3..=QID_COUNT as u64 { - ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - } - let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); - let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); - assert_ne!(first.qid(), second.qid()); - assert!(ids.initial_render.iter().all(Option::is_none)); - assert_eq!(ids.leased, QID_COUNT); - ids.publish(first).unwrap(); - assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); - assert_eq!(ids.leased, QID_COUNT); - ids.cancel_unpublished(second).unwrap(); - assert_eq!(ids.leased, QID_COUNT - 1); - assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); - assert_eq!(ids.leased, QID_COUNT - 1); - } - - #[test] - fn failed_second_render_id_returns_only_the_unpublished_first() { - let mut ids = table(); - for owner in 1..QID_COUNT as u64 { - let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - ids.publish(id).unwrap(); - } - let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); - assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); - ids.cancel_unpublished(first).unwrap(); - assert_eq!(ids.leased, QID_COUNT - 1); - let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); - assert_eq!(replacement.qid(), first.qid()); - assert_ne!(replacement, first); - assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); - assert_eq!(ids.leased, QID_COUNT); - assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); - } -} - // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -395,94 +147,10 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render(RenderRing), -} - -/// A render ring's pages and the shared lower reservation clients alias. -struct RenderStorage { - // The lower alias drops before its backing. - low: mmu::KernelMapping, - high: KernelObject, -} - -/// Render ring storage outlives its rings for the life of the device. -/// -/// A client VM keeps each render ring alias it installs until the VM closes, -/// while an installed pair whose identities were never published can be -/// reclaimed and its rings dropped. Freeing that storage would hand its shared -/// lower address to another ring; a client still aliasing the old pages there -/// that later receives the new ring could not alias it (`prepare_client_alias` -/// fails with EFAULT). Retired storage is therefore kept and reused by later -/// rings: an address keeps naming the same pages, so every alias stays valid. -#[pin_data] -pub(crate) struct RenderRings { - #[pin] - stock: Mutex, -} - -struct RingStock { - /// Retired storage; `true` marks storage of a ring that never wrote an - /// entry, which a later ring may reuse. Written rings stay retired. - retired: KVec<(bool, RenderStorage)>, - /// Storage created so far; `retired` keeps capacity for all of it, so a - /// dropped ring always returns its storage without allocating. - created: usize, -} - -impl RenderRings { - pub(crate) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - stock <- new_mutex!( - RingStock { retired: KVec::new(), created: 0 }, - "G17 render kick rings" - ), - }), - GFP_KERNEL, - ) - } - - fn take(&self) -> Option { - let mut stock = self.stock.lock(); - let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; - let last = stock.retired.len() - 1; - stock.retired.swap(index, last); - stock.retired.pop().map(|(_, storage)| storage) - } - - /// Reserves the return of one more, just created storage. - fn admit(&self) -> Result { - let mut stock = self.stock.lock(); - let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; - let additional = created.saturating_sub(stock.retired.len()); - stock.retired.reserve(additional, GFP_KERNEL)?; - stock.created = created; - Ok(()) - } - - fn put(&self, storage: RenderStorage, reusable: bool) { - let mut stock = self.stock.lock(); - if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { - // `admit` reserved room for every created storage. Never free it. - core::mem::forget(rejected); - } - } -} - -/// Storage lent to one render ring; returned to the stock when the ring drops. -struct RenderRing { - storage: ManuallyDrop, - rings: Arc, - /// Set once an entry was written: such storage is never reused. - written: bool, -} - -impl Drop for RenderRing { - fn drop(&mut self) { - // SAFETY: `storage` is taken exactly once, here, and not used afterwards. - let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; - self.rings.put(storage, !self.written); - } + Render { + low: mmu::KernelMapping, + high: KernelObject, + }, } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -518,9 +186,6 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); - if let Backing::Render(ring) = &mut self.queue.backing { - ring.written = true; - } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -555,45 +220,14 @@ impl Queue { }) } - /// Retired storage of a never-written ring is reused first. Its address - /// names the same pages in every client that aliases it, so an existing - /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - rings: &Arc, - ) -> Result<(Self, Option)> { + ) -> Result<(Self, mmu::KernelMapping)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } - if let Some(storage) = rings.take() { - let mut ring = RenderRing { - storage: ManuallyDrop::new(storage), - rings: rings.clone(), - written: false, - }; - // A fresh ring is zeroed; no firmware ever read this one. - ring.storage.high.bytes_mut().fill(0); - fence(Ordering::SeqCst); - let address = ring.storage.low.iova(); - let alias = - ring.storage - .high - .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); - // On failure the storage returns to the stock and a fresh ring is built. - if let Ok(alias) = alias { - return Ok(( - Self { - backing: Backing::Render(ring), - id, - timestamp: KickTimestamp::FIRST, - previous: None, - }, - alias, - )); - } - } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -621,26 +255,14 @@ impl Queue { Err(error) => return Err(error), } }; - let storage = RenderStorage { low, high }; - // Count the storage only once it exists. Without room to return it, - // nothing has aliased it but `client`, which drops first. - if let Err(error) = rings.admit() { - drop(client); - drop(storage); - return Err(error); - } Ok(( Self { - backing: Backing::Render(RenderRing { - storage: ManuallyDrop::new(storage), - rings: rings.clone(), - written: false, - }), + backing: Backing::Render { low, high }, id, timestamp: KickTimestamp::FIRST, previous: None, }, - Some(client), + client, )) } @@ -650,19 +272,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render(ring) => ring.storage.low.iova(), + Backing::Render { low, .. } => low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render(ring) => ring.storage.high.gpu_va(), + Backing::Render { high, .. } => high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render(ring) => &mut ring.storage.high, + Backing::Render { high, .. } => high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -675,7 +297,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render(ring) => &ring.storage.high, + Backing::Render { high, .. } => high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -731,20 +353,3 @@ impl Queue { }) } } - -impl QueueIds { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let mut engines = [0usize; 3]; - let mut published = 0; - for entry in self.entries.iter().flatten() { - engines[match entry.id.engine { - DataMaster::Tiling => 0, - DataMaster::Fragment => 1, - DataMaster::Compute => 2, - }] += 1; - published += usize::from(entry.published); - } - writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", - self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) - } -} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 1dbfcc8fd3ce5..9da90d2b0843a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,55 +12,39 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all -//! firmware-shared state that has no narrower owner. Steady-state compute publication into a -//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under -//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: -//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; -//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` -//! nests inside nothing. Capacity waits block on an availability domain signalled by the -//! release that provides the capacity, never on a sleep. - -mod availability; +//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all +//! firmware-shared state, including the bounded waits for the firmware that publication needs. +//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state +//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. + mod buffer; -mod calibration; mod channel; mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; -mod data_segment; mod dependency; -mod sync_input; -pub(crate) mod sync_output; -mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; -mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -pub(crate) mod hazard; mod ids; +pub(crate) mod hazard; mod initdata; mod job; mod kick; mod object; -mod port; -pub(crate) mod power; +mod power; mod preparation; mod qos; mod queue; mod recovery; -mod replay; -mod reports; mod runtime; -pub(crate) mod stats; pub(crate) mod status; mod teardown; -pub(crate) mod telemetry; mod timeout; mod validation; @@ -68,8 +52,6 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, - AtomicU32, - AtomicU64, Ordering, // }; @@ -77,12 +59,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, - impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, + impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -167,64 +149,12 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for - /// a single-instance firmware. - pub(crate) secondary_root_offset: Option, - /// Where the coprocessors' crash log buffers live. - pub(crate) crash_buffers: CrashBuffers, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. + pub(crate) secondary_root_offset: u64, + /// Crash log buffer of each [`Role`]. + pub(crate) crash_buffers: [FixedBuffer; 2], /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, - /// Register windows the firmware maps through the hardware data. - pub(crate) windows: &'static [fw::initdata::RegisterWindow], - /// Register-map slots that are reserved but have no window. - pub(crate) reserved_slots: &'static [usize], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'static fw::initdata::PerfConfig, - /// Number of operating points the device tree lists. - pub(crate) opps: usize, - /// UAT policy where it differs from the A18 Pro's. - pub(crate) uat_quirks: mmu::UatQuirks, - /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the - /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as - /// the firmware left it, which is the only way the M4 Pro has run. - pub(crate) adopt_root_entries: bool, - /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware - /// starts (A18 Pro). The M4 Pro has only run with neither written. - pub(crate) fabric_bridge: bool, - /// The SoC's values of region C. - pub(crate) region_c: &'static fw::initdata::RegionCConfig, - /// Physical address of the firmware's per-slot recovery keys, read while the firmware is - /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the - /// firmware's own blame). - pub(crate) recovery_slot_keys: Option, - /// GPU generation reported to userspace. - pub(crate) gpu_generation: hw::GpuGen, - /// The firmware builds this SoC runs. - pub(crate) firmware: fw::variant::Firmware, - /// Layouts of the command buffers that follow the GPU's core layout. - pub(crate) core_layout: CoreLayout, -} - -/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. -pub(crate) struct CoreLayout { - /// The measured cluster and enabled core counts; `None` admits every configuration. - pub(crate) cores: Option<(u32, u32)>, - /// An experimental layout: core slots and enabled cores per cluster. - pub(crate) alternate_core_masks: Option<(u32, u32)>, - /// A compute queue's operand-state buffer. - pub(crate) operand_state: fw::compute::OperandState, - /// A render's deflake buffer. - pub(crate) deflake: fw::render::Deflake, -} - -/// Where a G17 target's crash log buffers live. -pub(crate) enum CrashBuffers { - /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel - /// lower root, mapped by the driver before the coprocessors start. - Fixed([FixedBuffer; 2]), - /// The firmware places its crash log in its own data segment, the `fw-data` reserved - /// region, and asks for it there by physical address. - FirmwareData, } /// A firmware coprocessor instance. @@ -267,9 +197,6 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; -/// One-millisecond polls for a firmware that acknowledges its initialization only in status A -/// ([`fw::variant::InitAck::Status`]). -const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -280,15 +207,7 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: CrashLogMap, -} - -/// The CPU mapping of a crash log buffer. -enum CrashLogMap { - /// The driver's buffer ([`CrashBuffers::Fixed`]). - Object(shmem::VMapOwned), - /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). - FirmwareData(data_segment::DataSegmentLog), + map: shmem::VMapOwned, } impl rtkit::Buffer for CrashLog { @@ -297,10 +216,7 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - match &mut self.map { - CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), - CrashLogMap::FirmwareData(log) => log.buf(), - } + Ok(IoSysMap::from((&self.map).as_view())) } } @@ -309,8 +225,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. - crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, + crash_log: FixedBuffer, + crash_map: shmem::VMapOwned, crashed: AtomicBool, #[pin] ack: Opaque, @@ -375,7 +291,7 @@ impl rtkit::Operations for RoleOps { if !is_events_message(ep, msg) { return false; } - data.shared.queue_event_records(); + data.shared.queue_events(); true } @@ -389,81 +305,26 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if let Some((buffer, map)) = data.crash_log.as_ref() { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { - return Err(EINVAL); - } - return Ok(CrashLog { - iova, - map: CrashLogMap::Object(map.owner().owned_vmap()?), - }); + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size + { + return Err(EINVAL); } - // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( - &data.shared.dev, - iova, - size, - )?), + map: data.crash_map.owner().owned_vmap()?, }) } } -/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this -/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding -/// different locks can ring the same doorbell. -#[pin_data] -struct Mailbox { - #[pin] - lock: Mutex<()>, - rtkit: core::cell::UnsafeCell>, -} - -// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. -unsafe impl Sync for Mailbox {} - -impl Mailbox { - fn new(rtkit: rtkit::RtKit) -> Result>> { - KBox::pin_init( - pin_init!(Self { - lock <- new_mutex!((), "g17::Mailbox::lock"), - rtkit: core::cell::UnsafeCell::new(rtkit), - }), - GFP_KERNEL, - ) - } - - fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { - let _guard = self.lock.lock(); - // SAFETY: The lock excludes every other borrow of the RTKit. - f(Pin::new(unsafe { &mut *self.rtkit.get() })) - } - - fn send(&self, endpoint: u8, message: u64) -> Result { - self.with(|rtkit| rtkit.send_message(endpoint, message)) - } - - /// A handle on the receive thread's minimum utilization, usable from any context. - /// - /// # Safety - /// - /// The caller must not use the handle after this mailbox is dropped. - unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { - // SAFETY: The caller's obligation is the handle's. - self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) - } -} - /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: Pin>, + rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: Option, + _crash_buffer: KernelObject, } impl Coprocessor { @@ -477,32 +338,27 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let (crash_buffer, crash_log) = match &cfg.crash_buffers { - CrashBuffers::Fixed(buffers) => { - let buffer = &buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - let crash_map = crash_buffer.kernel_map()?; - let fixed = FixedBuffer { - va: buffer.va, - size: buffer.size, - }; - (Some(crash_buffer), Some((fixed, crash_map))) - } - CrashBuffers::FirmwareData => (None, None), - }; + let buffer = &cfg.crash_buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + + let crash_map = crash_buffer.kernel_map()?; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log, + crash_log: FixedBuffer { + va: buffer.va, + size: buffer.size, + }, + crash_map, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -516,7 +372,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, + rtkit: Self::receiver(dev, index, &state)?, state, _crash_buffer: crash_buffer, }) @@ -545,30 +401,24 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - self.rtkit.with(|rtkit| rtkit.boot()) - } - - /// Starts the firmware endpoints and hands the firmware its initialization root. An - /// acknowledgment message is awaited here; one in status A is the caller's to poll. - fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - self.rtkit.with(|mut rtkit| { - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - ) - })?; - match ack { - fw::variant::InitAck::Message => self.state.wait_for_ack(), - fw::variant::InitAck::Status => Ok(()), - } + Pin::new(&mut self.rtkit).boot() + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. + fn init(&mut self, root_va: u64) -> Result { + let mut rtkit = Pin::new(&mut self.rtkit); + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + )?; + self.state.wait_for_ack() } /// Sends a host notification to the firmware. - fn notify(&self, msg: u64) -> Result { - trace::message(trace::MAILBOX, msg); - self.rtkit.send(EP_DOORBELL, msg) + fn notify(&mut self, msg: u64) -> Result { + Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -602,14 +452,6 @@ impl Bridge { Ok(bridge) } - /// Keeps the register block for a target that runs without the bridge. - fn unused(regs: regs::G17Resources) -> Bridge { - Bridge { - regs, - enabled: false, - } - } - fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -630,25 +472,15 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance - /// firmware (M4 Pro) carries both roles on the primary. - secondary: Option, + secondary: Coprocessor, primary: Coprocessor, bridge: Bridge, - /// Publication state shared with publishers that do not hold the device mutex. - port: Arc, - /// Fast-path submissions already accounted by the idle check. - seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, - hold: power::Hold, registration_power_release_pending: bool, - variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, - /// Render kick ring storage, kept and reused for the life of the device. - render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -658,11 +490,8 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - if let Some(secondary) = self.secondary.as_mut() { - secondary.stop(); - } + self.secondary.stop(); self.primary.stop(); - self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. @@ -671,15 +500,6 @@ impl Drop for Firmware { } impl Firmware { - /// The firmware instances this GPU runs. - fn roles(&self) -> &'static [Role] { - if self.secondary.is_some() { - &[Role::Primary, Role::Secondary] - } else { - &[Role::Primary] - } - } - /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -699,115 +519,43 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { - // An idle hold keeps the cores powered past registration; it releases them itself. - if self.hold.active() { - self.registration_power_release_pending = false; - return Ok(()); - } power::release(self) } /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { - self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, - port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; - trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); - port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. Submissions published - /// without the device mutex count as activity, and a clear that races one is abandoned. - fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { - return; - } - self.rings.observe_statistics_power( - self.bridge.regs.g17_core_power() & 0xf != 0, - self.port.statistics_epoch(), - ); - } - + /// unretired work; quarantined work retaining mappings is excluded. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let submissions = self.port.submissions(); - let submitted = submissions != self.seen_submissions; - self.seen_submissions = submissions; - if submitted { - self.effort.submit(); - } - let effort = match self.effort.idle(now_ns, in_flight || submitted) { - power::IdleAction::None => None, + match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => Ok(None), power::IdleAction::Clear => { - self.port.set_effort_full(false); - if self.port.submissions() != submissions { - self.port.set_effort_full(true); - self.effort.submit(); - return Ok(Some(power::IDLE_HOLD_NS)); - } self.apply_effort(0)?; - None - } - power::IdleAction::Recheck(delay) => Some(delay), - }; - Ok( - match (effort, self.idle_hold(now_ns, in_flight || submitted)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }, - ) - } - - /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A - /// refused control (recovery in progress) is retried on a later check. - fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { - const RETRY_NS: u64 = 1_000_000; - let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); - for _ in 0..2 { - match self - .hold - .step(now_ns, in_flight, self.effort.submissions(), hold_ms) - { - power::HoldAction::None => return None, - power::HoldAction::Recheck(delay) => return Some(delay), - power::HoldAction::Assert => { - if self.set_idle_control(false).is_err() { - return None; - } - self.hold.asserted(); - } - power::HoldAction::Release => { - if self.set_idle_control(true).is_err() { - return Some(RETRY_NS); - } - self.hold.released(); - return None; - } + Ok(None) } + power::IdleAction::Recheck(delay) => Ok(Some(delay)), } - None - } - - fn set_idle_control(&mut self, enabled: bool) -> Result { - power::Control::set_idle(self, enabled) } } @@ -862,39 +610,14 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, - variant: &'static fw::Variant, - /// Layout of compute queues' operand-state buffers for this GPU's cores. - operand_state: &'static fw::compute::OperandState, - /// Layout of renders' deflake buffers for this GPU's cores. - deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, #[pin] feed_work: Work, preparations: Arc, - /// Compute admission waiting for a physical channel: signalled when a channel becomes - /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. - channels: Arc, - /// Compute admission waiting for retirement-bound capacity (work states, ring and - /// in-flight room, drain witnesses, pool releases) or for publication to reopen. - progress: Arc, - /// Logical-queue creation waiting for a returned queue slot or execution context. - queues: Arc, - /// Render publication waiting for render capacity, a pair on loan, a work - /// state or firmware room: signalled when a pair, lease or work state - /// returns, an owner closes, a teardown parks, and on every event pass - /// while a render publisher waits (firmware consumption has no event). - renders: Arc, - /// Unbound compute publishers which can reuse a retired channel. - compute_demand: AtomicU32, #[pin] work: Work, - /// The next event pass runs every housekeeping step: set by every [`Self::queue_events`] - /// request, not by the firmware's event notification ([`Self::queue_event_records`]). - housekeeping: AtomicBool, - /// When an event pass last ran every housekeeping step, in ns. - housekeeping_ns: AtomicU64, #[pin] poll: DelayedWork, #[pin] @@ -915,8 +638,6 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, - /// The port of the session in `state`; null before boot and once detachment begins. - port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -1006,37 +727,8 @@ impl Shared { drop(retired); } - /// Wakes the waiters for a returned pair, binding or capacity change. - fn note_changed(&self) { - self.changed.notify_all(); - self.renders.notify(); - } - - /// The session's port, valid while `lease` is held: shutdown and device loss join every - /// lease before they detach the session. - fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { - let _ = lease; - // SAFETY: The pointer is published before the session becomes reachable and cleared - // only after every lease has been joined, so it names a live port for 'a. - unsafe { self.port.load(Ordering::Acquire).as_ref() } - } - - /// Called after joining every preparation lease, before the session is detached. - fn forget_port(&self) { - self.port.store(core::ptr::null_mut(), Ordering::Release); - } - - /// Queues the event worker for a pass that runs every housekeeping step. Callable from any - /// context. + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { - self.housekeeping.store(true, Ordering::Release); - self.queue_event_records(); - } - - /// Queues the event worker for firmware records alone: completions, or records that arrived - /// during a pass. The pass skips the housekeeping that nothing needs (see - /// `runtime_event_turn`). Callable from any context. - fn queue_event_records(self: &Arc) { // Already queued if this fails, which is just as good. let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); } @@ -1050,21 +742,12 @@ impl Shared { } fn drain_reclaims(self: &Arc) { - let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); - reclaimed = true; - } - if reclaimed { - // Context release returns its execution root off-lock. The next - // device pass can now detach idle compute aliases and their QoS - // leases, without depending on a new client's submission/event. - self.queues.notify(); - self.queue_events(); } } @@ -1143,10 +826,6 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, - stats: Arc, - _capacity: Pin>>, - _reports: Pin>>, - _power: Pin>>, } impl Gpu { @@ -1157,22 +836,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { - let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { - Some(calibration::Calibration::read(dev).inspect_err(|_| { - dev_err!( - dev.as_ref(), - "Missing or invalid T6040 GPU calibration handoff\n" - ); - })?) - } else { - None - }; - let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; - if let Some(calibration) = calibration { - calibration.check_cores(&id.core_masks)?; - calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; - } if id.family != cfg.identity.family || id.variant != cfg.identity.variant || id.num_dies != cfg.identity.num_dies @@ -1192,74 +856,22 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let idle_delay = - power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; - power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); - dev_info!( - dev.as_ref(), - "GPU idle power-off delay: {} ms\n", - idle_delay - ); - let channels = availability::Availability::new()?; - let progress = availability::Availability::new()?; - let queues = availability::Availability::new()?; - let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; - let renders = availability::Availability::new()?; + let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); - let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { - clusters == id.num_clusters - && cores != active_cores - && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, enabled)| { - id.num_cores == slots - && id.core_masks.len() == clusters as usize - && id.core_masks.iter().all(|mask| mask.count_ones() == enabled) - }) - }); - if cfg - .core_layout - .cores - .is_some_and(|cores| cores != (id.num_clusters, active_cores)) - && !alternate - { - dev_err!( - dev.as_ref(), - "Unsupported GPU core layout: {} clusters with {} cores\n", - id.num_clusters, - active_cores - ); - return Err(ENODEV); - } - if alternate { - dev_warn!( - dev.as_ref(), - "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" - ); - } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, - variant, - operand_state: &cfg.core_layout.operand_state, - deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, - channels, - progress, - queues, - renders, - compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), - housekeeping: AtomicBool::new(true), - housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), grow <- new_work!("g17::Shared::grow"), @@ -1268,31 +880,10 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), - port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"aurora-g17"); - let capacity = KBox::pin_init( - directory.read_only_file(c"capacity", capacity::View(shared.clone())), - GFP_KERNEL, - )?; - let reports = KBox::pin_init( - directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), - GFP_KERNEL, - )?; - let power = KBox::pin_init( - directory.read_only_file(c"power", reports::PowerView(shared.clone())), - GFP_KERNEL, - )?; - let stats = Arc::new( - crate::stats::StatsSnapshot { - g17_reader: Some(stats::Reader(shared.clone())), - ..Default::default() - }, - GFP_KERNEL, - )?; let gpu = Arc::new( Gpu { cfg, @@ -1301,80 +892,15 @@ impl Gpu { geometry, ids: Default::default(), shared, - stats, - _capacity: capacity, - _reports: reports, - _power: power, }, GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; - gpu.shared - .port - .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) } - /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it - /// does not know, before anything is written to the GPU. - fn firmware_variant( - dev: &AsahiDrmDevice, - firmware: &fw::variant::Firmware, - ) -> Result<&'static fw::Variant> { - let builds = match firmware { - fw::variant::Firmware::Fixed(variant) => return Ok(variant), - fw::variant::Firmware::Identified(builds) => builds, - }; - let version = Self::os_firmware_version(dev); - // iBoot strings can be shared by OS releases. Prefer the reported release's - // candidate, but admit only an exact loaded image, regardless of that hint. - let preferred = builds - .iter() - .filter(|build| Some(build.os_version) == version); - let others = builds - .iter() - .filter(|build| Some(build.os_version) != version); - for build in Iterator::chain(preferred, others) { - if Self::identify_image(dev, build).is_ok() { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok(build.variant); - } - } - dev_err!( - dev.as_ref(), - "Unsupported GPU firmware image (reported macOS version {:?})\n", - version - ); - Err(ENODEV) - } - - /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size - /// of the `fw-data` region. The version property is only a selection hint. - fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { - let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; - let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; - let start = text.start(); - let end = start.checked_add(text.size()).ok_or(EINVAL)?; - crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) - .inspect_err(|e| { - dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) - })?; - Ok(()) - } - - /// The macOS version the boot loader reports for the boot firmware that loaded the GPU - /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. - fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { - let node = dev.as_ref().as_ref().of_node()?; - let cells = node - .get_property::>(c_str!("apple,firmware-version")) - .ok()?; - cells.as_slice().try_into().ok() - } - /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -1406,7 +932,6 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, - cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -1421,30 +946,13 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) - })?; - let secondary = match cfg.secondary_root_offset { - Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( - |e| { - dev_err!( - dev.as_ref(), - "Secondary coprocessor setup failed: {:?}\n", - e - ) - }, - )?, - ), - None => None, - }; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = match cfg.fabric_bridge { - true => Bridge::enable(regs)?, - false => Bridge::unused(regs), - }; + let bridge = Bridge::enable(regs)?; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -1453,27 +961,12 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new( - power::Demand::new(pdev.as_ref().as_raw())?, - shared.feed.clone(), - )?, - seen_submissions: 0, - rings: channel::Rings::new(shared.variant.graph)?, + rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), - hold: power::Hold::new(), registration_power_release_pending: false, - variant: shared.variant, - queues <- runtime::Registry::with_waiters( - shared.clusters, - shared.deflake, - shared.descriptor_flags, - shared.variant, - Some(shared.queues.clone()), - Some(shared.renders.clone()), - ), + queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, - render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -1484,107 +977,45 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - if let Some(secondary) = session.secondary.as_mut() { - secondary.boot()?; - } - // The firmware built its root and kernel-window tables while booting. - session.uat.invalidate_reserved_tables(); - let root_entries = match cfg.adopt_root_entries { - true => Some(session.uat.adopt_firmware_root_entries()?), - false => None, - }; + session.secondary.boot()?; + let root_entries = session.uat.adopt_firmware_root_entries()?; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, - shared.clusters, - shared.active_cores, - shared.variant, - )?; - if let Some(root_entries) = root_entries { - session.uat.confirm_firmware_root_entries(root_entries)?; - } - let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach( - session.init.port_views(mailbox)?, - session.bridge.regs.clone(), )?; + session.uat.confirm_firmware_root_entries(root_entries)?; session.uat.mirror_secondary_root()?; - if cfg.fabric_bridge { - session.bridge.regs.enable_g17_gating()?; - } - let ack = shared.variant.init_ack; + session.bridge.regs.enable_g17_gating()?; + session.primary.init(session.init.root_va(Role::Primary)?)?; session - .primary - .init(session.init.root_va(Role::Primary)?, ack)?; - if ack == fw::variant::InitAck::Status { - Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; - } - if let Some(secondary) = session.secondary.as_mut() { - secondary.init(session.init.root_va(Role::Secondary)?, ack)?; - } - if shared.variant.power_on_done { - session.primary.notify(MSG_POWER_ON_DONE)?; - if let Some(secondary) = session.secondary.as_mut() { - secondary.notify(MSG_POWER_ON_DONE)?; - } - } - Self::wait_for_opening( - dev, - &session.init, - &session.primary, - session.secondary.as_ref(), - )?; + .secondary + .init(session.init.root_va(Role::Secondary)?)?; + session.primary.notify(MSG_POWER_ON_DONE)?; + session.secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } - /// Waits for a firmware that sends no acknowledgment message to accept its - /// initialization in status A. - fn wait_for_init_accepted( - dev: &AsahiDrmDevice, - init: &initdata::InitData, - primary: &Coprocessor, - ) -> Result { - for poll in 0..=INIT_ACCEPT_POLLS { - if primary.state.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - if init.init_accepted()? { - return Ok(()); - } - if poll < INIT_ACCEPT_POLLS { - fsleep(Delta::from_millis(1)); - } - } - dev_err!( - dev.as_ref(), - "Firmware did not accept its initialization: status A {:#x?}\n", - init.status_read_words::<4>(Role::Primary, 0x10)? - ); - Err(ETIMEDOUT) - } - /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: Option<&Coprocessor>, + secondary: &Coprocessor, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) + || secondary.state.crashed.load(Ordering::Acquire) { return Err(EIO); } - if init.opening_retired(Role::Primary)? - && (secondary.is_none() || init.opening_retired(Role::Secondary)?) - { + if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1629,7 +1060,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - Some(self.stats.clone()) + None } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1650,20 +1081,12 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) - | sync_output::FEATURE_CUMULATIVE_SIGNALS; - let variant = self.shared.variant; - if variant.few_primitives { - params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); - } - if variant.ray_traversal { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); - } - params.gpu_generation = self.cfg.gpu_generation as u32; + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; + params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1692,18 +1115,9 @@ impl gpu::Gpu for Gpu { true } - /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not - /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 - /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads - /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case - /// failed; all pass with soft faults off). - fn supports_soft_faults(&self) -> bool { - false - } - fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: kick::queue_limit(self.shared.variant.qid_floor), + max_queues: mmu::MAX_EXECUTION_CONTEXTS, max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1739,39 +1153,15 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = queue::Frontend::new( - priority, - usc_exec_base, - self.geometry.user_usable_range(), - self.shared.variant, - )?; + let frontend = + queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; let policy = scheduling_policy(priority)?; - let admission_started = now_ns(); let (backend, slot) = loop { - let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, - Err(error) if error == ENOSPC => { - // Closed logical queues can still own execution/QoS IDs - // until the scheduler release is consumed and reclaimed. - // Backend construction has rolled back all temporary leases. - let waited = now_ns().saturating_sub(admission_started); - let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; - if waited >= limit { - return Err(EBUSY); - } - self.shared.queue_events(); - // A returned queue slot or reclaimed context wakes this. - self.shared.queues.block( - observed, - (limit - waited).div_ceil(1_000_000) as u32, - true, - )?; - } result => break result?, } }; @@ -1781,7 +1171,6 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, - usage, ) { Ok(frontend) => frontend, Err(error) => { @@ -1796,32 +1185,27 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - let mapping = { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )? - }; - // The buffer's CPU mapping takes the reservation lock: not under the device mutex. - crate::timestamp::Buffer::firmware(mapping) + ) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH -/// arrives as 1, MEDIUM as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so +/// MEDIUM arrives as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { - 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 87fdb0e8a62ad..46f4e0c56b6ae 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,17 +78,7 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => { - let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start - && range.end <= crate::hw::t8140::dynamic::LOWER.end { - super::capacity::Site::LowerVa - } else { - super::capacity::Site::UpperVa - }; - this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { - if error == ENOSPC { super::capacity::refused(site) } else { error } - })? - } + Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, }; this.mapping = Some(mapping); Ok(this) @@ -123,9 +113,6 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) - .map_err(|error| if error == ENOSPC { - super::capacity::refused(super::capacity::Site::AliasVa) - } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -172,20 +159,6 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) - .map_err(|error| if error == ENOSPC { - super::capacity::refused(super::capacity::Site::AliasVa) - } else { error }) - } - - fn alias_in_lowest( - &self, - vm: &mmu::Vm, - range: Range, - align: u64, - prot: mmu::Prot, - guard_size: usize, - ) -> Result { - vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -368,76 +341,6 @@ impl Allocator<'_> { cpu, ) } - - /// Place fresh backing at one address in the canonical and both alias VMs. - /// A client may retain mappings after their canonical reservation is freed. - /// Retry only address collisions, keeping the same backing throughout. - pub(crate) fn lower_with_aliases( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - aliases: [&mmu::Vm; 2], - ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { - if size == 0 || size % mmu::UAT_PGSZ != 0 { - return Err(EINVAL); - } - let mut object = KernelObject::backing(self.dev, size, cpu)?; - let mut range = crate::hw::t8140::dynamic::LOWER; - let mut searching = false; - loop { - // Keep ordinary best-fit placement on the uncontended path. A - // collision starts a lowest-address search over the whole arena. - let mapping = (if searching { - object.alias_in_lowest( - self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, - ) - } else { - object.obj.map_into_range( - self.uat.kernel_lower_vm(), range.clone(), align, prot, true, - ) - }).map_err(|error| if error == ENOSPC { - crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) - } else { error })?; - let address = mapping.iova(); - let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; - // Partial aliases are destroyed before releasing the canonical - // reservation, so a retry cannot expose its address prematurely. - let mapped: core::result::Result<_, (Error, usize)> = (|| { - let first = object.map_alias(aliases[0], address, prot) - .map_err(|error| (error, 0))?; - let second = object.map_alias(aliases[1], address, prot) - .map_err(|error| (error, 1))?; - Ok([first, second]) - })(); - match mapped { - Ok(mapped) => { - object.mapping = Some(mapping); - return Ok((object, mapped)); - } - Err((error, index)) if error == ENOSPC && end < range.end => { - drop(mapping); - let start = if searching { - address + mmu::UAT_PGSZ as u64 - } else { - range.start - }; - // Jump to an actual hole in the VM that refused the alias, - // rather than remapping backing at every intervening page. - let candidate = object.alias_in_lowest( - aliases[index], start..range.end, align, prot, 0, - ).map_err(|error| if error == ENOSPC { - crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) - } else { error })?; - range.start = candidate.iova(); - drop(candidate); - searching = true; - } - Err((error, _)) => return Err(error), - } - } - } } /// Families whose upper mappings remain cached and installed until firmware @@ -466,17 +369,8 @@ struct PoolInner { reserved: [usize; POOL_FAMILIES], bytes: usize, stopped: bool, - /// Objects the firmware may still reference after their owner dropped them, kept mapped - /// and out of reuse until firmware stop (see [`PooledObject::retain_until_stop`]). - kept: KVec, - kept_bytes: usize, - /// Kept objects still owned by their users, whose list entries are reserved. - kept_reserved: usize, } -/// Retained objects past this many are still retained, with one warning. -const KEPT_WARN: usize = 256; - /// Per-device cache-safe object recycling. Every converted mapping reserves /// one return slot before its page-table attributes change. Active leases and /// poisoned mappings count against the same capacity as reusable objects. @@ -499,9 +393,6 @@ struct CacheLease { pub(crate) struct PooledObject { object: core::mem::ManuallyDrop, lease: Option, - /// The pool keeping this object until firmware stop, and whether `reserve_kept` took its - /// list entry. - keep: Option<(kernel::sync::Arc, bool)>, } impl core::ops::Deref for PooledObject { @@ -515,28 +406,11 @@ impl core::ops::DerefMut for PooledObject { &mut self.object } } -impl PooledObject { - /// The firmware may still reference this object after its owner drops it: keep it mapped - /// and never hand it out again before firmware stop. It leaves the reuse cache, so it - /// does not take cache capacity. Call from sleepable context: this reserves the list - /// entry, so the drop, wherever it happens, does not allocate. - pub(crate) fn retain_until_stop(&mut self, pool: &kernel::sync::Arc) { - if self.keep.is_none() { - let reserved = pool.reserve_kept(); - self.keep = Some((pool.clone(), reserved)); - } - } -} - impl Drop for PooledObject { fn drop(&mut self) { // SAFETY: This destructor is the only operation that moves `object` // out; ManuallyDrop prevents a second implicit drop of the field. let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; - if let Some((pool, reserved)) = self.keep.take() { - pool.keep_object(object, self.lease.take(), reserved); - return; - } match self.lease.take() { Some(lease) => lease.pool.return_object(object, &lease), None => drop(object), @@ -552,10 +426,7 @@ impl Pool { } kernel::sync::Arc::pin_init( pin_init!(Self { - inner <- kernel::new_mutex!(PoolInner { - returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), - kept_bytes: 0, kept_reserved: 0, - }, "g17::object::Pool"), + inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), }), GFP_KERNEL, ) @@ -604,7 +475,6 @@ impl Pool { return Ok(PooledObject { object: core::mem::ManuallyDrop::new(entry.object), lease: Some(lease()), - keep: None, }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; @@ -629,7 +499,6 @@ impl Pool { let mut result = PooledObject { object: core::mem::ManuallyDrop::new(object), lease: reserved.then(lease), - keep: None, }; if !reserved { return Ok(result); @@ -689,67 +558,10 @@ impl Pool { core::mem::forget(object); } - /// Room for one more kept object, taken by the matching `keep_object`. Returns whether - /// it was reserved; after a failed allocation that drop leaks the object instead. - fn reserve_kept(&self) -> bool { - let mut inner = self.inner.lock(); - let pending = inner.kept_reserved + 1; - if inner.kept.capacity() - inner.kept.len() < pending - && inner.kept.reserve(pending, GFP_KERNEL).is_err() - { - return false; - } - inner.kept_reserved = pending; - true - } - - fn keep_object(&self, object: KernelObject, lease: Option, reserved: bool) { - let mut inner = self.inner.lock(); - if reserved { - inner.kept_reserved = inner.kept_reserved.saturating_sub(1); - } - if let Some(lease) = &lease { - // The object leaves the reuse cache and its capacity. - inner.reserved[lease.family as usize] -= 1; - inner.bytes -= object.size(); - } - if inner.stopped { - drop(inner); - drop(object); - return; - } - let size = object.size(); - if inner.kept.len() == KEPT_WARN { - pr_warn!("G17 keeps {} firmware-linked objects until the GPU firmware restarts\n", KEPT_WARN); - } - if !reserved { - // Another kept object's reservation must not be used: leak this one instead. - pr_err!("G17 could not record a firmware-linked object; leaking it\n"); - core::mem::forget(object); - return; - } - // retain_until_stop reserved the entry; this never allocates. - match inner.kept.push_within_capacity(object) { - Ok(()) => inner.kept_bytes += size, - Err(error) => { - // Never unmap an object the firmware may still reference. - pr_err!("G17 could not record a firmware-linked object; leaking it\n"); - core::mem::forget(error.0); - } - } - } - /// Releases retained mappings after both firmware processors have stopped. /// Active objects returned later observe the same stop witness. pub(crate) fn stop_after_firmware(&self) { - let kept = { - let mut inner = self.inner.lock(); - inner.stopped = true; - inner.kept_bytes = 0; - core::mem::take(&mut inner.kept) - }; - // Mapping teardown must not run while the pool mutex is held. - drop(kept); + self.inner.lock().stopped = true; for index in 0..POOL_FAMILIES * POOL_SLOTS { let entry = { let mut inner = self.inner.lock(); @@ -765,17 +577,3 @@ impl Pool { } } } - -impl Pool { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let inner = self.inner.lock(); - for family in 0..POOL_FAMILIES { - let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; - writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, - inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), - entries.iter().flatten().filter(|entry| entry.poisoned).count())?; - } - writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; - writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) - } -} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510..96b363bc27447 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,30 +3,8 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. -use core::sync::atomic::AtomicU32; - pub(super) const FULL_EFFORT: u32 = 0x10000; -pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; - -/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after -/// the last job. Not measured on other firmware; at or below it the hold stays off. -pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; - -/// Idle policy initialized from the existing DT `apple,idle-off-delay-ms` property. -/// The existing `idle_off_delay_ms` sysfs attribute may override it. Above -/// [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU cores powered until it has been idle -/// this long ([`Hold`]). -pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); - -/// Use the same bounds as the existing idle-delay attribute. Missing DT policy -/// preserves the native delay; malformed or out-of-range values fail probe. -pub(super) fn idle_delay(requested: Option) -> kernel::error::Result { - let delay = requested.unwrap_or(FIRMWARE_IDLE_OFF_MS); - if !(FIRMWARE_IDLE_OFF_MS..=1000).contains(&delay) { - return Err(kernel::error::code::EINVAL); - } - Ok(delay) -} +const IDLE_HOLD_NS: u64 = 16_000_000; #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { @@ -39,7 +17,6 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, - submissions: u64, } impl Effort { @@ -48,19 +25,12 @@ impl Effort { streaming: false, idle_since: None, applied: 0, - submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; - self.submissions = self.submissions.wrapping_add(1); - } - - /// Count of published submissions, for [`Hold::step`]. - pub(super) fn submissions(&self) -> u64 { - self.submissions } pub(super) fn applied(&self) -> u32 { @@ -92,83 +62,6 @@ impl Effort { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum HoldAction { - None, - /// Send the keep-powered idle policy. - Assert, - /// Send the power-off idle policy again. - Release, - Recheck(u64), -} - -/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job -/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer -/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off -/// policy once no work has been submitted or in flight for that long. Checked from the event -/// worker, which runs on every completion, well inside the firmware's 2 ms. -pub(super) struct Hold { - active: bool, - seen: u64, - since: Option, -} - -impl Hold { - pub(super) const fn new() -> Self { - Self { - active: false, - seen: 0, - since: None, - } - } - - pub(super) fn active(&self) -> bool { - self.active - } - - pub(super) fn step( - &mut self, - now_ns: u64, - in_flight: bool, - submissions: u64, - hold_ms: u32, - ) -> HoldAction { - let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; - let fresh = submissions != self.seen; - self.seen = submissions; - if in_flight || fresh { - self.since = None; - } - if !self.active { - return if enabled && (in_flight || fresh) { - HoldAction::Assert - } else { - HoldAction::None - }; - } - if in_flight { - return HoldAction::None; - } - let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; - let since = *self.since.get_or_insert(now_ns.max(1)); - let idle = now_ns.saturating_sub(since); - if idle >= hold_ns { - HoldAction::Release - } else { - HoldAction::Recheck(hold_ns - idle) - } - } - - pub(super) fn asserted(&mut self) { - self.active = true; - } - - pub(super) fn released(&mut self) { - self.active = false; - self.since = None; - } -} - /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -244,99 +137,3 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] -mod tests { - use super::*; - - #[test] - fn dt_idle_delay_is_bounded_and_absence_preserves_native_policy() { - assert_eq!(idle_delay(None), Ok(FIRMWARE_IDLE_OFF_MS)); - for delay in [2, 16, 1000] { - assert_eq!(idle_delay(Some(delay)), Ok(delay)); - } - for delay in [0, 1, 1001, u32::MAX] { - assert!(idle_delay(Some(delay)).is_err()); - } - } - - #[test] - fn frame_hold_survives_depth_one_gaps_but_releases_at_idle() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, 16), HoldAction::Assert); - hold.asserted(); - for submission in 1..=8 { - // A completed depth-one dispatch followed by another short gap. - let now = submission * 3_000_000; - assert_eq!(hold.step(now, false, submission, 16), HoldAction::Recheck(16_000_000)); - assert!(hold.active()); - } - assert_eq!(hold.step(39_999_999, false, 8, 16), HoldAction::Recheck(1)); - assert_eq!(hold.step(40_000_000, false, 8, 16), HoldAction::Release); - hold.released(); - assert!(!hold.active()); - } - - #[test] - fn holds_until_idle_for_the_delay_then_releases() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); - hold.asserted(); - assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); - assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); - // A new submission restarts the idle period. - assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); - assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); - hold.released(); - assert!(!hold.active()); - } - - #[test] - fn the_default_delay_never_holds_and_lowering_it_releases() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); - assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); - hold.asserted(); - assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); - } -} - -unsafe extern "C" { - fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; - fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); - fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); -} - -/// DT-selected memory-demand provider. Its independent allocation outlives a -/// detached provider; a device link orders supplier removal and suspend. -pub(super) struct Demand(*mut core::ffi::c_void); - -// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. -unsafe impl Send for Demand {} -// SAFETY: shared access only changes that locked request, never its lifetime. -unsafe impl Sync for Demand {} - -impl Demand { - pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { - let mut out = core::ptr::null_mut(); - // SAFETY: live platform device and writable result; C returns an owned reference. - let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; - if ret < 0 { - Err(kernel::error::Error::from_errno(ret)) - } else { - Ok(Self(out)) - } - } - - pub(super) fn set_active(&self, active: bool) { - // SAFETY: the retained reference is live; NULL denotes no DT provider. - unsafe { asahi_clpc_gpu_set_active(self.0, active) }; - } -} - -impl Drop for Demand { - fn drop(&mut self) { - // SAFETY: releases exactly the reference returned by get, clearing demand first. - unsafe { asahi_clpc_gpu_put(self.0) }; - } -} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index c7d1761699a24..8563beda979e6 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -6,7 +6,6 @@ //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is //! stored after its tentative mappings so those mappings are destroyed before the join completes. -use core::cell::Cell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::{ new_condvar, new_mutex, @@ -21,46 +20,6 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; -/// One ready packet's monotonic compute-admission budget. The callback checks -/// packet cancellation, logical close, VM status and device loss even when no -/// capacity/recovery event arrives. Wakes and nested constructors never renew it. -pub(super) struct Admission<'a> { - deadline: Cell, - started: u64, - check: &'a dyn Fn() -> Result, -} - -impl<'a> Admission<'a> { - pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { - Self { deadline: Cell::new(deadline), started: super::now_ns(), check } - } - - /// A recovery is pending or this owner's spared work drains: wait as long as the - /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing. - pub(super) fn extend_for_recovery(&self) { - let limit = self - .started - .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000); - if self.deadline.get() < limit { - self.deadline.set(limit); - } - } - - pub(super) fn check(&self) -> Result { - (self.check)()?; - if super::now_ns() >= self.deadline.get() { return Err(EBUSY); } - Ok(()) - } - - /// Ignore condvar remaining-time reports: only the monotonic deadline counts. - pub(super) fn slice_ms(&self) -> Result { - self.check()?; - let remaining = self.deadline.get().saturating_sub(super::now_ns()); - if remaining == 0 { return Err(EBUSY); } - Ok(remaining.div_ceil(1_000_000).min(10) as u32) - } -} - #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -68,37 +27,20 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, - /// Scheduler admission waiting for publication to reopen. - available: Option>, } impl Gate { - #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new() -> Result> { - Self::with_waiters(None) - } - - /// A gate whose reopening, failure and removal also wake `available`. - pub(super) fn with_waiters( - available: Option>, - ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), - available, }), GFP_KERNEL, ) } - fn wake_waiters(&self) { - if let Some(available) = &self.available { - available.notify(); - } - } - pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -150,28 +92,6 @@ impl Gate { } } - /// Only a recovery closes the gate: packet admission waits for it up to - /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery). - /// Called with no device mutex or existing preparation lease. - pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { - admission.check()?; - match self.try_enter() { - Err(error) if error == EAGAIN => admission.extend_for_recovery(), - result => return result, - } - let mut wait = self.wait.lock(); - loop { - admission.check()?; - match self.try_enter() { - Err(error) if error == EAGAIN => admission.extend_for_recovery(), - result => return result, - } - let _ = self.changed.wait_timeout( - &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); - } - } - fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -193,66 +113,20 @@ impl Gate { } } - /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); - let _wait = self.wait.lock(); - self.changed.notify_all(); - } - - /// Opaque snapshot for wait_closure(), taken where recovery was observed. - pub(super) fn snapshot(&self) -> u64 { - self.state.load(Ordering::Acquire) - } - - /// A publisher that observed recovery (host pending, or the firmware's state - /// ahead of the event worker, which always follows with a restart and the - /// closure) sleeps until this gate has closed since `seen`. enter() then - /// blocks until recovery reopens it. The deadline is only a failure bound. - pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { - let mut wait = self.wait.lock(); - loop { - let state = self.state.load(Ordering::Acquire); - if state & (REMOVED | FAILED) != 0 { - return Err(ENODEV); - } - if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { - return Ok(()); - } - let remaining = deadline.saturating_sub(super::now_ns()); - if remaining == 0 { - return Err(ETIMEDOUT); - } - let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); - let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); - } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let wait = self.wait.lock(); + let _wait = self.wait.lock(); self.changed.notify_all(); - drop(wait); - self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let wait = self.wait.lock(); + let _wait = self.wait.lock(); self.changed.notify_all(); - drop(wait); - self.wake_waiters(); - } - - /// Runtime failure/recovery must not join an indefinitely stalled builder. - /// Timeout retains every graph and reservation; it never authorizes freeing. - pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { - let mut wait = self.wait.lock(); - while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { - if super::now_ns() >= deadline { return Err(ETIMEDOUT); } - let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); - } - Ok(()) } /// Join builders with no device, queue or mapping lock held. @@ -265,7 +139,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let wait = self.wait.lock(); + let _wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -279,8 +153,6 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); - drop(wait); - self.wake_waiters(); return true; } Err(current) => previous = current, @@ -312,118 +184,3 @@ impl Drop for Lease { } } } - -#[kunit_tests(asahi_g17_growth_gate)] -mod growth_gate_tests { - use super::*; - #[test] - fn recovery_can_join_between_growth_admissions() { - let gate = Gate::new().unwrap(); - let outer = gate.try_enter().unwrap(); - // The growth handoff must drop this lease before enter can wait. - drop(outer); - gate.close(); - assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); - gate.wait_drained(); - assert!(gate.reopen()); - let growth = gate.enter(false).unwrap(); - assert!(growth.is_current()); - drop(growth); - } -} - -#[kunit_tests(asahi_g17_compute_deadline)] -mod compute_deadline_tests { - use super::*; - - #[test] - fn a_recovery_extends_the_deadline_once_from_the_start() { - let check = || Ok(()); - let admission = Admission::new(super::super::now_ns(), &check); - assert_eq!(admission.check(), Err(EBUSY)); - admission.extend_for_recovery(); - assert_eq!(admission.check(), Ok(())); - let limit = admission.started - + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000; - assert_eq!(admission.deadline.get(), limit); - admission.extend_for_recovery(); - assert_eq!(admission.deadline.get(), limit); - } - - #[test] - fn spent_budget_refuses_even_an_open_gate() { - let gate = Gate::new().unwrap(); - let check = || Ok(()); - let admission = Admission::new(super::super::now_ns(), &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn closed_gate_waits_past_the_budget_for_recovery() { - let gate = Gate::new().unwrap(); - gate.close(); - let start = super::super::now_ns(); - // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting. - let check = || { - if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) } - }; - let admission = Admission::new(start + 20_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); - assert!(super::super::now_ns() - start < 500_000_000); - gate.wait_drained(); - assert!(gate.reopen()); - } - - #[test] - fn cancellation_without_capacity_or_recovery_events() { - let gate = Gate::new().unwrap(); - gate.close(); - let start = super::super::now_ns(); - let check = || { - if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } - }; - let admission = Admission::new(start + 2_000_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); - assert!(super::super::now_ns() - start < 500_000_000); - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn recovery_and_binding_retry_churn_cannot_renew_budget() { - let gate = Gate::new().unwrap(); - let deadline = super::super::now_ns() + 20_000_000; - let check = || Ok(()); - let admission = Admission::new(deadline, &check); - loop { - // Re-entry models a channel taken between admission/publication. - // Recovery joins each dropped lease, then wakes the same budget. - gate.close(); - gate.changed.notify_all(); - gate.wait_drained(); - assert!(gate.reopen()); - // However often the gate reopened, an attempt that starts at the deadline - // is refused. Timing the whole loop would also count time descheduled. - let attempted = super::super::now_ns(); - match gate.enter_until(&admission) { - Ok(lease) => { - assert!(attempted < deadline); - drop(lease); - } - Err(error) => { assert_eq!(error, EBUSY); break; } - } - } - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn packet_status_and_cancel_precede_available_capacity() { - let gate = Gate::new().unwrap(); - for error in [ECANCELED, EIO] { - let check = || Err(error); - let admission = Admission::new(u64::MAX, &check); - assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); - } - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index f09e629aa047e..1c0d6822c395f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,7 +9,6 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ - fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -44,46 +43,22 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, - /// Fragment kick of the published render; a later same-VM overlap of - /// another pair waits for it in firmware. - fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; -/// A wake never renews admission's budget. Short observations also notice a -/// packet cancellation that has no corresponding pair-release notification. -fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { - let remaining = deadline_ns.saturating_sub(now_ns); - if remaining == 0 { - return Err(EBUSY); - } - Ok(remaining.div_ceil(1_000_000).min(10) as u32) -} - struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, - /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail - /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. - spared: u64, kill_hold: u64, - /// Slots whose kill hold ended at their killed context's consumed scheduler release, - /// and when. A new owner takes such a pair last, the one released longest ago first. - killed: u64, - killed_at: [u64; cfg::RENDER_SLOTS], - /// The physical queue table still has room for a pair built in an unowned slot. The - /// runtime updates this whenever the table changes. - build_room: bool, released: u64, epoch: u64, footprints: KVec, next_ticket: u64, } - impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; const fn new() -> Self { @@ -93,11 +68,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, - spared: 0, kill_hold: 0, - killed: 0, - killed_at: [0; cfg::RENDER_SLOTS], - build_room: true, released: 0, epoch: 0, footprints: KVec::new(), @@ -121,7 +92,7 @@ impl State { & Self::MASK == 0 => { - Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) + Err(ENOSPC) } _ => Ok(()), } @@ -131,15 +102,6 @@ impl State { self.epochs[slot] = self.epoch; } fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { - if self.owners[slot] != owner { - crate::g17::trace::teardown( - 6, - slot as u64, - self.owners[slot], - owner | u64::from(self.killed & (1u64 << slot) != 0) << 63, - ); - self.killed &= !(1u64 << slot); - } self.refs[slot] += 1; self.active |= 1u64 << slot; self.released &= !(1u64 << slot); @@ -153,20 +115,18 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - let idle = (0..cfg::RENDER_SLOTS) - .filter(|slot| { - self.owners[*slot] == owner - && self.refs[*slot] == 0 - && self.quarantined & (1u64 << slot) == 0 - }) - .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); - if let Some(slot) = idle { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) + for slot in 0..cfg::RENDER_SLOTS { + if self.owners[slot] == owner + && self.refs[slot] == 0 + && self.quarantined & (1u64 << slot) == 0 { - self.claim_sibling(owner); + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) + { + self.claim_sibling(owner); + } + return Some(self.take_idle(slot, owner)); } - return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -185,23 +145,13 @@ impl State { } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; - let released = |slot: usize| self.released & (1u64 << slot) != 0; - let unowned = || (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0); - // Reuse a closed, exactly retired owner before consuming a new graph. A killed - // context's pair comes after a new graph while one can still be built: on Neo, new - // owners of such pairs had renders that executed but never stored their stamps. - // Once the queue table is full it is reused, the one released longest ago first, - // so killed pairs never strand physical queues. + // Reuse a closed, exactly retired owner before consuming a new graph. let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && released(*slot) && self.killed & (1u64 << slot) == 0) + .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) .min_by_key(|slot| self.epochs[*slot]) - .or_else(|| if self.build_room { unowned() } else { None }) .or_else(|| { - (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && released(*slot)) - .min_by_key(|slot| self.killed_at[*slot]) - }) - .or_else(unowned)?; + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + })?; let first = self.take_idle(slot, owner); self.claim_sibling(owner); Some(first) @@ -210,8 +160,7 @@ impl State { if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { return; } - // A speculative sibling never takes a killed context's pair. - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold & !self.killed; + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; let sibling = (0..cfg::RENDER_SLOTS) .filter(|slot| { @@ -221,61 +170,13 @@ impl State { }) .min_by_key(|slot| self.epochs[*slot]) .or_else(|| { - // Nor an unowned slot once no pair can be built there. - if !self.build_room { - return None; - } (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) }); if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - // A speculative sibling has no GPU-use recency yet. - self.epochs[slot] = 0; - } - } - - /// The device lock separately proves the physical graph was never published. - /// An open owner keeps another usable pair. Exclude every live borrower, - /// including off-lock sibling construction; a closed owner needs no spare. - fn reassign_unused_spare( - &mut self, - slot: u8, - expected: Option, - owner: Option, - ) -> bool { - let slot = usize::from(slot); - if slot == 0 - || slot >= cfg::RENDER_SLOTS - || owner.is_some_and(|owner| { - owner == 0 - || self.owners.iter().filter(|value| **value == owner).count() - >= cfg::RENDER_PAIRS_PER_OWNER - }) - { - return false; - } - let bit = 1u64 << slot; - let old = self.owners[slot]; - if old == 0 - || self.killed & bit != 0 - || owner == Some(old) - || expected.is_some_and(|expected| expected != old) - || self.refs[slot] != 0 - || (self.active | self.quarantined | self.kill_hold) & bit != 0 - || (self.released & bit == 0 - && !self.owners.iter().enumerate().any(|(other, value)| { - other != slot - && *value == old - && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 - })) - { - return false; + self.advance(slot); } - self.owners[slot] = owner.unwrap_or(0); - self.released &= !bit; - self.epochs[slot] = 0; - true } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -287,31 +188,9 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 - && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } - fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { - let slot = self.warmable_sibling(owner, publishing)?; - self.refs[usize::from(slot)] += 1; - self.active |= 1u64 << slot; - Some(slot) - } - fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { - let slot = usize::from(slot); - let bit = 1u64 << slot; - if self.owners[slot] == owner - && self.refs[slot] == 1 - && (self.quarantined | self.kill_hold) & bit == 0 - { - self.owners[slot] = 0; - self.released &= !bit; - self.epochs[slot] = 0; - true - } else { - false - } - } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -334,28 +213,6 @@ impl State { ) } - /// Whether `pick` would return a slot, without taking it. - fn can_pick(&self, owner: u64, plan: Plan) -> bool { - let usable = |slot: usize| { - self.owners[slot] == owner - && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 - && self.refs[slot] < cfg::RENDER_DEPTH - }; - match plan { - Plan::Wait => false, - Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), - Plan::Any if owner == 0 => false, - Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), - Plan::Any => { - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; - (0..cfg::RENDER_SLOTS).any(|slot| { - free & (1u64 << slot) != 0 - && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) - }) - } - } - } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -394,7 +251,6 @@ impl State { slot, ticket, footprint: request.footprint.clone(), - fragment: None, }, GFP_KERNEL, ) @@ -402,12 +258,6 @@ impl State { Some((owner, request.sequence)) } - fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { - self.footprints.iter().position(|entry| { - entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence - }) - } - fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -433,55 +283,23 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, - /// Logical queues admitted at most: each must be able to hold a render pair. - max_queues: u32, - /// Logical-queue creation waiting for a returned slot. - waiters: Option>, - /// Render publication waiting for render capacity: woken with every - /// release event above. - renders: Option>, } impl Pool { - #[cfg(CONFIG_KUNIT = "y")] - pub(crate) fn new(max_queues: u32) -> Result> { - Self::with_waiters(max_queues, None, None) - } - pub(crate) fn with_waiters( - max_queues: u32, - waiters: Option>, - renders: Option>, - ) -> Result> { - if max_queues > MAX_EXECUTION_CONTEXTS { - return Err(EINVAL); - } + pub(crate) fn new() -> Result> { Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), - max_queues, - waiters, - renders, }), GFP_KERNEL, ) } - /// Every pair or lease return, owner close and quarantine change. - fn returned(&self) { - self.released.notify_all(); - if let Some(renders) = &self.renders { - renders.notify(); - } - } - /// Logical queues currently counted, including one being created. - pub(crate) fn queues(&self) -> u32 { - self.queues.load(Ordering::Acquire) - } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= self.max_queues { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); + if count >= MAX_EXECUTION_CONTEXTS { + return Err(ENOSPC); } match self.queues.compare_exchange( count, @@ -494,36 +312,20 @@ impl Pool { } } } - /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. - pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { - let state = self.state.lock(); - state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) - } - pub(crate) fn acquire_render( self: &Arc, owner: u64, request: Option>, - deadline_ns: u64, - check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let started = crate::g17::now_ns(); - let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); - let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); + let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); + let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); let mut expired = false; let mut ticket = None; let result = (|| loop { - check()?; - let now = crate::g17::now_ns(); - let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -535,31 +337,20 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the - // pair to drain until the caller's deadline, which the caller extends while it - // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); - if now >= quarantine_deadline && !spared { + if quarantine_budget == 0 { return Err(EIO); } - let slice = if spared { - remaining_ms - } else { - remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?) + quarantine_budget = match self + .released + .wait_interruptible_timeout(&mut state, quarantine_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, }; - if matches!(self.released.wait_interruptible_timeout( - &mut state, kernel::time::msecs_to_jiffies(slice) - ), CondVarTimeoutResult::Signal { .. }) { - return Err(ERESTARTSYS); - } continue; } state.refusal(owner)?; - expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -585,17 +376,18 @@ impl Pool { } } } - } - // Keep the established hazard wait/fallback rule, but cap every - // sleep with the caller's original deadline and cancellation poll. - let slice = if plan != Plan::Any { - remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) - } else { - remaining_ms - }; - if matches!(self.released.wait_interruptible_timeout( - &mut state, kernel::time::msecs_to_jiffies(slice) - ), CondVarTimeoutResult::Signal { .. }) { + match self + .released + .wait_interruptible_timeout(&mut state, conflict_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => { + conflict_budget = 0; + expired = true; + } + CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + } + } else if self.released.wait_interruptible(&mut state) { return Err(ERESTARTSYS); } })(); @@ -628,51 +420,8 @@ impl Pool { record, })) } - /// Whether `acquire_render` would admit this request now, without - /// admitting it: the scheduler's prepare step waits for this. - pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { - let state = self.state.lock(); - if owner == 0 || state.refusal(owner).is_err() { - return false; - } - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { - return false; - } - let plan = state.plan(owner, request, None); - state.can_pick(owner, plan) - } - pub(crate) fn warmable_sibling( - self: &Arc, - owner: u64, - publishing: u8, - ) -> Option { - let mut state = self.state.lock(); - let slot = state.borrow_warm_sibling(owner, publishing)?; - // This is a host construction borrower, not a GPU-use epoch. In - // particular it must not clear a logical close or age an unused spare. - Some(RenderLease { - pool: self.clone(), - slot, - record: None, - }) - } - pub(crate) fn reassign_unused_spare( - &self, - slot: u8, - expected: Option, - owner: Option, - ) -> bool { - self.state - .lock() - .reassign_unused_spare(slot, expected, owner) - } - pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { - self.state.lock().owners.get(slot) == Some(&owner) + pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + self.state.lock().warmable_sibling(owner, publishing) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -682,7 +431,7 @@ impl Pool { } } drop(state); - self.returned(); + self.released.notify_all(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -697,97 +446,9 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; - state.spared &= !released; state.kill_hold = 0; - state.killed = 0; drop(state); - self.returned(); - } - - /// Whether the physical queue table can hold another pair (see `State::build_room`). - pub(crate) fn note_build_room(&self, room: bool) { - let mut state = self.state.lock(); - if state.build_room != room { - state.build_room = room; - drop(state); - self.returned(); - } - } - - /// A killed context's pair is free for a new owner. - pub(crate) fn killed_pair_free(&self) -> bool { - let state = self.state.lock(); - state.killed & state.released & !state.active & !state.quarantined & !state.kill_hold & State::MASK != 0 - } - -} - -#[kunit_tests(asahi_g17_admission_accounting)] -mod accounting_tests { - use super::*; - - #[test] - fn every_logical_owner_fits_without_speculative_spares() { - let mut state = State::new(); - for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot = state.acquire(owner).unwrap(); - assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); - state.release(slot); - } - assert_eq!(state.active, 0); - assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); - } - - #[test] - fn close_does_not_reuse_an_active_lease() { - let mut state = State::new(); - let slot = state.acquire(1).unwrap(); - state.released |= 1u64 << slot; - assert!(!state.reassign_unused_spare(slot, Some(1), None)); - state.release(slot); - assert!(state.reassign_unused_spare(slot, Some(1), None)); - assert_eq!(state.owners[usize::from(slot)], 0); - assert_eq!(state.refs[usize::from(slot)], 0); - assert_eq!(state.active & (1u64 << slot), 0); - } - - #[test] - fn warming_quarantine_and_exit_hold_exclude_reassignment() { - let mut state = State::new(); - let primary = state.acquire(1).unwrap(); - // A retained legacy sibling still obeys all quarantine/kill guards. - state.owners[2] = 1; - let spare = state.borrow_warm_sibling(1, primary).unwrap(); - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.release(spare); - state.quarantined |= 1u64 << spare; - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.quarantined = 0; - state.kill_hold |= 1u64 << spare; - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.kill_hold = 0; - assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); - assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); - assert!(!state.reassign_unused_spare(primary, Some(1), None)); - state.release(primary); - assert!(!state.reassign_unused_spare(primary, Some(1), None)); - } - - #[test] - fn logical_queue_count_returns_on_drop() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - for _ in 0..MAX_EXECUTION_CONTEXTS { - leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); - assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); - drop(leases.pop()); - let replacement = pool.reserve_queue().unwrap(); - drop(leases); - assert_eq!(pool.queues.load(Ordering::Acquire), 1); - drop(replacement); - assert_eq!(pool.queues.load(Ordering::Acquire), 0); + self.released.notify_all(); } } @@ -795,18 +456,9 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } -impl QueueSlot { - /// The device's admission pool, which lives as long as the firmware session. - pub(crate) fn pool(&self) -> &Arc { - &self.pool - } -} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); - if let Some(waiters) = &self.pool.waiters { - waiters.notify(); - } } } @@ -816,123 +468,41 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { - /// The device lock proves this slot has no physical identities or graph. - /// Give back an unbound reservation before retrying on a retained channel. - pub(crate) fn cancel_empty(self, owner: u64) -> bool { - let cancelled = { - let mut state = self.pool.state.lock(); - state.cancel_empty_slot(self.slot, owner) - }; - drop(self); - cancelled - } - pub(crate) fn slot(&self) -> u8 { self.slot } - /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine - /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. - pub(crate) fn quarantine(&self, spared: bool) { - let bit = 1u64 << self.slot; - let mut state = self.pool.state.lock(); - if !spared { - state.spared &= !bit; - } else if state.quarantined & bit == 0 { - state.spared |= bit; - } - state.quarantined |= bit; - drop(state); - self.pool.returned(); + pub(crate) fn quarantine(&self) { + self.pool.state.lock().quarantined |= 1u64 << self.slot; + self.pool.released.notify_all(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - let mut state = self.pool.state.lock(); - state.quarantined &= !(1u64 << self.slot); - state.spared &= !(1u64 << self.slot); + self.pool.state.lock().quarantined &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { self.rehabilitate_after_processor_stop(); } } -impl RenderLease { - /// Called under the device mutex once the render's kicks are committed, - /// including a replay, which supersedes the earlier kick. - pub(crate) fn note_publication(&self, fragment: KickDependency) { - let Some(record) = self.record else { - return; - }; - let mut state = self.pool.state.lock(); - if let Some(index) = state.position(self.slot, record) { - state.footprints[index].fragment = Some(fragment); - } - } -} - -impl Pool { - /// Under the device mutex, which orders every publication: the newest - /// published fragment kick of each other pair whose live same-VM render - /// overlaps `footprint`. Quarantined pairs are skipped as in admission. - pub(crate) fn publication_barriers( - &self, - owner: u64, - slot: u8, - footprint: &Footprint, - barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], - ) -> Result { - let state = self.state.lock(); - let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; - let mut count = 0; - for entry in state.footprints.iter() { - let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { - continue; - }; - if other == slot - || entry.owner == owner - || state.quarantined & (1u64 << other) != 0 - || !footprint.same_vm(&entry.footprint) - || !footprint.overlaps(&entry.footprint) - { - continue; - } - match slots[..count].iter().position(|known| *known == other) { - // One fragment ring completes in kick order; same QID, so the - // word orders by timestamp. - Some(index) => { - if fragment.word() > barriers[index].word() { - barriers[index] = fragment; - } - } - // Admission bounds the pairs; see hazard::plan. - None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), - None => { - slots[count] = other; - barriers[count] = fragment; - count += 1; - } - } - } - Ok(count) - } -} - impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some(record) = self.record { - if let Some(index) = state.position(self.slot, record) { + if let Some((owner, sequence)) = self.record { + if let Some(index) = state.footprints.iter().position(|entry| { + entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence + }) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.returned(); + self.pool.released.notify_all(); } } impl State { - fn hold_for_exit_kill(&mut self, owner: u64, slots: u64, executed: u64) -> bool { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { if owner == 0 || slots & !Self::MASK != 0 { return false; } @@ -941,10 +511,10 @@ impl State { if slots & bit == 0 { continue; } - // A pair whose commands all ran is held with them (see `Pair::executed_for_kill`). if self.owners[slot] != owner - || (executed & bit == 0 - && (self.refs[slot] != 0 || self.active & bit != 0 || self.released & bit == 0)) + || self.refs[slot] != 0 + || self.active & bit != 0 + || self.released & bit == 0 { return false; } @@ -955,280 +525,17 @@ impl State { } impl Pool { /// Hold the departed owner's idle pairs before publishing its exit record. - /// Successful publication retains these bits until the owner's scheduler release is - /// consumed (`release_killed_slots` at detach), or until both processors stop. - pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64, executed: u64) -> bool { - self.state.lock().hold_for_exit_kill(owner, slots, executed) + /// Successful publication retains these bits until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots) } - /// Undo a hold whose control record was not published. + /// Undo only a hold whose control record was not published. pub(crate) fn release_exit_kill_hold(&self, slots: u64) { if slots == 0 { return; } self.state.lock().kill_hold &= !slots; - self.returned(); - } - - /// End the kill hold of `slots` once their killed context's scheduler release was - /// consumed (the firmware no longer names these pairs). A new owner takes them last. - pub(crate) fn release_killed_slots(&self, slots: u64) { - let mut state = self.state.lock(); - let held = state.kill_hold & slots & State::MASK; - if held == 0 { - return; - } - let now = crate::g17::now_ns(); - state.kill_hold &= !held; - state.killed |= held; - for slot in 0..cfg::RENDER_SLOTS { - if held & (1u64 << slot) != 0 { - state.killed_at[slot] = now; - } - } - drop(state); - self.returned(); - } -} - -impl Pool { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; - let state = self.state.lock(); - let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { - bits | if *owner != 0 { 1u64 << slot } else { 0 } - }) & State::MASK; - let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; - writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", - used.count_ones(), State::MASK.count_ones(), retained.count_ones(), - (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), - (state.kill_hold & State::MASK).count_ones()) - } -} - -#[kunit_tests(asahi_g17_fault_admission)] -mod fault_capacity_tests { - use super::*; - - #[test] - fn a_guilty_failure_makes_a_spared_quarantine_guilty() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let bit = 1u64 << lease.slot(); - assert!(!pool.spared_quarantined(1)); - lease.quarantine(true); - assert_ne!(pool.state.lock().spared & bit, 0); - assert!(pool.spared_quarantined(1)); - lease.quarantine(false); - assert_eq!(pool.state.lock().spared & bit, 0); - assert!(!pool.spared_quarantined(1)); - // Once guilty, a later spared failure does not make it spared again. - lease.quarantine(true); - assert_eq!(pool.state.lock().spared & bit, 0); - assert_ne!(pool.state.lock().quarantined & bit, 0); - pool.release_owner(1); - lease.rehabilitate_after_firmware_drain(); - drop(lease); - drop(queue); - let state = pool.state.lock(); - assert_eq!(state.quarantined | state.spared, 0); - } - - #[test] - fn closed_faulting_clients_return_leases_after_drain() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - for owner in 1..=512 { - let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - lease.quarantine(false); - pool.release_owner(owner); - let bit = 1u64 << lease.slot(); - assert_ne!(pool.state.lock().active & bit, 0); - assert_ne!(pool.state.lock().quarantined & bit, 0); - // The runtime, not an error fence, supplies this exact drain witness. - lease.rehabilitate_after_firmware_drain(); - drop(lease); - drop(queue); - let state = pool.state.lock(); - assert_eq!(state.active, 0); - assert_eq!(state.quarantined, 0); - assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); - assert_eq!(pool.queues.load(Ordering::Acquire), 0); - } - } - - #[test] - fn published_exit_kill_holds_survive_close_and_drain() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let bit = 1u64 << lease.slot(); - pool.release_owner(1); - drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); - assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); - // A failed (unpublished) control may roll back its hold. Published kills - // keep the existing processor-stop rule; this test must not weaken it. - pool.release_retained_owners_after_processor_stop(); - assert_eq!(pool.state.lock().kill_hold, 0); - } - - #[test] - fn consumed_scheduler_release_ends_the_kill_hold() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let slot = lease.slot(); - let bit = 1u64 << slot; - pool.release_owner(1); - drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); - assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - pool.release_killed_slots(bit); - assert_eq!(pool.state.lock().kill_hold, 0); - assert_eq!(pool.state.lock().killed, bit); - assert!(pool.killed_pair_free()); - // The killed context's pair is never reassigned as a spare, and a new owner takes - // a new graph before it. - assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_ne!(next.slot(), slot); - drop(next); - pool.release_retained_owners_after_processor_stop(); - assert_eq!(pool.state.lock().killed, 0); - } - - #[test] - fn killed_pairs_are_taken_last_oldest_first() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = 100 + slot as u64; - state.released |= 1u64 << slot; - state.killed |= 1u64 << slot; - state.killed_at[slot] = 1000 - slot as u64; - } - // One pair released without a kill. - state.killed &= !(1u64 << 7); - } - let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(first.slot(), 7); - let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); - assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); - } - - #[test] - fn a_full_queue_table_reuses_killed_pairs_before_unowned_slots() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..8 { - state.owners[slot] = 100 + slot as u64; - state.released |= 1u64 << slot; - state.killed |= 1u64 << slot; - state.killed_at[slot] = 1000 - slot as u64; - } - } - // With room in the queue table a new owner builds a pair in an unowned slot. - let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - assert!(usize::from(built.slot()) >= 8); - // Without room it takes the killed pair released longest ago. - pool.note_build_room(false); - let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(reused.slot(), 7); - assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); - // It claims no sibling: neither another killed pair nor an unowned slot. - let state = pool.state.lock(); - for slot in 1..7 { - assert_eq!(state.owners[slot], 100 + slot as u64); - } - assert_eq!(state.owners.iter().filter(|owner| **owner == 2).count(), 1); - } -} - -impl Pool { - pub(crate) fn owner_active(&self, owner: u64) -> bool { - let state = self.state.lock(); - state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) - } -} - -#[kunit_tests(asahi_g17_shared_render_depth)] -mod shared_depth_tests { - use super::*; - #[test] - fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { - let mut state = State::new(); - let first = state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } - assert_eq!(state.acquire(1), None); - assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); - state.release(first); - assert_eq!(state.acquire(1), Some(first)); - } -} - -#[kunit_tests(asahi_g17_admission_deadline)] -mod deadline_tests { - use super::*; - - #[test] - fn all_pairs_busy_no_progress_expires_without_release() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - // No pair has room, including the requesting owner's retained pair. - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = slot as u64; - state.refs[slot] = cfg::RENDER_DEPTH; - } - state.active = State::MASK; - } - let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); - assert!(matches!(result, Err(error) if error == EBUSY)); - assert!(crate::g17::now_ns() - start < 500_000_000); - assert!(pool.state.lock().footprints.is_empty()); - } - - #[test] - fn busy_pair_observes_cancellation_without_a_release_event() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = slot as u64; - state.refs[slot] = cfg::RENDER_DEPTH; - } - state.active = State::MASK; - } - let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 2_000_000_000, || { - if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } - }); - assert!(matches!(result, Err(error) if error == ECANCELED)); - assert!(crate::g17::now_ns() - start < 500_000_000); - assert!(pool.state.lock().footprints.is_empty()); - } - - #[test] - fn cancellation_precedes_available_capacity() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), - Err(error) if error == ECANCELED)); - assert_eq!(pool.state.lock().active, 0); - } - - #[test] - fn unrelated_wake_storm_never_renews_absolute_deadline() { - let deadline = 2_000_000_000; - // Each wake is allowed to report the full jiffy budget remaining. Only - // monotonic time, not that report or a wake count, bounds admission. - for now in (0..deadline).step_by(100_000) { - assert!(wait_slice_ms(deadline, now).unwrap() <= 10); - } - assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); - assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + self.released.notify_all(); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 55994066825f8..b78598cfbb760 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,13 +9,8 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{ - channels::ControlRecord, - compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, - queue::*, - variant::PointerLayout, - }, - object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, + fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -30,21 +25,10 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; -const NATIVE_WAIT_OFFSET: usize = 0x900; -const NATIVE_WAIT_STRIDE: usize = 0x60; -// A submission can name every physical compute channel. Keep the complete -// wait list inside its own cached slot, including the last retained slot. -const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET - + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) - .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; -/// Producer/consumer words live on their own uncached page after the graph, -/// which the firmware reads through a cached mapping (queue record, job-list -/// head and item ring), as native does. -const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -94,32 +78,24 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - // Drop the GPU alias before the retained storage returns to its pool. - low: mmu::KernelMapping, - object: PooledObject, + object: KernelObject, + high: mmu::KernelMapping, } impl DescriptorRing { - /// Firmware reads every descriptor through a cached kernel mapping, as it - /// does for render. The pool keeps that mapping until processor stop; the - /// hardware reads the same pages through an uncached canonical low alias. - pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { - let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; - let object = objects.allocate( - alloc, - Family::Descriptor, - size, - size, - mmu::PROT_GPU_FW_SHARED_RW, + pub(crate) fn new(alloc: &Allocator<'_>) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let object = alloc.lower( + DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, + page, + prot, CpuMap::WriteCombined, )?; - let low = object.alias_in( - alloc.uat.kernel_lower_vm(), - dynamic::LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - )?; - Ok(Self { low, object }) + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + Ok(Self { object, high }) } } @@ -131,80 +107,47 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: PooledObject, + graph: KernelObject, context_low: KernelObject, context_high: KernelObject, - /// Announcements, which stay uncached for the firmware. records: KernelObject, - /// Configuration, predecessor and completion-wait records at the same - /// offsets, which the firmware reads through a cached mapping. - cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, - /// Host words of the pointer block in this firmware build. - pointers: PointerLayout, - /// Layout of the operand-state buffer for the GPU's cores. - operand_state: &'static OperandState, } impl Graph { - fn new( - alloc: &Allocator<'_>, - objects: &Arc, - context: &Context, - pointers: PointerLayout, - operand_state: &'static OperandState, - ) -> Result { + fn new(alloc: &Allocator<'_>, context: &Context) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc, objects)?; + let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(operand_state.size, page, prot, cpu)?; + let operand = alloc.lower(0x14000, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = objects.allocate( - alloc, - Family::QueueState, - GRAPH_SIZE + mmu::UAT_PGSZ, - GRAPH_SIZE, - prot, - cpu, - )?; + let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + CURSORS as u64, + pointers_va: graph_va + POINTERS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(&pointers)?; + view.pointers = PointerBlock::new(); view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; - graph.initialize::(CURSORS, |view| { - *view = PointerBlock::new(&pointers)?; - Ok(()) - })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; - let cached_records = objects.allocate( - alloc, - Family::QueueConfig, - RECORD_SLOTS * CACHED_RECORD_STRIDE, - RECORD_SLOTS * CACHED_RECORD_STRIDE, - prot, - cpu, - )?; Ok(Self { ring: Some(ring), preempt, @@ -215,11 +158,8 @@ impl Graph { context_low, context_high, records, - cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, - pointers, - operand_state, }) } @@ -281,12 +221,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self - .graph - .word(CURSORS + self.pointers.producer)? - .load(Ordering::Relaxed), - consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), - count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), + count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -297,12 +234,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) + Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -321,7 +258,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EAGAIN); + return Err(EBUSY); } } Ok(()) @@ -336,14 +273,10 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph - .dword(ITEMS + index * 8)? - .store(0, Ordering::Relaxed); + self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph - .word(CURSORS + self.pointers.producer)? - .store(cursors.consumer, Ordering::Relaxed); + self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -456,21 +389,11 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous> { - binding: Option, - context: C, +struct Previous { + binding: Option, + context: Arc, publication: Option<(u64, u64)>, } -impl Previous { - /// The exact drain witness covers the replacement publication even when - /// its public fence failed. An unpublished replacement still needs rollback. - fn retire_witnessed(previous: &mut Option) { - if previous.as_ref().is_some_and(|p| p.publication.is_some()) { - drop(previous.take()); - } - } -} - /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -530,21 +453,12 @@ pub(crate) enum PoolPhase { Vacant, } -/// Builder ownership of one physical release transaction. The preparation -/// lease pins this registry until abort/commit; owner close may remove lookups. -#[derive(Clone, Copy)] -pub(crate) struct Growth { - qid: u8, - phase: PoolPhase, -} - pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, - release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -557,8 +471,6 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, - /// This binding has never crossed the successor visibility boundary. - unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -568,24 +480,12 @@ pub(crate) struct Queue { replay_error: Option, replay_prepared: bool, exit: Option, - /// Since when the kill of this failed queue's context waits for other contexts' compute, - /// and whether that wait was reported. - kill_wait: Option<(u64, bool)>, installed: bool, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, - /// Last configuration record the firmware consumed, and its recovery - /// epoch. A command repeats the record only when it differs. - config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, - /// The stamp of the newest command the firmware reported failed ([`Self::note_fault`]), - /// while it is newer than the stored stamp. - fault_stamp: Option, - /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). - claims: u32, - last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -599,8 +499,6 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, - source_readers: Arc, - source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -608,26 +506,20 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, - objects: &Arc, backing: Backing, owner: u64, context: Arc, - pointers: PointerLayout, - operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, &context)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); - context.note_compute_binding(true); - crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, - release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -635,7 +527,6 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, - unpublished_binding: true, kick, pool, graph, @@ -645,16 +536,11 @@ impl Queue { replay_error: None, replay_prepared: false, exit: None, - kill_wait: None, installed: false, outer_started: 0, published_config: None, - config_cache: None, ordinal: 0, submitted: 0, - fault_stamp: None, - claims: 0, - last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -668,44 +554,8 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, - source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, - source_hint: crate::g17::sync_input::ReaderHint::new(), - }) - } - /// The firmware failed the command whose completion stamp is `stamp`. It writes a - /// zero-duration completion record for it, which quarantines the queue, but stores no - /// stamp: without this the quarantine would wait for its retirement witness forever and - /// keep the queue's backing. Commands published after the failed one still need their own - /// stamp or record. - pub(crate) fn note_fault(&mut self, stamp: u32) { - if self - .fault_stamp - .is_none_or(|old| retirement::stamp_newer(stamp, old)) - { - self.fault_stamp = Some(stamp); - } - } - - /// The firmware failed a command of this quarantined queue, and the queue still waits for - /// its retirement witness: commands published after the failed one, which the firmware - /// never runs, or the failed command itself, whose ring entry it does not consume (a ray - /// traversal's, on Neo). Only a kill of the queue's context gives them back. - pub(crate) fn fault_pending(&self) -> bool { - self.fault_stamp.is_some() - && self.quarantined - && self.awaiting_witness - && self.exit.is_none() - } - - /// The stamp that proves retirement: the stored one, or a newer reported fault's. - fn witness_stamp(&self) -> Result { - let stored = self.graph.stamp()?; - Ok(match self.fault_stamp { - Some(fault) if retirement::stamp_newer(fault, stored) => fault, - _ => stored, }) } - pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } @@ -716,40 +566,16 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } - /// The first command after an owner change away from `context` has been - /// published and not retired: the queue still runs that scheduler's state. - pub(crate) fn switching_from(&self, context: &Arc) -> bool { - self.previous.as_ref().is_some_and(|previous| { - previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) - }) - } - /// The context these queue records were bound to before an owner change - /// whose first command is not published yet. - pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { - self.previous - .as_ref() - .filter(|previous| previous.publication.is_none()) - .map(|previous| &previous.context) - } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self - .context() - .is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - /// The oldest command is a published long job (ray traversal or AS build) whose - /// completion record the driver has not seen yet. - pub(crate) fn long_job_running(&self) -> bool { - !self.quarantined - && self.active.front().is_some_and(|active| { - active.observed.is_none() - && active.packet.is_published() - && active.packet.command.long_job() - }) + pub(crate) fn room(&self) -> bool { + !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { @@ -757,10 +583,9 @@ impl Queue { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: active.packet.context.vm().id(), + vm: Arc::as_ptr(active.packet.completion.status()) as usize, published: active.published, finished: active.observed.is_some(), - failed: self.quarantined, }) } @@ -778,27 +603,6 @@ impl Queue { visit(active.packet.completion.status()); } } - /// A recovery record names this queue and the completion stamp of the - /// command it interrupted, `kick << 8` like the queue's stamp. Without a - /// stamp match the oldest published command is marked. - pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { - let sequence = stamp >> 8; - if let Some(active) = self.active.iter().find(|active| { - active - .ticket - .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) - }) { - active.packet.completion.note_interrupted(); - return recovery::Interrupted::Exact; - } - match self.active.iter().find(|active| active.ticket.is_some()) { - Some(active) => { - active.packet.completion.note_interrupted(); - recovery::Interrupted::Oldest - } - None => recovery::Interrupted::None, - } - } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -812,10 +616,7 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self - .binding - .as_ref() - .map_or(0, |binding| binding.context.qos_id()), + qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -825,62 +626,38 @@ impl Queue { !self.quarantined && self.retire_pending } - /// A release has no completion event: keep observing the control consumer - /// until its witness lets the idle physical queue be re-armed. - pub(crate) fn pool_release_pending(&self) -> bool { - matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) - } - - pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() - && crate::g17::timeout::background_stalled(self.release_sent_ns, now) - } - - /// A closed owner can still retain its last commands or their completion - /// stamps. Admission may wait for retirement, without treating that close - /// as permission to reuse any firmware-owned backing. - pub(crate) fn release_settling(&self) -> bool { - // release_owner keeps the old owner while its last commands retire. - // Waiting only once owner is None misses precisely that interval. - release_pending( - self.released, - self.quarantined, - self.active.len, - self.retire_pending, - ) - } - fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan, - /// whichever queues the scan's batch retires. - pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { - if self.quarantined { + /// Stage every unlatched exact identity before the device's single shared-ring scan. + pub(crate) fn request_observations( + &self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { return Ok(false); } - batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; - let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - let stamp = match stamp { - Some(stamp) => stamp, - None => *stamp.insert(self.graph.stamp()?), - }; - batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; + batch.request(&ticket)?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { - if self.quarantined { + pub(crate) fn latch_observations( + &mut self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { return Ok(()); } for active in self.active.iter_mut() { @@ -984,7 +761,6 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { - let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -1004,18 +780,9 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = !retirement::stamp_covers(stamp, ticket.kick) - && match retirement::stamp_wait(front.observed_at, now_ns(), || { - // A node that cannot be read keeps the bound from the record. - front.work.drained().unwrap_or(true) - }) { - retirement::StampWait::Waiting => false, - retirement::StampWait::Restart(now) => { - restart = Some(now); - false - } - retirement::StampWait::Aged => true, - }; + let record_aged = front + .observed_at + .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); let timestamps = retirement::poll( &ticket, front.observed, @@ -1047,16 +814,6 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; - if let Some(ticket) = front.ticket { - crate::g17::recovery::note_retired(); - crate::g17::trace::complete( - crate::g17::trace::COMPUTE, - ticket.qid, - ticket.ordinal, - &front.packet, - [timestamps[0], timestamps[1]], - ); - } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -1072,13 +829,9 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; - self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { - if restart.is_some() { - self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; - } self.retire_pending = self.front_finished(); break; } @@ -1129,7 +882,7 @@ impl Queue { if !retirement::quarantined_retired( &newest, self.graph.cursors()?, - self.witness_stamp()?, + self.graph.stamp()?, self.active.len as u32, self.submitted, )? { @@ -1173,14 +926,9 @@ impl Queue { drop(self.active.pop()); } self.awaiting_witness = false; - self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; - // The stamp and fully consumed ring proved the replacement retired. - // Keeping its predecessor only on the fault path prevents recycling - // forever: neither another retirement nor an exit kill can release it. - Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -1189,9 +937,15 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; + if self + .previous + .as_ref() + .is_some_and(|p| p.publication.is_some()) + { + drop(self.previous.take()); + } if self.released { self.owner = None; - self.released_at = now_ns(); } } Ok(()) @@ -1273,245 +1027,6 @@ impl Queue { } } -/// Validate the complete unpublished item budget before changing descriptors, -/// work counts or queue producers. Leave one item empty across ring wrap. -fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); - return Err(EAGAIN); - } - Ok(()) -} - -/// Retained commands are capacity, not a failed VM. Nothing has been written -/// when this check runs, so the outer publisher can wait for retirement under -/// its original admission deadline, just as it does for a full firmware ring. -fn retained_capacity( - quarantined: bool, - released: bool, - active: usize, - replays: usize, - replay_front: bool, -) -> Result { - if quarantined || released { - return Err(EBUSY); - } - if active >= IN_FLIGHT || (replays != 0 && !replay_front) { - return Err(EAGAIN); - } - Ok(()) -} - -/// Logical close promises a future healthy handoff, not immediate reuse. -/// Exact retirement still owns the transition. -fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { - released && !quarantined && (active != 0 || retiring) -} - -#[kunit_tests(asahi_g17_compute_admission)] -mod admission_tests { - use super::*; - - #[test] - fn complete_native_wait_list_fits_every_cached_slot() { - let waits = crate::g17::kick::COMPUTE_CHANNELS; - for slot in 0..RECORD_SLOTS { - for index in 0..waits { - let start = record_start( - slot, - NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, - CACHED_RECORD_STRIDE, - ) - .unwrap(); - assert!(start >= slot * CACHED_RECORD_STRIDE); - assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); - } - } - } - - #[test] - fn record_window_cannot_overwrite_the_next_slot() { - // The nineteenth wait used to cross the old slot boundary silently. - let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; - assert_eq!( - record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), - Err(EINVAL) - ); - assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); - assert_eq!( - record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), - Ok(CACHED_RECORD_STRIDE - 1) - ); - } - - #[test] - fn retained_ring_pressure_waits_without_discarding_work() { - let mut active = Fifo::new(); - for sequence in 0..IN_FLIGHT { - active.push(sequence).unwrap(); - } - assert_eq!( - retained_capacity(false, false, active.len, 0, false), - Err(EAGAIN) - ); - assert_eq!(active.front(), Some(&0)); - assert_eq!(active.len, IN_FLIGHT); - assert_eq!(active.pop(), Some(0)); - assert_eq!( - retained_capacity(false, false, active.len, 0, false), - Ok(()) - ); - assert_eq!(active.front(), Some(&1)); - } - - #[test] - fn replay_front_and_terminal_ownership_remain_distinct() { - assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); - assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!( - retained_capacity(false, false, IN_FLIGHT, 1, true), - Err(EAGAIN) - ); - for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!( - retained_capacity(quarantined, released, 0, 1, true), - Err(EBUSY) - ); - } - } - - #[test] - fn item_refusal_is_retryable_and_preserves_the_empty_slot() { - let n = ITEM_RING_ENTRIES; - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 4, - count: n - }, - false - ), - Ok(()) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 4, - count: n - }, - true - ), - Err(EAGAIN) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n - 2, - consumer: 2, - count: n - }, - false - ), - Ok(()) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n - 2, - consumer: 2, - count: n - }, - true - ), - Err(EAGAIN) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 10, - consumer: 10, - count: n - }, - true - ), - Ok(()) - ); - } - - #[test] - fn bad_item_cursors_are_not_backpressure() { - let n = ITEM_RING_ENTRIES; - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 0, - count: 0 - }, - false - ), - Err(EIO) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n, - consumer: 0, - count: n - }, - false - ), - Err(ERANGE) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: n, - count: n - }, - true - ), - Err(ERANGE) - ); - } - - #[test] - fn closed_owner_waits_for_its_last_command() { - assert!(release_pending(true, false, 1, false)); - assert!(release_pending(true, false, 0, true)); - assert!(!release_pending(true, false, 0, false)); - } - - #[test] - fn live_or_quarantined_owners_do_not_promise_capacity() { - assert!(!release_pending(false, false, 1, true)); - assert!(!release_pending(true, true, 1, true)); - } -} - impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1531,9 +1046,7 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { - return false; - } + if self.growing { return false; } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1549,15 +1062,11 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() - // Borrow acquisition is serialized by the device mutex. Avoid - // touching the separate loan counter for an otherwise busy queue. - && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } - self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1597,15 +1106,7 @@ impl Queue { self.retained_owner = None; } context.mark_published(); - context.note_compute_binding(true); - crate::g17::trace::bind( - crate::g17::trace::COMPUTE, - self.qid(), - context.id(), - if same_vm { 2 } else { 1 }, - ); self.owner = Some(owner); - self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1614,9 +1115,6 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { - if let Some(binding) = &self.binding { - binding.context.note_compute_binding(false); - } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1636,13 +1134,6 @@ impl Queue { } self.retirement_ready = true; } - // Rebinding an already detached queue has no Previous. If its - // first command is rejected or cancelled before publication, the - // empty active FIFO is still the original idle witness. No future - // completion can turn retirement_ready back on for this owner. - if self.previous.is_none() { - self.retirement_ready = true; - } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1671,10 +1162,7 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.pool_phase == PoolPhase::Populated - && self.pool.populated() - && (self.reusable() || self.recycled_pending()) - && !self.source_hint.has_readers(&self.source_readers) + (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1732,7 +1220,6 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; - self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1742,7 +1229,6 @@ impl Queue { self.spared_deferred = false; self.retire_pending = false; self.awaiting_witness = false; - self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.replay_error = None; @@ -1768,11 +1254,11 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = - ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = ControlRecord::FreeListRelease( + self.pool.release_record(self.free_list_generation), + ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1784,9 +1270,7 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { - return Ok(None); - } + if self.growing { return Ok(None); } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1795,18 +1279,11 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self - .binding - .as_mut() - .and_then(|binding| binding.pool_alias.take()); + let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self - .binding - .as_mut() - .and_then(|binding| binding.descriptors.take()); + let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); self.pool_phase = PoolPhase::Vacant; - self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1821,13 +1298,12 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { + if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; - self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1837,50 +1313,30 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth( - &mut self, - init: &crate::g17::initdata::InitData, - blocks: usize, - ) -> Result { + pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { if self.growing { - return Err(EAGAIN); + if blocks < self.growth_blocks { return Err(EAGAIN); } + self.growth_blocks = blocks; + return Ok(false); } - if self.quarantined - || self.released - || self.exit.is_some() - || self.active.len != 0 - || self.replays.len != 0 - || self.previous.is_some() + if self.quarantined || self.released || self.exit.is_some() + || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated - { - return Err(EAGAIN); - } + || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } if self.installed { - let record = - ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { - self.pool_phase = PoolPhase::Vacant; - self.release_sent_ns = None; - } + } else { self.pool_phase = PoolPhase::Vacant; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth( - &mut self, - init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, - blocks: usize, - ) -> Result { - if !self.growing || blocks < self.growth_blocks { - return Err(EAGAIN); - } + alias: &mut Option, blocks: usize) -> Result { + if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1888,45 +1344,16 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { - return Err(EINVAL); - } - let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - // No ownership moves before every fallible descriptor/header operation. - // The old release is consumed; no queue can use this row until commit. - init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; - self.pool.replace_pages(pages)?; - let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); + let old_alias = binding.pool_alias.take(); + let old_pages = self.pool.take_pages(); + self.pool.rearm(pages.take().ok_or(EINVAL)?)?; + init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; + binding.pool_alias = alias.take(); self.pool_phase = PoolPhase::Populated; - self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { - _alias: old_alias, - _pages: pages.take(), - _descriptors: None, - _ring: None, - }) - } - - pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { - qid: self.qid(), - phase: self.pool_phase, - }) - } - - /// Transfer an abandoned builder's release back to the event worker. Do not - /// revoke the published release or free backing here: vacancy still requires - /// its exact control-consumer witness. A different transaction is untouched. - pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { - if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { - return Ok(None); - } - self.growing = false; - self.growth_blocks = 0; - self.release_owner().map(Some) + Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -2009,17 +1436,14 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { - let Some(target) = self.graph.pointers.recovery_target else { - return Ok(()); - }; let producer = self .graph .graph - .word(CURSORS + self.graph.pointers.producer)? + .word(POINTERS + PointerBlock::PRODUCER)? .load(Ordering::Relaxed); self.graph .graph - .word(CURSORS + target)? + .word(POINTERS + PointerBlock::RECOVERY_TARGET)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -2056,7 +1480,9 @@ impl Queue { let ticket = active.ticket.ok_or(EIO)?; let state = crate::g17::job::fence_status(&active.packet.completion.fence()); if state < 0 { - return Err(crate::g17::dependency::PRODUCER_FAILED); + return Err(crate::g17::dependency::inherited_error(Error::from_errno( + state, + ))); } if state > 0 { return Ok(None); @@ -2069,17 +1495,6 @@ impl Queue { } } -/// Validate the individual slot before deriving an offset into its allocation. -/// Allocation-wide checks cannot detect a write into the following live slot. -fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { - if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { - return Err(EINVAL); - } - slot.checked_mul(stride) - .and_then(|start| start.checked_add(offset)) - .ok_or(EINVAL) -} - impl Graph { fn record( &mut self, @@ -2089,15 +1504,12 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { - (&mut self.records, RECORD_STRIDE) - } else { - (&mut self.cached_records, CACHED_RECORD_STRIDE) - }; - (object, record_start(slot, offset, window, stride)?) + if slot >= RECORD_SLOTS { + return Err(EINVAL); + } + (&mut self.records, slot * RECORD_STRIDE + offset) } else { - let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; - (&mut *self.graph, 0x6800 + start) + (&mut self.graph, 0x6800 + offset) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -2123,7 +1535,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(CURSORS + self.pointers.producer)?; + let producer = self.graph.word(POINTERS + 0x40)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -2149,7 +1561,8 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) + -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -2167,17 +1580,6 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.publish_with_native_waits(packet, dependencies, &[], host, defer) - } - - pub(in crate::g17) fn publish_with_native_waits( - &mut self, - packet: Arc, - dependencies: &[crate::g17::fw::kick::KickDependency], - native: &[crate::g17::sync_input::NativeWait], - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -2212,26 +1614,10 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; - // The first prototype only waits on an installed consumer channel. - // A retry without a published source falls back to completion on CPU. - if first && !native.is_empty() { - return Err(EAGAIN); - } - if native.len() > crate::g17::kick::COMPUTE_CHANNELS { - return Err(EINVAL); - } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); - item_capacity(self.graph.cursors()?, entry_signal)?; - if !native.is_empty() { - let cursors = self.graph.cursors()?; - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { - return Err(EAGAIN); - } - } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -2254,7 +1640,6 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, - operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -2276,10 +1661,6 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let cached = self - .config_cache - .and_then(|(config, at)| (at == epoch).then_some(config)); - let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -2301,7 +1682,6 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; - self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -2327,7 +1707,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write_zeroed(&args), + |descriptor| descriptor.write(&args), )?; if first { self.graph.context_high.write( @@ -2344,25 +1724,17 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = if send_config { - Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) - } else { - None - }; + let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; - for (index, wait) in native.iter().enumerate() { - let record = wait.bytes( - crate::g17::fw::stamp_value(kick_timestamp.get() as u32), - packet.order.sequence, - ); - let address = self.graph.record( - record_slot, - NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, - record, - )?; - wait_vas.push(address, GFP_KERNEL)?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + return Err(ENOSPC); } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; @@ -2371,7 +1743,6 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; - self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -2389,7 +1760,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va.ok_or(EINVAL)?, || {}) + .append(producer, config_va, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -2398,27 +1769,16 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = (|| -> Result { - let mut before_descriptor = cursors.producer; - for &wait in &wait_vas { - before_descriptor = self.graph.append(before_descriptor, wait, || { - packet.mark_published(); - visible = true; - })?; - } - self.graph.append(before_descriptor, descriptor_high, || { - packet.mark_published(); - visible = true; + producer = self + .graph + .append(cursors.producer, descriptor_high, || { + packet.mark_published() }) - })() - .or_else(|error| { - if !visible { + .or_else(|error| { self.pool.cancel_unpublished(target)?; - } - Err(error) - })?; + Err(error) + })?; visible = true; - self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -2429,16 +1789,14 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - if let Some(config_va) = config_va { - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; - } + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; } if first { prepared_kick.prepare_compute_install()?; @@ -2468,12 +1826,10 @@ impl Queue { self.outer_started & ring == 0, )?, )?; + let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); - if send_config { - self.config_cache = Some((config, epoch)); - } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -2498,13 +1854,13 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - // Every publication rings its own pipe and priority class, as render - // publication does: pipe 2 (compute) in bits 1:0, priority class in - // bits 3:2. The generic work doorbell left the firmware to find the - // ring itself and delayed dispatch. + if notify_activation { + fence(Ordering::SeqCst); + // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + } fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); + host.notify((0x83 << 48) | 0x10)?; host.note_submission() })(); if let Err(error) = result { @@ -2542,13 +1898,11 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity( - self.quarantined, - self.released, - self.active.len, - self.replays.len, - replay_front, - )?; + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } if !packet.context.is_current() || !self .binding @@ -2615,7 +1969,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(CURSORS + self.pointers.producer)?; + let producer = self.graph.word(POINTERS + 0x40)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -2628,307 +1982,3 @@ impl Queue { self.terminal.take() } } - -impl Queue { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", - self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), - u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), - u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; - match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!( - f, - "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, - cursors.count - 1 - ), - _ => writeln!(f, "items_unavailable=1"), - }?; - writeln!( - f, - "compute_settling qid={} release={} pool_release={}", - self.qid(), - u8::from(self.release_settling()), - u8::from(self.pool_release_pending()) - )?; - if let Some(binding) = self.binding.as_ref() { - binding - .context - .write_capacity(f, "compute", usize::from(self.qid()))?; - } - if let Some(previous) = self.previous.as_ref() { - previous - .context - .write_capacity(f, "compute_previous", usize::from(self.qid()))?; - } - Ok(()) - } -} - -#[kunit_tests(asahi_g17_fault_ownership)] -mod fault_ownership_tests { - use super::*; - use core::sync::atomic::AtomicUsize; - - struct Lease(Arc); - impl Lease { - fn new(used: &Arc) -> Self { - used.fetch_add(1, Ordering::Relaxed); - Self(used.clone()) - } - } - impl Drop for Lease { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::Relaxed); - } - } - - // This tests the production predecessor ownership operation without firmware - // storage. Runtime tests must separately establish its drain/kill witness. - fn faulting_clients(publication: Option<(u64, u64)>) { - let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); - for _ in 0..512 { - let mut previous = Some(Previous { - binding: Some(Lease::new(&used)), - context: Lease::new(&used), - publication, - }); - assert_eq!(used.load(Ordering::Relaxed), 2); - Previous::retire_witnessed(&mut previous); - assert_eq!(previous.is_none(), publication.is_some()); - if publication.is_none() { - assert_eq!(used.load(Ordering::Relaxed), 2); - // An unpublished replacement returns its original owner on close. - drop(previous.take()); - } - assert_eq!(used.load(Ordering::Relaxed), 0); - Previous::retire_witnessed(&mut previous); - assert_eq!(used.load(Ordering::Relaxed), 0); - } - } - - #[test] - fn faulted_published_replacements_return_to_baseline() { - faulting_clients(Some((1, 7))); - } - - #[test] - fn unpublished_replacements_survive_until_rollback() { - faulting_clients(None); - } - - #[test] - fn same_vm_replacement_without_binding_returns_its_context() { - let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); - let mut previous = Some(Previous:: { - binding: None, - context: Lease::new(&used), - publication: Some((9, 3)), - }); - Previous::retire_witnessed(&mut previous); - assert_eq!(used.load(Ordering::Relaxed), 0); - } -} - -impl Queue { - /// Offer an open client's idle physical channel to another ready job. - /// Logical queue/context ownership is unchanged. The existing binding is - /// retained through the successor's exact replacement witness. - pub(crate) fn share_idle(&mut self) -> Result { - if !self.shareable() { - return Ok(false); - } - self.release_owner()?; - Ok(self.reusable()) - } - - /// Another owner could take this channel now: idle and released, shareable, or vacant. - pub(crate) fn channel_free(&self) -> bool { - self.reusable() || self.shareable() || self.can_rearm() - } - - /// A pool replacement is in flight for this queue. - pub(crate) fn growing(&self) -> bool { - self.growing - } - - /// The queue's first command has been published: its identity and configuration exist. - pub(crate) fn installed(&self) -> bool { - self.installed - } - - /// A packet of the bound owner passed prepare_job and will publish here. - pub(crate) fn claim(&mut self) { - self.claims = self.claims.saturating_add(1); - } - - pub(crate) fn release_claim(&mut self) { - self.claims = self.claims.saturating_sub(1); - } - - /// An open client's idle channel that [`Self::share_idle`] can hand over. - pub(crate) fn shareable(&self) -> bool { - self.claims == 0 - && self.installed - && self.owner.is_some() - && self.retirement_ready - && !self.growing - && !self.quarantined - && !self.awaiting_witness - && self.exit.is_none() - && self.active.len == 0 - && self.replays.len == 0 - && self.previous.is_none() - && self.pool_phase == PoolPhase::Populated - && !self.source_hint.has_readers(&self.source_readers) - } - - /// Device publication order of this channel's newest command: lower is - /// idle longer. - pub(crate) fn last_published(&self) -> u64 { - self.last_published - } -} - -impl Queue { - pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant - && (self.reusable() || self.recycled_pending()) - && !self.source_hint.has_readers(&self.source_readers) - } -} - -impl Queue { - /// A failed first attempt has no future completion to undo its binding. - /// Return the replaced aliases to the caller for off-lock destruction. The - /// caller validates the current preparation epoch and cancels only a true - /// result's never-published QID; installed rings and identities stay intact. - pub(crate) fn rollback_unpublished( - &mut self, - owner: u64, - context: &Arc, - ) -> Result<(bool, Option)> { - if self.owner != Some(owner) - || !self.unpublished_binding - || self - .binding - .as_ref() - .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing - || self.quarantined - || self.awaiting_witness - || self.exit.is_some() - || self.active.len != 0 - || self.replays.len != 0 - || self.retire_pending - || self.failure_status_pending - || self.pool_phase != PoolPhase::Populated - || self - .previous - .as_ref() - .is_some_and(|previous| previous.publication.is_some()) - { - return Ok((false, None)); - } - if !self.installed { - return Ok((true, None)); - } - let mut retired = None; - if let Some(previous) = self.previous.as_ref() { - self.graph.set_owner(&previous.context)?; - } - if let Some(mut previous) = self.previous.take() { - if let Some(binding) = previous.binding.take() { - retired = self.binding.replace(binding); - } else { - self.binding.as_mut().ok_or(EIO)?.context = previous.context; - } - } - self.unpublished_binding = false; - self.retirement_ready = true; - self.released = true; - self.owner = None; - self.released_at = now_ns(); - Ok((false, retired)) - } -} - -impl Queue { - /// Same immutable singleton identity and current physical epoch. - pub(crate) fn has_input_source( - &self, - source: &Arc, - status: &Arc, - epoch: u64, - ) -> bool { - !self.quarantined - && self.replays.len == 0 - && self.active.iter().any(|active| { - active.ticket.is_some_and(|ticket| ticket.epoch == epoch) - && Arc::ptr_eq(active.packet.completion.status(), status) - && active.packet.completion.is_input_source(source) - }) - } - - pub(crate) fn source_dependency( - &mut self, - source: &crate::g17::sync_input::Source, - status: &Arc, - epoch: u64, - ) -> Result> { - for active in self.active.iter() { - if !active.packet.completion.is_input_source(&source.submission) { - continue; - } - if !Arc::ptr_eq(active.packet.completion.status(), status) { - return Err(EINVAL); - } - if self.quarantined || self.replays.len != 0 { - return Err(EAGAIN); - } - let ticket = active.ticket.ok_or(EAGAIN)?; - if ticket.epoch != epoch { - return Err(EAGAIN); - } - self.valid_ticket(active, &ticket)?; - if active.packet.cancel_requested() { - return Err(ECANCELED); - } - source.submission.source_status()?; - let dependency = crate::g17::fw::kick::KickDependency::new( - self.qid(), - crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, - ) - .ok_or(EIO)?; - let hold = self.source_hint.hold(&self.source_readers)?; - return Ok(Some(( - active.published, - dependency, - self.graph.support.gpu_va() + STAMP as u64, - u32::from(self.qid()), - hold, - ))); - } - Ok(None) - } - - /// Preparing a possible wait does not create a firmware consumer. Mark it - /// only once the consumer's first item is visible, under the device mutex. - pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { - for active in self.active.iter() { - if active.packet.completion.is_input_source(&source.submission) { - active.packet.completion.note_firmware_consumer(); - break; - } - } - } -} - -/// Publication, dependency, stamp address, queue and the retained source hold. -pub(crate) type SourceDependency = ( - u64, - crate::g17::fw::kick::KickDependency, - u64, - u32, - crate::g17::sync_input::Hold, -); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 719a74787ca70..187fd27f8e78e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,12 +83,11 @@ impl Transaction { } impl Queue { - /// A running command with ray traversal scratch is never killed. On Neo, killing a + /// A command with ray traversal scratch is never killed. On Neo, killing a /// traversal that stopped making progress either crashed the GPU firmware /// when the queue's pool was then released, or stalled every later render /// when it was retained; left alone, the rest of the GPU kept working. Such - /// a queue stays quarantined and retained until its commands retire. A - /// traversal the firmware reported failed no longer runs (see `fault_pending`). + /// a queue stays quarantined and retained until its commands retire. fn active_traversal(&self) -> bool { self.active.iter().any(|active| { matches!( @@ -102,7 +101,7 @@ impl Queue { && self.released && self.active.len != 0 && self.exit.is_none() - && (!self.active_traversal() || self.fault_pending()) + && !self.active_traversal() && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { @@ -111,10 +110,6 @@ impl Queue { CANCELLED_DELAY_NS } } - /// A kill of `context` was published through this queue. - pub(crate) fn exit_published(&self, context: &Arc) -> bool { - self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - } pub(crate) fn exit_owner(&self, context: &Arc) -> Option { if !self .active @@ -129,23 +124,6 @@ impl Queue { .and_then(|active| active.ticket.map(|ticket| ticket.owner)) }) } - /// The kill of this queue's failed context waits; returns how long it has waited, and - /// whether this is the first time it crossed `slow`. - pub(crate) fn note_kill_wait(&mut self, now: u64, slow: u64) -> (u64, bool) { - let (since, warned) = self.kill_wait.get_or_insert((now, false)); - let waited = now.saturating_sub(*since); - let crossed = waited >= slow && !*warned; - *warned |= crossed; - (waited, crossed) - } - /// How long the kill of this queue's failed context has been waiting, if it is. - pub(crate) fn kill_waiting(&self, now: u64) -> Option { - self.kill_wait.map(|(since, _)| now.saturating_sub(since)) - } - /// The kill was published: how long it waited, if it did. - pub(crate) fn take_kill_wait(&mut self, now: u64) -> u64 { - self.kill_wait.take().map_or(0, |(since, _)| now.saturating_sub(since)) - } pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { self.exit = Some(Transaction { cookie, @@ -184,7 +162,6 @@ impl Queue { return false; } exit.phase = Phase::Acknowledged; - crate::g17::trace::teardown(4, cookie, u64::from(self.qid()), 0); true } pub(crate) fn settle_exit( @@ -243,9 +220,6 @@ impl Queue { ))?; drop(self.active.pop()); } - // The acknowledged kill and command settlement replace the ordinary - // drain witness, including a failed first publication after rebinding. - super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index d7f821bbce68d..b4d4bcef24421 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::{CompletionFold, ComputeCompletion}; +use crate::g17::initdata::ComputeCompletion; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -52,12 +52,6 @@ impl Cursors { } } -/// Whether stamp `a` was stored after stamp `b` (both kick stamps, modulo wrap). -pub(crate) const fn stamp_newer(a: u32, b: u32) -> bool { - let delta = a.wrapping_sub(b); - delta != 0 && delta < 0x8000_0000 -} - /// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); @@ -72,69 +66,21 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone, counted from when its ioctl's work-state node drained. -/// -/// The firmware stores the stamp after the cache maintenance that follows the record, so the -/// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on -/// the ioctl's work-state node has completed: while other commands of the same submit ioctl -/// still run, the stamp is not late, however long they run (a command buffer of a dozen render -/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the -/// node drains. Only a stamp still missing well past that is treated as never stored. The bound -/// stays far below the scheduler timeout, so such a command cannot fail its VM. -pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; - -// A command that waits out the fallback must still retire well inside the scheduler timeout. -const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); - -/// Whether a completion record latched at `observed_at` (host time, ns) has waited out -/// [`STAMP_FALLBACK_NS`] by `now`. -pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { - observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) -} - -/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum StampWait { - /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. - Waiting, - /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not - /// late, and the wait counts again from the given host time. - Restart(u64), - /// The node drained and the stamp is still missing after the bound: retire on the record. - Aged, -} - -/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and -/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the -/// command's work-state node completed; it is asked only once the bound has passed. -pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) - -> StampWait -{ - if !record_aged(observed_at, now) { - StampWait::Waiting - } else if drained() { - StampWait::Aged - } else { - StampWait::Restart(now) - } -} +/// before it retires on the record alone. The firmware stores the stamp after the cache +/// maintenance that follows the record; it occasionally never stores it for a command, most +/// visibly while a render of the same VM is running. +pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub(crate) fn stampless_total() -> u64 { - STAMPLESS.load(core::sync::atomic::Ordering::Relaxed) -} - -/// Counts a command that retired on its record because its completion stamp never arrived; -/// logs the first and every 64th. +/// Counts a command that retired on its record without its completion stamp; logs the first and +/// every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: lifetime total {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", count, - STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -144,14 +90,7 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery -/// generation. Recovery may consume items without completing their writes, so an old record -/// and consumed items cannot replace the stamp across a restart. In both cases its items -/// must be consumed. -#[expect( - clippy::too_many_arguments, - reason = "Retirement validates the ticket against independent firmware observations." -)] +/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -165,9 +104,7 @@ pub(crate) fn poll( if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - let same_generation = !matches!(recordless, Recordless::CrossedRecovery); - let completed = stamp_covers(stamp, ticket.kick) - || (same_generation && observed.is_some() && record_aged); + let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } @@ -203,122 +140,35 @@ pub(crate) fn quarantined_retired( && submitted == pending) } -/// Walks a completion ring of `seen.len()` slots from `start` over the records written since -/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, -/// so the first slot still holding it is where the firmware writes next. `record` folds one -/// slot; the walk also ends once it reports every request found, or at a requested record -/// still being written, which the next walk reads again. Returns where the next walk starts. -pub(crate) fn walk_completions( - start: usize, - seen: &mut [u64], - mut control: impl FnMut(usize) -> Result, - mut record: impl FnMut(usize) -> Result, -) -> Result { - let slots = seen.len(); - if slots == 0 { - return Err(EINVAL); - } - let start = start % slots; - for step in 0..slots { - let slot = (start + step) % slots; - let word = control(slot)?; - if word == seen[slot] { - return Ok(slot); - } - match record(slot)? { - CompletionFold::Incomplete => return Ok(slot), - CompletionFold::Passed => seen[slot] = word, - CompletionFold::Done => { - seen[slot] = word; - return Ok((slot + 1) % slots); - } - } - } - Ok(start) -} - #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, - covered: bool, - next: u16, -} - -const NO_REQUEST: u16 = u16::MAX; -const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); - -fn request_bucket(qid: usize, kick: u64) -> usize { - qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, - /// Per-QID kick buckets; collisions retain the complete identity check. - heads: KVVec, slots: [(usize, usize); QIDS], open: Option, - /// Requests with no observation yet; the scan stops when this reaches zero. - unobserved: usize, - /// Unobserved requests whose queue's completion stamp already covered them when staged. - /// The firmware stores the stamp after the record, so each has a record in the ring. - covered: usize, - /// Per QID, the next kick its queue had not published when the batch was staged - /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs - /// to a command published during the scan, which a later batch stages. - frontiers: [u64; QIDS], } -const NO_FRONTIER: u64 = u64::MAX; -const KICK_MASK: u64 = (1 << 40) - 1; - impl Batch { pub(crate) fn new() -> Result { - let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; - for _ in 0..QIDS * IN_FLIGHT { - heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; - } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, - heads, slots: [(0, 0); QIDS], open: None, - unobserved: 0, - covered: 0, - frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; - self.unobserved = 0; - self.covered = 0; - self.frontiers.fill(NO_FRONTIER); - } - /// Records the first kick `qid`'s queue had not yet published when this batch was staged. - pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { - *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; - Ok(()) - } - /// A record of `qid` with `kick` names a command published after this batch was staged. - fn published_after_staging(&self, qid: usize, kick: u64) -> bool { - let frontier = self.frontiers[qid]; - frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 - } - /// True once every requested identity has an observation. - pub(crate) fn complete(&self) -> bool { - self.unobserved == 0 - } - /// Some request whose record must already be in the ring is still unobserved. - pub(crate) fn covered_pending(&self) -> bool { - self.covered != 0 } - /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. - pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { + pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -330,460 +180,63 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { - let begin = ticket.qid as usize * IN_FLIGHT; - self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; - let bucket = request_bucket(ticket.qid as usize, ticket.kick); - let next = *self.heads.get(bucket).ok_or(EIO)?; - let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, - covered, - next, }) .map_err(|_| ENOSPC)?; - *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); - self.unobserved += 1; - self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once. Most slots belong to an older kick of a - /// selected QID; reject those before reading any additional shared words. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - if !self.fold_reader(|word| record.read(word))? { - return Ok(CompletionFold::Incomplete); - } - Ok(self.folded()) - } - - /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. - pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { - self.fold_reader(|word| record.read(word)).map(|_| ()) - } - - fn folded(&self) -> CompletionFold { - if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } - } - - /// Ok(false) when a pending request's record is still being written. - fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { - let control = read(0)?; + /// Capture the control word once, then read only a requested identity's payload. + /// Records can be overwritten during a scan: incomplete or older identities are skipped. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + let control = record.read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (_, count) = self.slots[qid]; - let kick = - (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let (start, count) = self.slots[qid]; if count == 0 { - return Ok(!self.published_after_staging(qid, kick)); + return Ok(()); } - let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; - let mut descriptor = None; - let mut found = None; - for _ in 0..count { - if index == NO_REQUEST { - break; - } - let request = self.requests.get(usize::from(index)).ok_or(EIO)?; - if request.kick == kick { - let value = if let Some(value) = descriptor { - value - } else { - let value = read(6)? & !1; - if value == 0 || read(7)? == 0 { - return Ok(request.observed.is_some()); - } - descriptor = Some(value); - value - }; - if request.descriptor == value { - // Chains run newest first. Retain the original first-match - // rule even if a caller stages the same identity twice. - found = Some(usize::from(index)); - } - } - index = request.next; + let descriptor = record.read(6)? & !1; + if descriptor == 0 || record.read(7)? == 0 { + return Ok(()); } - let Some(index) = found else { - return Ok(!self.published_after_staging(qid, kick)); + let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; + let Some(request) = requests + .iter_mut() + .find(|r| r.descriptor == descriptor && r.kick == kick) + else { + return Ok(()); }; - let request = self.requests.get_mut(index).ok_or(EIO)?; - let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; - if timestamps[0] == 0 || timestamps[1] == 0 { - return Ok(request.observed.is_some()); - } - if request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; + if timestamps[0] == 0 + || timestamps[1] == 0 + || request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(true); - } - if request.observed.is_none() { - self.unobserved -= 1; - self.covered -= usize::from(request.covered); + return Ok(()); } request.observed = Some(Observation { timestamps }); - Ok(true) + Ok(()) } - - /// A command published after this batch staged its queue's requests was not part of the - /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - Ok(self + let request = self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .and_then(|request| request.observed.take())) - } -} - -#[kunit_tests(asahi_g17_compute_stamp_wait)] -mod tests { - use super::*; - - const KICK: u64 = 0x12_3456_7801; - - fn ticket() -> Ticket { - Ticket { - owner: 1, - submission: 1, - context_id: 1, - qid: 28, - descriptor: 0x1000, - work_node: 0x2000, - kick: KICK, - ordinal: 1, - item_producer: 4, - item_count: 64, - epoch: 1, - } - } - - fn consumed() -> Cursors { - Cursors { - producer: 4, - consumer: 4, - count: 64, - } - } - - fn own(kick: u64) -> u32 { - (kick as u32).wrapping_shl(8) - } - - #[test] - fn stamp_covers_its_kick_and_earlier_ones() { - assert!(stamp_covers(own(KICK), KICK)); - assert!(stamp_covers(own(KICK + 1), KICK)); - assert!(!stamp_covers(own(KICK - 1), KICK)); - assert!(!stamp_covers(own(KICK) | 1, KICK)); - assert!(stamp_covers(0, 0xff_ffff)); - } - - #[test] - fn a_reported_fault_stamp_is_newer_only_until_stored() { - assert!(stamp_newer(own(KICK), own(KICK - 1))); - assert!(!stamp_newer(own(KICK), own(KICK))); - assert!(!stamp_newer(own(KICK - 1), own(KICK))); - // Across the wrap of the 24-bit kick in the stamp word. - assert!(stamp_newer(own(0), own(0xff_ffff))); - } - - #[test] - fn a_late_stamp_is_waited_for() { - // Under games on J700 the stamp lands up to about 4 ms after the record while the - // rest of the command's ioctl runs: the command keeps its mappings until then. - let record = Some(Observation { timestamps: [10, 20] }); - let at = 1_000_000; - assert!(!record_aged(Some(at), at + 4_000_000)); - assert!(!record_aged(Some(at), at + 10_000_000)); - let waiting = poll( - &ticket(), - record, - consumed(), - own(KICK - 1), - Recordless::NotYet, - 0, - None, - record_aged(Some(at), at + 10_000_000), - ); - assert_eq!(waiting, Ok(None)); - let stamped = poll( - &ticket(), - record, - consumed(), - own(KICK), - Recordless::NotYet, - 0, - None, - false, - ); - assert_eq!(stamped, Ok(Some([10, 20]))); - } - - #[test] - fn a_later_kick_stamp_retires_it() { - let record = Some(Observation { timestamps: [10, 20] }); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK + 3), - Recordless::NotYet, - 0, - None, - false, - ); - assert_eq!(result, Ok(Some([10, 20]))); - } - - #[test] - fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { - let record = Some(Observation { timestamps: [0x1111, 0x2222] }); - let at = 1_000_000; - assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); - assert!(!record_aged(None, u64::MAX)); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK - 1), - Recordless::NotYet, - 0, - None, - record_aged(Some(at), at + STAMP_FALLBACK_NS), - ); - // The record's own timestamps are published, never synthesized ones. - assert_eq!(result, Ok(Some([0x1111, 0x2222]))); - } - - #[test] - fn a_stamp_waits_for_the_rest_of_its_ioctl() { - let at = 1_000_000; - let late = at + STAMP_FALLBACK_NS; - // The node is asked only once the bound passed. - assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); - assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), - StampWait::Waiting); - // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. - assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); - assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); - // The node drained and the stamp is still missing after the bound. - assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); - assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); - } - - #[test] - fn an_aged_record_still_waits_for_consumed_items() { - let record = Some(Observation { timestamps: [10, 20] }); - let busy = Cursors { - producer: 4, - consumer: 2, - count: 64, - }; - let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); - assert_eq!(result, Ok(None)); - } - - #[test] - fn recovery_cannot_turn_an_aged_record_into_a_release_witness() { - let record = Some(Observation { timestamps: [10, 20] }); - // Recovery can advance the consumer without completing cache maintenance. - // Even a record observed before the restart and aged past the fallback - // must retain its command's mappings until a real stamp covers it. - for stamp in [0, own(KICK - 1)] { - assert_eq!(poll(&ticket(), record, consumed(), stamp, - Recordless::CrossedRecovery, 0, None, true), Ok(None)); - } - // A real stamp restores the release witness; completion's interrupted - // classification independently prevents false successful fences. - for stamp in [own(KICK), own(KICK + 1)] { - assert_eq!(poll(&ticket(), record, consumed(), stamp, - Recordless::CrossedRecovery, 0, None, true), Ok(Some([10, 20]))); - } - let busy = Cursors { producer: 4, consumer: 2, count: 64 }; - assert_eq!(poll(&ticket(), record, busy, own(KICK), - Recordless::CrossedRecovery, 0, None, true), Ok(None)); - // The existing same-generation fallback remains available. - assert_eq!(poll(&ticket(), record, consumed(), own(KICK - 1), - Recordless::Allowed, 0, None, true), Ok(Some([10, 20]))); - } - - #[test] - fn a_zero_duration_record_fails() { - let record = Some(Observation { timestamps: [5, 5] }); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK), - Recordless::Allowed, - 0, - None, - true, - ); - assert_eq!(result, Err(ENODATA)); - } -} - - -#[kunit_tests(asahi_g17_completion_index)] -mod index_tests { - use super::*; - - fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { - Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, - work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } - } - fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { - let mut words = [0; 8]; - words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) - | (u64::from(ticket.qid) << 9); - words[2] = timestamps[0]; words[5] = timestamps[1]; - words[6] = ticket.descriptor | 1; words[7] = 1; - words - } - #[test] - fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000), false).unwrap(); - let words = record(&ticket(7, 17, 0x1000), [10, 20]); - batch.fold_reader(|word| { - assert_eq!(word, 0); - Ok(words[word]) - }).unwrap(); - assert!(!batch.complete()); - } - #[test] - fn collisions_and_reused_descriptors_keep_full_kick_identity() { - let mut batch = Batch::new().unwrap(); - let first = ticket(7, 33, 0x1000); - let second = ticket(7, 49, 0x1000); - batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); - let words = record(&first, [10, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(!batch.complete()); - let words = record(&second, [30, 40]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(batch.complete()); - assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); - assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); - } - #[test] - fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { - let mut batch = Batch::new().unwrap(); - let first = ticket(7, 33, 0x1000); - let second = ticket(7, 33, 0x2000); - batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); - let words = record(&first, [20, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); - assert!(batch.take(&second).unwrap().is_none()); - } - #[test] - fn duplicate_identity_keeps_the_first_staged_request() { - let mut batch = Batch::new().unwrap(); - let current = ticket(7, 33, 0x1000); - batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); - let words = record(¤t, [10, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); - assert_eq!(batch.unobserved, 1); - } - #[test] - fn clear_restarts_buckets_and_later_publication_stays_unobserved() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); - let current = ticket(7, 49, 0x1000); - batch.request(¤t, false).unwrap(); - let stale = record(&ticket(7, 33, 0x1000), [10, 20]); - batch.fold_reader(|word| Ok(stale[word])).unwrap(); - assert!(!batch.complete()); - assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); - let words = record(¤t, [30, 40]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(batch.complete()); - } - #[test] - fn a_requested_record_still_being_written_is_read_again() { - let mut batch = Batch::new().unwrap(); - let current = ticket(7, 33, 0x1000); - batch.request(¤t, true).unwrap(); - let mut words = record(¤t, [10, 20]); - words[7] = 0; - assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); - words[7] = 1; - words[5] = 0; - assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); - assert!(batch.covered_pending()); - let words = record(¤t, [10, 20]); - assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); - assert!(batch.complete() && !batch.covered_pending()); - } - #[test] - fn a_command_published_during_the_scan_is_left_for_the_next_batch() { - let mut batch = Batch::new().unwrap(); - batch.stage_frontier(7, 50).unwrap(); - batch.request(&ticket(7, 49, 0x1000), false).unwrap(); - batch.stage_frontier(9, 3).unwrap(); - let late = record(&ticket(7, 50, 0x3000), [30, 40]); - assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); - let idle = record(&ticket(9, 3, 0x3000), [30, 40]); - assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); - let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); - assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); - let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); - assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); - } - fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) - -> usize - { - walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() - } - #[test] - fn a_walk_reads_only_the_records_written_since_the_previous_one() { - let mut seen = [0u64; 8]; - let mut ring = [0u64; 8]; - assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); - ring[0] = 10; ring[1] = 11; ring[2] = 12; - let read = core::cell::Cell::new(0); - assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); - assert_eq!(read.get(), 3); - ring[3] = 13; - assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); - // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. - for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } - assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); - } - #[test] - fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { - let mut seen = [0u64; 8]; - let ring = [10, 11, 12, 13, 0, 0, 0, 0]; - assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } - else { CompletionFold::Passed }), 2); - assert_eq!(seen[..3], [10, 11, 0]); - assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } - else { CompletionFold::Passed }), 3); - assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); - } - #[test] - fn an_unrequested_or_stale_record_is_passed() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 49, 0x1000), false).unwrap(); - let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); - stale[7] = 0; - assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); - let other = record(&ticket(9, 33, 0x1000), [10, 20]); - assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); - assert!(!batch.complete() && !batch.covered_pending()); + .ok_or(EIO)?; + Ok(request.observed.take()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b7037a1c35f75..569b5e289e7d8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,8 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - dependency::FailureRecord, - fence::Outputs, + fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -28,6 +27,7 @@ use kernel::{ xarray, }; +const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -56,28 +56,21 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, - variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new( - priority: u32, - usc_base: u64, - range: Range, - variant: &'static crate::g17::fw::Variant, - ) -> Result { + pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { if priority > 3 { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, allocation_failure, range, window, - variant, }) } } @@ -85,7 +78,6 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, - usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -93,24 +85,15 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, - variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], - /// Finished frontier entries pruned during this ioctl. Their last references - /// can tear a whole completion down; that waits until the ioctl has published. - finished: KVec, - failures: [FailureRecord; 2], + failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } -type ValidatedBatch = ( - KVec<(Validated, Order, Destinations)>, - KVec, -); - impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -118,20 +101,17 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, - usage: Arc, ) -> Result { let Frontend { fences, allocation_failure, range, window, - variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { dev: dev.into(), context, - usage, backend, render, compute: None, @@ -139,19 +119,16 @@ impl Queue { allocation_failure, range, window, - variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), - finished: KVec::new(), - failures: [FailureRecord::new(), FailureRecord::new()], + failures: [None, None], _slot: slot, }) } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Validate every request, including an existing scheduler. Physical - // admission happens after its DRM dependencies, in backend publication. + // Admission is checked even when this queue already owns a scheduler. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -165,40 +142,27 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; - // Without room the entries are released here, as they always could be. - let mut finished = if self - .finished - .reserve(self.frontiers[index].len(), GFP_KERNEL) - .is_ok() - { - Some(&mut self.finished) - } else { - None - }; self.frontiers[index].retain(|(sequence, fences)| { - let status = fences.status(); + let status = job::fence_status(&fences.completed); if status == 0 { return true; } - if let Some(finished) = finished.as_mut() { - let _ = finished.push_within_capacity(fences.clone()); + // One earliest failed ordinal covers every later prefix without + // retaining its packet, mappings, or every failed fence. + if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { + *failure = Some((*sequence, fences.completed.clone())); } - failure.note(*sequence, status, fences); false }); } - #[expect( - clippy::too_many_arguments, - reason = "Enqueue keeps validated command, ordering, and destination owners explicit." - )] fn enqueue( &mut self, id: u64, command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &super::fence::Aggregate, + aggregate: &Arc, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -208,62 +172,22 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self - .backend - .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { - return Err(EINVAL); - } + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { - return Err(EINVAL); - }; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } - // Only convert inputs when no independent host-prefix wait could keep - // the retained source bookkeeping queued behind an external signal. - if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); - } - let sources = if engine == Engine::Compute - && !inputs.is_empty() - && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| { - !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 - }) - && self.frontiers[1].iter().all(|(sequence, fences)| { - !Order::contains(order.wait_through[1], *sequence) - || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0 - }) - && self.failures.iter_mut().enumerate().all(|(index, failure)| { - failure.earliest().is_none_or(|(sequence, _)| { - !Order::contains(order.wait_through[index], sequence) - }) - }) { - self.backend.input_sources(inputs) - } else { - None - }; - if engine == Engine::Compute { - super::trace::inputs( - self.context.id(), - order.sequence, - 1, - inputs.len(), - sources.as_deref(), - ); - } - // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -275,12 +199,9 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - self.backend.feed(), - sources.clone(), - self.usage.clone(), + if engine == Engine::Render { self.backend.feed() } else { None }, )?; - let publication = completion - .publication() + let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -311,17 +232,14 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); - let mut failures: [Option; 2] = [None, None]; - for (index, failure) in self.failures.iter_mut().enumerate() { - failures[index] = failure.earliest().and_then(|(sequence, fences)| { - Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone()) - }); - } + let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { + self.failures[index].as_ref().and_then(|(sequence, fence)| { + Order::contains(order.wait_through[index], *sequence).then_some(fence) + }) + }); let failure_count = failures.iter().flatten().count(); - let mut dependencies = KVec::with_capacity( - inputs.len() + count + timestamp_count + failure_count, - GFP_KERNEL, - )?; + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -342,7 +260,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if fences.status() > 0 { + if job::fence_status(&fences.completed) > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -352,33 +270,13 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { - // An input that already signalled without error orders nothing and has no status - // left to consume (like a completed frontier producer above). Keeping it would - // send the render through the scheduler thread instead of inline publication: - // most compositor and client renders carry such already-signalled buffer fences. - if job::fence_status(fence) > 0 { - continue; - } - if sources - .as_ref() - .is_some_and(|sources| sources.contains_input(fence)) - { - continue; - } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - if let Some(sources) = &sources { - for ready in sources.publication_waits() { - dependencies.push(ready.clone(), GFP_KERNEL)?; - checked_inputs.push(ready.clone(), GFP_KERNEL)?; - } - } - // Publish from the ioctl whenever nothing has to wait, at any depth. - // The backend falls back to the scheduler behind any backlog or on any - // capacity miss, so submission order is preserved. - let early = dependencies.is_empty(); + let early = engine == Engine::Render + && dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( id, self.context.clone(), @@ -392,11 +290,10 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = - scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -410,7 +307,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result { + ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -423,7 +320,6 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, - self.variant, &command.payload, )?; let engine = match &payload { @@ -463,10 +359,8 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [ - super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), - ]; + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -480,28 +374,19 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } - let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending - .iter() - .any(|(command, _, _)| matches!(command, Validated::Render { .. })) - { - self.context.note_render(); - } - if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { - command.allow_long_job(); - } - } + let (pending, guards) = self.prepare_submission(bytes, objects)?; let mut guards = guards.into_iter(); - let firmware_source = pending.len() == 1 - && matches!(&pending[0].0, Validated::Compute { .. }) - && !pending[0].2.has_outputs(); - let aggregate = outputs.publish(self.context.clone(), firmware_source)?; + let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -516,8 +401,6 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); - // Everything this ioctl publishes is visible; release what it pruned. - self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2436cfd33ed35..99be455b48e40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -pub(super) mod publication; +mod publication; pub(crate) mod retirement; mod warm; @@ -46,9 +46,6 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, - /// Fragment kicks of other pairs' overlapping same-VM renders. - pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], - pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -58,8 +55,6 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, - hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], - hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -83,18 +78,15 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - /// Publishes `owners` in order with one pass over the QoS table. - fn qos_publish_pair( + fn qos_publish( &mut self, - owners: [qos::Owner; 2], + owner: qos::Owner, scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result<[qos::Publication; 2]>; + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - /// The next publication number, shared with compute (timeout ordering). - fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -128,8 +120,6 @@ struct Prepared { mcache: Option, } struct Active { - /// Publication number, shared with compute (crate::g17::timeout). - published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -142,8 +132,6 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, - /// First observation of this group executed and consumed while its stamps lagged. - stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -164,11 +152,8 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, - graph_started: bool, clusters: u32, - deflake: abi::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -183,31 +168,14 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// Retired, never-executed groups awaiting republication on this pair, in - /// their original order. Nothing else is prepared on the pair before them. - replay: Tracker>, - /// When the oldest parked group was parked or the last one republished. - replay_since: Option, - /// Scheduler page of the context of the last published command: the - /// scheduler the firmware last ran these queues for. Zero for a fresh graph. - executed: u64, - /// An owner change in flight: the previous scheduler page and the ordinal of - /// the new owner's first command, until that command leaves the pair. - switch: Option<(u64, u64)>, - /// Completion values of the two kicks of the last publication, kept with the configuration - /// cache: what each queue holds once that publication completed. - published_completions: Option<[u64; 2]>, + /// A retired, never-executed group awaiting republication on this pair. + replay: Option>, } impl Pair { #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "The packet constructor spells out the paired render firmware inputs." - )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, - rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -218,9 +186,7 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, - deflake: abi::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -245,10 +211,10 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters, variant.programs)?; + let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -260,10 +226,9 @@ impl Pair { pm_generation, 0, word, - variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([ta_alias, fragment_alias]), + kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), memory, kicks: [tiling, fragment], free_list, @@ -274,11 +239,8 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, - graph_started: false, clusters, - deflake, descriptor_flags, - variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -289,12 +251,8 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: Tracker::new(), - replay_since: None, + replay: None, context_update: true, - executed: 0, - switch: None, - published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -322,29 +280,20 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.len() != 0 + self.active.len() != 0 || self.replay.is_some() } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) - } - pub(crate) fn replay_stalled(&self, now: u64) -> bool { - self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) + self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.front().cloned() + self.replay.clone() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self - .replay - .front() - .is_some_and(|front| Arc::ptr_eq(front, packet)) - { + if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay.pop(); - // Progress: the expiry restarts for the next parked group, if any. - self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); + self.replay = None; Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -366,20 +315,13 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self - .replay - .front() - .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, - attachments, - ray_query, - .. + pass, attachments, ray_query, .. } = &packet.command else { return Err(EINVAL); @@ -389,21 +331,17 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; - self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; - let deflake_layout = self.deflake; + let layout = target::Layout::new(pass, self.clusters)?; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory - .generation(ordinal)? - .prepare(ordinal, pass, deflake_layout.viewport)?; + self.memory.generation(ordinal)?.prepare(ordinal, pass)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -433,7 +371,6 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, - gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -465,18 +402,13 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, - programs: self.variant.programs, - deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; - // Without an MCache table kick-entry bit 58 stays clear. - let attachments = match self.variant.mcache { - true => attachments[1].as_slice(), - false => &[][..], - }; - let result = - self.memory - .write_descriptors(&args, attachments, packet.context.id().try_into()?); + let result = self.memory.write_descriptors( + &args, + attachments[1].as_slice(), + packet.context.id().try_into()?, + ); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -504,7 +436,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], + fragment: [KickDependency; 3], fragment_count: usize, } @@ -515,7 +447,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; + let mut fragment_barriers = [own_tiling; 3]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -541,11 +473,6 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } - // Only tile stores overlap: the tiling kick stays concurrent. - for hazard in &dependencies.hazards[..dependencies.hazard_count] { - fragment_barriers[fragment_count] = *hazard; - fragment_count += 1; - } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -576,17 +503,9 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; - self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); - let kick_ids = self.kicks.each_ref().map(kick::Queue::id); - // Command preparation assigns context channel state before publishing - // producers. Pin identities before that assignment; warming never - // enters this path and can still give its unused identities back. - for id in kick_ids { - host.publish_qid(id)?; - } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -596,19 +515,8 @@ impl Pair { let stamps = host.stamps(qids)?; fence(Ordering::Acquire); let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory.completion_links( - ordinal, - kicks, - payload, - self.variant.fragment_busy && !covered[1], - )?; - let leads = match self.variant.lead_commands { - true => { - let init_manager = !self.memory.manager.initialized(); - self.memory.write_leads(ordinal, init_manager)? - } - false => [false; 2], - }; + self.memory + .completion_links(ordinal, kicks, payload, !covered[1])?; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -666,28 +574,10 @@ impl Pair { } else { [None; 2] }; - // A covered queue holds the completion value of its newest kick. When that kick is the - // previous publication of this pair in this recovery epoch, the seed would reload the - // value already held, so the configuration record is needed only for its identity. - // Each record costs the firmware time before it starts the kick. - let held = self - .published_completions - .take() - .filter(|_| self.recovery_generation == epoch); - let present = [0, 1].map(|stage| { - let mut config = configs[stage]; - let reload = kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)); - if config.completion_seed_valid != 0 - && self.installed[stage] - && held.is_some_and(|held| held[stage] == reload) - { - config.completion_seed_valid = 0; - config.completion_seed = 0; - } - publication::required(cache[stage], config) - }); + let present = [ + publication::required(cache[0], configs[0]), + publication::required(cache[1], configs[1]), + ]; let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -696,10 +586,7 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { - pass, ray_query, .. - } = &packet.command - else { + let Validated::Render { pass, ray_query, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -732,7 +619,6 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), - self.variant.programs, )?, None, ); @@ -742,7 +628,6 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), - self.variant.programs, )?, mcache, ); @@ -764,7 +649,6 @@ impl Pair { configs[0], present[0], kicks[0], - leads[0], )?, publication::Lane::new( &memory.graph, @@ -774,7 +658,6 @@ impl Pair { configs[1], present[1], kicks[1], - leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -802,16 +685,17 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory + .graph + .support + .word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; - let published = host.next_publication()?; self.active .push(Active { - published, qos_completed: false, flist_completed: false, outer_published: false, @@ -827,7 +711,6 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, - stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -838,17 +721,18 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - let [fragment, tiling] = host.qos_publish_pair( - [qos_owners[1], qos_owners[0]], - context.scheduler_va(), - policy, - )?; - qos_publications = [Some(tiling), Some(fragment)]; + qos_publications[1] = + Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); + qos_publications[0] = + Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } + for kick in &self.kicks { + host.publish_qid(kick.id())?; + } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -863,12 +747,7 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => { - if leads[0] { - self.memory.manager.mark_initialized(); - } - next - } + Ok(next) => next, Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -890,9 +769,6 @@ impl Pair { } }; self.ordinal = next_ordinal; - if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { - packet.completion.note_render_publication(fragment); - } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -901,16 +777,10 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; - self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } - let scheduler = packet.context.scheduler_va(); - if self.executed != 0 && self.executed != scheduler { - self.switch = Some((self.executed, ordinal)); - } - self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -957,11 +827,7 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self - .registered - .is_some_and(|class| class != policy.priority()) - && self.in_flight() - { + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -999,12 +865,6 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; - crate::g17::trace::publish( - crate::g17::trace::RENDER, - self.qids()[1], - self.ordinal as u32, - packet, - ); host.note_submission() })(); if let Err(error) = publish { @@ -1057,141 +917,13 @@ impl Pair { }, ] } - /// Bounded first-fault evidence while the recovery owner has firmware - /// halted, before it acknowledges state 1. Never changes retirement. - pub(crate) fn report_recovery(&self, stamps: Result<[u32; 2]>) { - pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", - self.slot, self.qids(), self.active.len(), self.prepared.is_some(), - self.quarantined, stamps, self.free_list.counters()); - self.memory.manager.report_recovery(); - for active in self.active.iter().take(4) { - if let Validated::Render { pass, .. } = &active.packet.command { - pr_warn!("G17: recovery render geometry ordinal={} width={} height={} layers={} layout={:?}\n", - active.ticket.ordinal, pass.width, pass.height, pass.layers, - target::Layout::new(pass, self.clusters, self.variant.programs)); - } - pr_warn!( - "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", - active.ticket.ordinal, - active.outer_published, - active.packet.context.id(), - active.packet.order.sequence, - observation::snapshot(&self.memory, active.ticket.ordinal) - ); - } - } - - /// Latches when the front group executed and the firmware consumed its ring entries while - /// `stamps` still lag it, and reports whether that has held for - /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it - /// on its pass record instead of waiting for stamps no later event will move. - pub(crate) fn note_stamp_wait( - &mut self, - stamps: [u32; 2], - state: u32, - now: u64, - ) -> Result { - if self.quarantined || self.prepared.is_some() || state != 0 { - return Ok(false); - } - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if stamps - .into_iter() - .all(|stamp| retirement::stamp_covers(stamp, published)) - { - front.stamp_wait_since = None; - return Ok(false); - } - let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; - if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { - front.stamp_wait_since = None; - return Ok(false); - } - let since = *front.stamp_wait_since.get_or_insert(now); - Ok(Self::stamp_wait_expired(front, since, now)) - } - - /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks - /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair - /// stays latched for the rescan. - fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { - let drained = || { - // A node that cannot be read keeps the bound from the latch. - front - .packet - .completion - .work_state() - .and_then(|work| work.drained()) - .unwrap_or(true) - }; - match retirement::stamp_wait(since, now, drained) { - retirement::StampWait::Waiting => false, - retirement::StampWait::Restart(now) => { - front.stamp_wait_since = Some(now); - false - } - retirement::StampWait::Expired => true, - } - } - - /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair - /// without a completion notification, which may never come for such a group. - pub(crate) fn stamp_waiting(&self) -> bool { - self.active - .front() - .is_some_and(|front| front.stamp_wait_since.is_some()) - } - - /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed - /// it and consumed its ring entries while the stamps lag, it retires on its pass record now - /// instead of failing. Failing would give up on the stamps as well. - pub(crate) fn expire_stamp_wait( - &mut self, - packet: &Arc, - stamps: [u32; 2], - state: u32, - ) -> Result { - if !self - .active - .front() - .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) - { - return Ok(false); - } - if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { - return Ok(false); - } - if let Some(front) = self.active.front_mut() { - front.stamp_wait_since = Some(0); - } - Ok(true) - } - fn selected(&self, masks: Option<[u64; 2]>) -> bool { - masks.is_none_or(|masks| completion_noted(self.qids(), masks)) - } - /// The oldest published group still running, for cross-VM timeout attribution - /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains - /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. - pub(crate) fn timeout_head(&self) -> Option { - self.active - .iter() - .filter(|active| active.outer_published) - .find(|active| active.drained_result.is_none()) - .map(|active| crate::g17::timeout::Head { - vm: active.packet.context.vm().id(), - published: active.published, - finished: false, - failed: self.quarantined, - }) + masks.is_none_or(|masks| { + self.qids() + .into_iter() + .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + }) } - pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -1202,7 +934,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) + self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -1223,10 +955,8 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory - .manager - .note_required(required, crate::g17::now_ns()); + let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1251,28 +981,6 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } - /// A recovery record names one of this pair's queues and the completion - /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's - /// stamps. Without a stamp match the oldest published group is marked. - pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { - let sequence = stamp >> 8; - if let Some(active) = self - .active - .iter() - .filter(|active| active.outer_published) - .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) - { - active.packet.completion.note_interrupted(); - return recovery::Interrupted::Exact; - } - match self.active.iter().find(|active| active.outer_published) { - Some(active) => { - active.packet.completion.note_interrupted(); - recovery::Interrupted::Oldest - } - None => recovery::Interrupted::None, - } - } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1289,10 +997,7 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self - .active - .iter() - .map(|active| active.ticket.ordinal) + let ordinal = self.active.iter().map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1304,7 +1009,6 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; - self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1324,11 +1028,7 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self - .active - .iter() - .any(|active| active.packet.completion.spared()) - { + if self.active.iter().any(|active| active.packet.completion.spared()) { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1384,272 +1084,44 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { - return Err(EIO); - } - let stampless = front - .stamp_wait_since - .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { + loop { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { return Err(EIO); } - if stampless { - retirement::poll_stampless(&mut front.ticket, &mut observer) - } else { + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { + return Err(EIO); + } retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } - } else if stampless { - retirement::poll_stampless(&mut front.ticket, &mut observer) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - if stampless { - retirement::note_stampless(u16::from(self.slot), ordinal); - } - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; - } - crate::g17::recovery::note_retired(); - crate::g17::trace::complete( - crate::g17::trace::RENDER, - qids[1], - ordinal as u32, - &front.packet, - [timestamps[0], timestamps[3]], - ); - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - Ok(true) - } - - /// Every retained group was retired and unlinked by firmware without - /// executing (job list empty, stamps cover it, no recovery, fragment end - /// never written): after another VM's recovery the firmware drops an - /// innocent pair's queued groups this way (seen on 25G83 and 26A428). - /// Nothing of them reached their attachments, so the same commands are - /// published again, in order, instead of failing their owners. Only the - /// whole set is replayed; a replayed prefix would run after a live - /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS - /// apart (worker polling); the render timeout, long after publication, - /// takes it once as its retirement does. A single sample can still - /// straddle a firmware update of the fragment end record; the cost is - /// that a group which did execute runs a second time (blending twice). - fn replay_abandoned( - &mut self, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - grace: bool, - ) -> Result { - use retirement::Memory; - if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { - return Ok(false); - } - if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { - return Ok(progressed); - } - use crate::g17::replay::{self, Decision, Group}; - let qids = self.qids(); - let memory = &self.memory; - let mut failure = None; - let groups = self.active.iter().map(|active| { - let innocent = - active.outer_published && !active.failed && !active.packet.completion.guilty(); - let blamed = active.packet.completion.blamed(); - let exhausted = - active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is still live, an error included. - let (abandoned, ran, lost) = if innocent && !blamed { - (|| -> Result<(bool, bool, bool)> { - let mut observer = Observer { - memory, - qids, - host: &mut *host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - // A torn sample is no proof; a later poll or the timeout looks again. - let witness = |executed| { - matches!( - retirement::abandoned( - &active.ticket, - snapshot, - stamps, - state, - executed - ), - Ok(true) - ) - }; - let unexecuted = witness(false); - // Interrupted work is run again only if it never started, and work that - // another queue waited on never: the wait ended when firmware retired it. - let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) - || active.packet.completion.has_firmware_consumer(); - Ok(( - unexecuted && !lost, - !unexecuted && witness(true), - unexecuted && lost, - )) - })() - .unwrap_or_else(|error: Error| { - failure = Some(error); - (false, false, false) - }) } else { - (false, false, false) + retirement::poll(&mut front.ticket, &mut observer) }; - Group { - innocent, - blamed, - abandoned, - exhausted, - ran, - lost, - } - }); - let decision = replay::decide(groups); - if let Some(error) = failure { - return Err(error); - } - let now = crate::g17::now_ns(); - let front = self.active.front_mut().ok_or(EIO)?; - match decision { - Decision::Refuse => return Ok(false), - Decision::Wait => { - front.abandoned_since = None; - return Ok(false); - } - Decision::Replay | Decision::Fail => {} - } - if grace { - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); - } - } - if decision == Decision::Fail { - return self.fail_dropped(host, defer); - } - // decide() lets only the front be exhausted. An exhausted front fails, and its - // successors, which consume its output, fail with it rather than run without it. - let mut failing = self.active.front().is_some_and(|front| { - front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS - }); - if failing { - pr_warn!( - "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", - self.slot, - self.active.len() - ); - } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_warn!( - "G17: render slot {}: {} groups retired without execution; replaying\n", - self.slot, - self.active.len() - ); - } - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - // Every group parked below shares this expiry (an early return keeps it). - self.replay_since = Some(now); - while let Some(front) = self.active.front_mut() { - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.flist_completed = true; - front.qos_completed = true; - } let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - failing = failing || !packet.completion.begin_replay(); - if failing { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); - } - } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - continue; - } - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.published_completions = None; - self.replay.push(packet).map_err(|_| EIO)?; - } - Ok(true) - } - - /// Firmware retired every group of the pair after a restart, but some were dropped - /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, - /// the ones that ran too: they consumed output the dropped ones never wrote. The work is - /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, - /// instead of waiting for the render timeout to fail it with ETIMEDOUT. - fn fail_dropped( - &mut self, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - pr_warn!( - "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", - self.slot, - self.active.len() - ); - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - while let Some(front) = self.active.front_mut() { + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { host.complete_render( (!front.qos_completed).then_some(qos), @@ -1657,73 +1129,74 @@ impl Pair { front.generation, buffer, )?; - front.flist_completed = true; front.qos_completed = true; + front.flist_completed = true; } - let ordinal = front.ticket.ordinal; - let completion = front.packet.completion.clone(); - completion.classify(recovery::Class::Spared); - defer(Deferred::Retired(completion, Err(ENODATA)))?; + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + return Ok(true); } - Ok(true) } - /// A front group a restart interrupted after its tiler started, which firmware - /// then retired without writing its fragment end, may have run partly. It is - /// not replayed (blending or storage writes could apply twice) and fails now, - /// reported to its owner, instead of waiting out the render timeout. Returns - /// None when the front is not such a group. - fn settle_interrupted_front( + /// The front group was retired and unlinked by firmware without executing + /// (job list empty, stamps cover it, no recovery, fragment end never + /// written). Nothing of this group reached its attachments, so the same + /// command is published again instead of failing the owner's VM. The + /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, + /// and only for a lone group whose completion still accepts a replay. + fn replay_abandoned( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, - grace: bool, - ) -> Result> { + ) -> Result { use retirement::Memory; - let qids = self.qids(); - let Some(front) = self.active.front() else { - return Ok(None); - }; - if !front.outer_published || front.failed || !front.packet.completion.interrupted() { - return Ok(None); + if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + return Ok(false); + } + let front = self.active.front().ok_or(EIO)?; + if !front.outer_published || front.failed { + return Ok(false); } - let ordinal = front.ticket.ordinal; - let ticket = front.ticket; let mut observer = Observer { memory: &self.memory, - qids, - host: &mut *host, + qids: self.qids(), + host, }; core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(ordinal)?; + let snapshot = observer.snapshot(front.ticket.ordinal)?; let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; - if snapshot.untouched() - || !matches!( - retirement::abandoned(&ticket, snapshot, stamps, state, false), - Ok(true) - ) - { - return Ok(None); - } + let abandoned = + retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if grace { - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(Some(false)); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(Some(false)); - } + if !abandoned { + front.abandoned_since = None; + return Ok(false); + } + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_info!( + "render slot {} retired without execution (pass {:?}); replaying\n", + self.slot, + snapshot.pass + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; host.complete_render( (!front.qos_completed).then_some(qos), pool, @@ -1733,32 +1206,39 @@ impl Pair { front.flist_completed = true; front.qos_completed = true; } - pr_warn!( - "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", - self.slot, - ordinal - ); - let packet = self.active.front().ok_or(EIO)?.packet.clone(); - defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + if !packet.completion.begin_replay() { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - Ok(Some(true)) + packet.reset_publication_for_replay(); + self.replay = Some(packet); + Ok(true) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). Its successors cannot run without it and fail with - /// it. ECANCELED leaves the VM usable; any other error is the ordinary - /// device-loss outcome the replay tried to avoid. + /// publication refusal). ECANCELED leaves the VM usable; any other error + /// is the ordinary device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - self.replay_since = None; - while let Some(packet) = self.replay.pop() { - defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; - } - Ok(()) + let Some(packet) = self.replay.take() else { + return Ok(()); + }; + defer(Deferred::Retired(packet.completion.clone(), Err(error))) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1786,44 +1266,7 @@ impl Pair { let counters = self.free_list.counters()?; let stamps = host.stamps(self.qids())?; let (_, state) = host.epoch()?; - Ok(retirement::drained(self.ordinal, stamps, counters, state) - || (state == 0 && self.executed_unstamped() && self.context_killed())) - } - - /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer - /// walks that context's work list, so the commands it never unlinked can be released. - fn context_killed(&self) -> bool { - crate::g17::queue::compute::exit::State::read(&self.memory.context) - .is_ok_and(|state| state.killed != 0) - } - - /// The pair's context may be killed although its commands were not retired: they all ran - /// (`executed_unstamped`), so the kill leaves nothing queued for the firmware to run. - /// Its memory stays until the kill is acknowledged. - pub(crate) fn executed_for_kill(&self, context: &Arc) -> bool { - Arc::ptr_eq(context, &self.memory.context) && self.executed_unstamped() - } - - /// Every command of this quarantined pair ran, its ring entries were consumed and its - /// free-list blocks returned, yet the firmware never stores the stamp of the last one. - /// On Neo this is the render work of a context whose compute command failed: it runs - /// about a second after the failure, but its stamp never arrives and it stays linked in - /// the firmware's job list, so neither the stamp proof nor the abandoned-group proof - /// ever holds and the pair would stay quarantined, keeping its context, until the - /// processors stop. - fn executed_unstamped(&self) -> bool { - self.quarantined - // Only for a closed owner: a live owner would go on publishing into a context - // whose work list still links the unstamped node. - && self.owner.is_none() - && self.prepared.is_none() - && self.active.len() != 0 - && self.free_list.counters().is_ok_and(|counters| counters[0] == counters[1]) - && self.active.iter().all(|active| { - active.outer_published - && observation::snapshot(&self.memory, active.ticket.ordinal) - .is_ok_and(retirement::Snapshot::executed_and_consumed) - }) + Ok(retirement::drained(self.ordinal, stamps, counters, state)) } /// Account one witnessed tracker, preserving all completion ownership. @@ -1837,10 +1280,6 @@ impl Pair { let qids = self.qids(); let qos = self.qos_owners(); let buffer = self.buffer_id(); - // Commands the firmware never unlinked are released only after it acknowledged the - // kill of their context, on every path (see `firmware_drained`). - let unstamped = self.executed_unstamped(); - let killed = unstamped && self.context_killed(); let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { return Ok(false); }; @@ -1851,12 +1290,6 @@ impl Pair { }; let result = match retirement::poll(&mut active.ticket, &mut observer)? { retirement::Poll::Complete(ticks) => Ok(ticks), - // Its commands ran without a stamp (see `executed_unstamped`): they already failed - // with the quarantine, and the client sees them cancelled. - retirement::Poll::Pending | retirement::Poll::Retry if killed => { - crate::g17::trace::teardown(2, u64::from(self.slot), active.ticket.ordinal, 0); - Err(ECANCELED) - } retirement::Poll::Pending | retirement::Poll::Retry => { core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); let snapshot = observer.snapshot(active.ticket.ordinal)?; @@ -1924,49 +1357,22 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - // Groups the firmware dropped unexecuted are published again, with their successors. - if self.replay_abandoned(host, defer, false)? { - return Ok(true); - } - // Fallback when a live or exhausted group prevents an ordered replay: a spared front group - // the firmware retired without executing during another VM's recovery is retired on its - // own even with successors queued, so the pair is not quarantined into a sticky EIO. It - // fails with ECANCELED, which spared work never reports as device loss; consecutive - // abandoned spared successors are retired the same way. - let front_spared = self - .active - .front() - .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); - if !self.quarantined && (self.active.len() == 1 || front_spared) { - let mut retired = false; - while let Some(active) = self.active.front() { - let spared = active.packet.completion.spared(); - if retired && !spared { - break; - } - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - // A torn sample is no proof, as in replay_abandoned(): the group stays retained - // (quarantined with the pair if nothing was retired) instead of the error - // stopping the runtime for every client. - if !active.outer_published - || !matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) - ) - { - break; - } + if !self.quarantined && self.active.len() == 1 { + let active = self.active.front().ok_or(EIO)?; + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if active.outer_published + && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? + { let qos = self.qos_owners(); let buffer = self.buffer_id(); - let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1978,30 +1384,19 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !spared { + if !active.failed && !active.packet.completion.spared() { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } - pr_warn!( - "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", - active.ticket.ordinal, - left - ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(if spared { ECANCELED } else { EIO }), + Err(EIO), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - retired = true; - if !spared { - break; - } - } - if retired { return Ok(true); } } @@ -2029,8 +1424,7 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - self.replay_since = None; - while let Some(packet) = self.replay.pop() { + if let Some(packet) = self.replay.take() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -2184,87 +1578,6 @@ impl Pair { } } } - /// The scheduler page these queues last ran a command for, when the next - /// publication for `context` changes the queues' owner. - /// The steady state in which a command may publish without the device - /// mutex: the graph is installed and registered at `priority`, it last ran - /// `context`'s scheduler, and nothing is replayed, settled, aliased or - /// switched on it. - pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { - self.graph_started - && self.installed == [true; 2] - && self.registered == Some(priority) - && priority < 8 - && self.outer_started & (1u8 << priority) != 0 - && self.executed == context.scheduler_va() - && self.switch.is_none() - && self.owner.is_some() - && !self.quarantined - && self.terminal.is_none() - && self.pending_aliases.is_none() - && self.kick_aliases.is_none() - && self.prepared.is_none() - && self.replay.len() == 0 - && self.displaced.is_none() - && self.active.room() - } - /// The render prefix of `packet` on this pair, as the registry collects it - /// for an owner whose only pair this is. - pub(crate) fn own_dependencies( - &self, - owner: u64, - packet: &Arc, - ) -> Result { - let mut dependencies = RenderDependencies::EMPTY; - let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut frontier, - false, - )?; - for dependency in frontier.as_slice() { - *dependencies - .render - .get_mut(dependencies.render_count) - .ok_or(EOVERFLOW)? = *dependency; - dependencies.render_count += 1; - } - if dependencies.render_count != 0 - && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) - { - let mut tiling = crate::g17::dependency::Frontier::new(); - self.collect_tiling_dependencies( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut tiling, - )?; - for dependency in tiling.as_slice() { - *dependencies - .tiling - .get_mut(dependencies.tiling_count) - .ok_or(EOVERFLOW)? = *dependency; - dependencies.tiling_count += 1; - } - } - Ok(dependencies) - } - pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { - (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) - } - /// The previous scheduler page of an owner change whose first command has - /// not left the pair yet. - pub(crate) fn switch_in_flight(&self) -> Option { - let (from, first) = self.switch?; - (self.replay.len() != 0 - || self - .active - .iter() - .any(|active| active.ticket.ordinal == first)) - .then_some(from) - } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -2308,10 +1621,6 @@ impl Pair { self.owner = None; } #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Binding keeps the render owners and first packet explicit." - )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -2345,7 +1654,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = target::Layout::new(pass, self.clusters)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -2361,7 +1670,6 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), - self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -2428,9 +1736,6 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { - self.graph_started = false; - } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -2439,45 +1744,3 @@ impl Pair { Ok(()) } } - -/// Whether `masks`, the stamp notifications collected for a pair whose tiling and fragment -/// queues are `qids`, can mean a finished pass: the fragment queue's bit. A fragment kick starts -/// only after its fragment barrier saw the tiling stamp, and retirement still checks both stamps -/// in memory, so the tiling notification is not needed to retire. -pub(crate) fn completion_noted(qids: [u8; 2], masks: [u64; 2]) -> bool { - let fragment = usize::from(qids[1]); - masks[fragment / 64] & (1u64 << (fragment % 64)) != 0 -} - -#[kunit_tests(asahi_g17_render_completion_noted)] -mod completion_noted_tests { - use super::completion_noted; - - fn bit(qid: u8) -> [u64; 2] { - let mut masks = [0; 2]; - masks[usize::from(qid) / 64] |= 1u64 << (qid % 64); - masks - } - - #[test] - fn fragment_notification_alone_selects() { - assert!(completion_noted([3, 4], bit(4))); - assert!(completion_noted([70, 100], bit(100))); - } - - #[test] - fn tiling_notification_alone_does_not_select() { - assert!(!completion_noted([3, 4], bit(3))); - assert!(!completion_noted([70, 100], bit(70))); - assert!(!completion_noted([3, 4], [0; 2])); - } - - #[test] - fn other_queues_do_not_select() { - assert!(!completion_noted([3, 4], bit(5))); - assert!(!completion_noted([3, 68], bit(4))); - let mut both = bit(3); - both[1] |= bit(68)[1]; - assert!(completion_noted([3, 68], both)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 90a11ca977684..85319585b3d13 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,9 +31,6 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; -/// The lead command of a stage (InitBM before tiling), after the stage's configuration record -/// in its slot ([`crate::g17::fw::Variant::lead_commands`]). -pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -97,8 +94,6 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], - /// Host words of the pointer blocks in this firmware build. - pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -106,7 +101,6 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], - pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -160,7 +154,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(&pointers)?; + *view = queue::PointerBlock::new(); Ok(()) })?; } @@ -172,7 +166,6 @@ impl Graph { config, support, owners: [None; RECORDS], - pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -195,13 +188,11 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { - let Some(target) = self.pointers.recovery_target else { - return Ok(()); - }; for at in POINTERS { - let producer = self.queues.word(at + self.pointers.producer)? + let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? .load(Ordering::Relaxed); - self.queues.word(at + target)?.store(producer, Ordering::Relaxed); + self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); } Ok(()) } @@ -217,7 +208,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), + write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), }; } Ok(values) @@ -319,36 +310,24 @@ impl Generation { ) .map_err(Into::into) } - /// With `viewport` false the deflake buffer has no room for the viewport words - /// ([`abi::Deflake::viewport`]). - pub(super) fn prepare( - &mut self, - ordinal: u64, - pass: &abi::RenderPass, - viewport: bool, - ) -> Result { + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = match viewport { - true => Some(storage::Viewport::new(pass.width, pass.height)?), - false => None, - }; + let viewport = storage::Viewport::new(pass.width, pass.height)?; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - if let Some(viewport) = viewport { - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); - } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -386,10 +365,6 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Render memory construction keeps each allocation and firmware input explicit." - )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -401,11 +376,10 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, - variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; + let mut graph = Graph::new(alloc, pool, &context, qids)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -528,44 +502,12 @@ impl Memory { } const _: () = { - // Every unretired render of a pair holds its own command record. - assert!(DEPTH <= RECORDS); assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); - assert!(size_of::() <= LEAD); - assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { - /// Requests the tiling descriptor slot and both queue configuration records of - /// `ordinal` for writing. The firmware accessed them for the command that used them - /// last, so the host no longer holds their lines; requesting them before the rest of - /// the preparation overlaps those transfers with it instead of stalling the writes one - /// line at a time. - pub(super) fn prefetch_descriptors(&self, ordinal: u64) { - let record = ordinal as usize % RECORDS; - let config = size_of::(); - let ranges = [ - (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), - (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), - (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), - ]; - for (object, offset, size) in ranges { - let Ok(base) = object.pointer(offset, size) else { - continue; - }; - for line in (0..size).step_by(64) { - // SAFETY: A prefetch hint inside the retained mapping; it has no - // architectural effect on memory. - unsafe { - core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), - options(nostack, preserves_flags, readonly)); - } - } - } - } - pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -600,10 +542,21 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - // Kick entries bind the arrays embedded in this descriptor - // (register_bindings); the client register page is never read. - for array in &fragment.descriptor.arrays { - array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; + self.registers.bytes_mut().fill(0); + for (index, array) in fragment.descriptor.arrays.iter().enumerate() { + for (entry, value) in array + .entries + .get(..(array.size & 0xffff) as usize) + .ok_or(EINVAL)? + .iter() + .enumerate() + { + self.registers.write( + index * size_of::() + + entry * size_of::(), + *value, + )?; + } } fragment.set_mcache( args.fragment.descriptor_va, @@ -645,33 +598,6 @@ impl Memory { } Ok(()) } - - /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the - /// tiling descriptor when `init_manager`. Returns which stages have one. - pub(super) fn write_leads(&mut self, ordinal: u64, init_manager: bool) -> Result<[bool; 2]> { - if !self.owns(ordinal) { - return Err(EIO); - } - let ta = self - .graph - .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? - .cast::(); - // SAFETY: The view was checked during preparation and the ordinal still owns it; only - // integer fields are read. - let ta = unsafe { &*ta }; - let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; - if init_manager { - let init = queue::InitBufferManager::new( - ta.context_id, - ta.pb_slot, - ta.pb_va, - ta.stamp_value, - ); - self.graph.config.write(CONFIG[0] + slot, init)?; - } - Ok([init_manager, false]) - } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..39941de815e07 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,19 +25,12 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, - /// Scheduler page these queues last ran a command for. A pair parked - /// without a consumed release (an unused warm pair taken for another - /// owner) keeps it, so its next owner change still waits while that - /// scheduler is released. - executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -55,20 +48,11 @@ pub(crate) struct Prepared { } impl Pair { - /// The current graph never entered command publication. Physical identities - /// may have older users; parking retains their rings and retirement owners. - pub(crate) fn can_yield_unused(&self) -> Result { - Ok(!self.graph_started && self.can_park_backing(self.context())?) - } - /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { - Ok(self.owner.is_none() && self.can_park_backing(context)?) - } - - fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) + && self.owner.is_none() && self .previous .as_ref() @@ -92,15 +76,12 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, - deflake: self.deflake, descriptor_flags: self.descriptor_flags, - variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, - executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -115,22 +96,11 @@ impl Pair { } impl ParkedPair { - /// The release of `scheduler` has been consumed: a later owner change no - /// longer has to wait for it. - pub(crate) fn release_consumed(&mut self, scheduler: u64) { - if self.executed == scheduler { - self.executed = 0; - } - } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Preparation forwards the render binding inputs without changing their ownership." - )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -155,7 +125,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = target::Layout::new(pass, self.clusters)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -171,7 +141,6 @@ impl ParkedPair { pm_generation, self.ordinal, word, - self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -221,7 +190,6 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { - graph_started: false, kick_aliases: None, memory, kicks, @@ -234,9 +202,7 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, - deflake: self.deflake, descriptor_flags: self.descriptor_flags, - variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -247,12 +213,8 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: Tracker::new(), - replay_since: None, + replay: None, context_update: true, - executed: self.executed, - switch: None, - published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index d65c40c7cfade..0fc9c1d9a15dc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { +pub(super) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 @@ -28,9 +28,9 @@ pub(in crate::g17::queue) fn required(previous: Option, config: Que } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 4], + items: [&'a AtomicU64; 3], producer: &'a AtomicU32, - values: [u64; 4], + values: [u64; 3], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -41,10 +41,6 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { - #[expect( - clippy::too_many_arguments, - reason = "The publication lane describes independent fields of the firmware wire record." - )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -53,9 +49,8 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, - lead: bool, ) -> Result { - let count = 2 + u32::from(present) + u32::from(lead); + let count = 2 + u32::from(present); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -77,30 +72,21 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, - graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - // Item order: [lead command], descriptor, [configuration], announcement. - let mut values = [0; 4]; - let mut n = 0; - let mut push = |value: u64| { - values[n] = value; - n += 1; + let values = if present { + [ + descriptor, + graph.config.gpu_va() + config_offset as u64, + announce_va, + ] + } else { + [descriptor, announce_va, 0] }; - if lead { - push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); - } - push(descriptor); - if present { - push(graph.config.gpu_va() + config_offset as u64); - } - push(announce_va); Ok(Self { items, - producer: graph - .queues - .word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, values, config, config_destination, @@ -130,7 +116,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = core::ptr::from_ref(&self.config).cast::(); + let source = (&self.config as *const QueueConfig).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 4377bbfe3e311..d0a2d566bdc8d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,17 +122,6 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } - - /// Neither the tiler start nor the fragment end was written: nothing ran. - pub(crate) fn untouched(self) -> bool { - self.pass == [0, 0] - } - - /// The command ran (tiler start and fragment end written) and the firmware consumed both - /// queues' ring entries. - pub(crate) fn executed_and_consumed(self) -> bool { - self.executed() && self.cursors.iter().all(|cursors| cursors.done == cursors.write) - } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -155,85 +144,6 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result bool) -> StampWait { - if now.saturating_sub(since) < STAMP_FALLBACK_NS { - StampWait::Waiting - } else if since == 0 || drained() { - StampWait::Expired - } else { - StampWait::Restart(now) - } -} - -static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -/// Counts a render group that retired on its pass record because its completion stamps never -/// covered it; logs the first and every 64th. -pub(crate) fn note_stampless(slot: u16, ordinal: u64) { - let count = STAMPLESS.fetch_add(1, Ordering::Relaxed) + 1; - if count == 1 || count % 64 == 0 { - pr_info!( - "G17: lifetime total {} render groups retired on their pass record after {} ms without completion stamps (latest: slot {} ordinal {})\n", - count, - STAMP_FALLBACK_NS / 1_000_000, - slot, - ordinal - ); - } -} - -/// Retire an executed, consumed front group without its stamps, once [`STAMP_FALLBACK_NS`] -/// has passed. The caller decides that bound; this re-checks execution and consumption. Unlike -/// the stamped path, it has no post-maintenance evidence; see [`STAMP_FALLBACK_NS`]. -pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { - fence(Ordering::Acquire); - let snapshot = memory.snapshot(ticket.ordinal)?; - let Ok(observation) = observe(ticket, snapshot) else { - return Ok(Poll::Retry); - }; - if !observation.complete { - return Ok(Poll::Pending); - } - let [start, end] = memory.pass(ticket.ordinal)?; - Ok(Poll::Complete([start, start, start, end])) -} - /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -293,12 +203,8 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - match observe(ticket, snapshot) { - Ok(observation) => complete_poll(ticket, memory, snapshot, observation), - // The borrowed snapshot can straddle a firmware cursor update like any other sample: - // take the ordinary poll's fresh snapshot and reread instead of failing the client. - Err(_) => poll(ticket, memory), - } + let observation = observe(ticket, snapshot)?; + complete_poll(ticket, memory, snapshot, observation) } fn complete_poll( @@ -440,94 +346,3 @@ impl Tracker { .filter_map(Option::as_mut) } } - -#[kunit_tests(asahi_g17_render_retirement)] -mod executed_tests { - use super::*; - - fn snapshot(done: u32, write: u32, pass: [u64; 2]) -> Snapshot { - Snapshot { - cursors: [Cursors { done, read: done, write }; 2], - job_list_empty: false, - pass, - } - } - - #[test] - fn executed_and_consumed_needs_both_passes_and_consumed_rings() { - assert!(snapshot(3, 3, [1, 2]).executed_and_consumed()); - assert!(!snapshot(3, 3, [0, 0]).executed_and_consumed()); - assert!(!snapshot(3, 3, [1, 0]).executed_and_consumed()); - assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); - assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); - } - - /// The firmware state of a stalled pair as read on hardware: the front group executed and - /// its two ring entries per queue were consumed, but both stamps hold the previous kick. - struct Stalled { - snapshot: Snapshot, - stamps: [u32; 2], - } - impl Memory for Stalled { - fn snapshot(&mut self, _ordinal: u64) -> Result { - Ok(self.snapshot) - } - fn stamps(&mut self) -> Result<[u32; 2]> { - Ok(self.stamps) - } - fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { - Ok(self.snapshot.pass) - } - } - - fn stalled_ticket() -> Ticket { - let window = Window::new(1097, 2, 1280).unwrap(); - let previous = Cursors { done: 1097, read: 1097, write: 1097 }; - Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } - } - - fn stalled(pass: [u64; 2]) -> Stalled { - let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; - Stalled { - snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, - stamps: [406 << 8; 2], - } - } - - #[test] - fn lagging_stamps_keep_an_executed_group_pending() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 20]); - assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); - } - - #[test] - fn stampless_poll_retires_an_executed_consumed_group() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 20]); - assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); - } - - #[test] - fn stampless_poll_waits_for_the_fragment_end() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 0]); - assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); - } - - #[test] - fn stamps_wait_for_the_rest_of_the_ioctl() { - let since = 1_000_000; - let late = since + STAMP_FALLBACK_NS; - // The node is asked only once the bound passed. - assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); - // Other kicks of the group's ioctl still run: the bound restarts. - assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); - assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); - // A drained node with lagging stamps is the stall the bound exists for. - assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); - // The watchdog's last resort does not wait for the node. - assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), - StampWait::Expired); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index da90488bfedec..4db471f254ec6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,11 +36,10 @@ impl Pair { } *block = pointer.cast(); } - let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(&layout, 0)?; + unsafe { &*block }.rebase(0); } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ca8da8b03545f..ba368341d84d2 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,13 +9,11 @@ mod attribution; mod slot; -mod safety; -pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { - slot::key(slot, slot_keys?).ok() +pub(super) fn slot_key(slot: usize) -> Option { + slot::key(slot).ok() } use super::status::VmStatus; @@ -25,34 +23,20 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the -/// work it interrupted has drained; with several clients running long ray-traversal jobs that -/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest -/// execution allowance (`timeout::POLL_MS` plus renewals). -const RECOVERY_POLLS: usize = 10_000; +const RECOVERY_POLLS: usize = 500; -/// Records in the firmware recovery table. -pub(super) const RECOVERY_RECORDS: usize = 256; - -/// Per-QID classifications from the firmware recovery table. +/// Per-QID classifications from the complete firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, - /// Valid records, each naming one command that was in flight at the restart. - pub(super) listed: u32, - /// Listed commands found by their exact stamp, and by the oldest-work fallback. - pub(super) exact: u32, - pub(super) fallback: u32, } impl Entries { - /// Returns false at the first invalid record: the table is a packed list and - /// records after it are left over from an earlier restart. - pub(super) fn include(&mut self, flags: u32) -> bool { + pub(super) fn include(&mut self, flags: u32) { if flags & 1 == 0 { - return false; + return; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -62,22 +46,9 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } - self.listed += 1; - true } } -/// How a listed recovery record was matched to retained work. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum Interrupted { - /// No retained command of the queue. - None, - /// The command whose completion stamp equals the record's. - Exact, - /// No stamp matched; the queue's oldest unretired command was marked instead. - Oldest, -} - /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -99,12 +70,6 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; - /// Mark the command a recovery record lists as interrupted. It may have - /// partly executed, so it must never report success; see - /// `Completion::note_interrupted`. - fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; - /// Optional diagnostics only; absence cannot alter attribution or release. - fn report_blamed_render(&self, _qid: u8) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -119,9 +84,7 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - /// One recovery record's flags and completion stamp, read while firmware - /// is halted in state 3. - fn entry(&self, index: usize) -> Result<(u32, u32)>; + fn entries(&self) -> Result; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -136,8 +99,6 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, - timeouts: safety::Storm, - restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -151,14 +112,8 @@ impl State { Self { accepted: 0, pending: None, - timeouts: safety::Storm::new(), - restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { - self.timeouts.note_unless_progress(now, safety::retired()) - } - pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -230,8 +185,6 @@ pub(super) fn service( } } if !ready { - pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", - request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -243,15 +196,6 @@ pub(super) fn service( host.clear_timestamps()?; } let mut sources = host.fault_sources()?; - sources.report(request.generation, super::queue::compute::retirement::stampless_total()); - if let Some(qid) = request.blamed { - host.report_blamed_render(qid); - } - // The keys, monitor and pass-start evidence must describe the same halted - // instant. Firmware may update pass timestamps as soon as state 3 is - // acknowledged; sampling them during classification can blame a bystander. - let attribution = sources.attribute(request.blamed, - |qid, stamp| host.render_pass_started(qid, stamp)); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; @@ -266,41 +210,19 @@ pub(super) fn service( host.wait_tick(); } if !recovered { - pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", - request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - // Every listed command was in flight when firmware stopped. Firmware may - // still retire it with stamps and records that look like success, so it is - // marked before firmware resumes and never reports success. - let entries = read_entries(host); + let entries = host.entries().unwrap_or_default(); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); - let (cancelled, blame) = classify(host, entries, request.blamed, attribution); - if listed != 0 { - pr_warn!( - "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", - listed, - exact, - fallback - ); - } + let cancelled = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { - pr_warn!( - "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", - sources.reason, - qid, - if blame { "failing its oldest render" } else { "attribution cleared it; spared" } - ); - if blame { - host.settle_blamed_render(qid, EIO); - } + host.settle_blamed_render(qid, EIO); } if cancelled != 0 { host.settle_cancelled_renders(cancelled); @@ -310,44 +232,21 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } -/// Walks the packed recovery table up to its first invalid record and marks -/// every listed command interrupted. A failed read keeps what was read so far. -fn read_entries(host: &mut H) -> Entries { - let mut entries = Entries::default(); - for index in 0..RECOVERY_RECORDS { - let Ok((flags, stamp)) = host.entry(index) else { - break; - }; - if !entries.include(flags) { - break; - } - match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { - Interrupted::Exact => entries.exact += 1, - Interrupted::Oldest => entries.fallback += 1, - Interrupted::None => {} - } - } - entries -} - /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed -/// render that had not started while others had (or with no culprit at all) is innocent, and its -/// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; + let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} Attribution::Queues(qids) => guilty_qids = qids, Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; - let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return (0, blame); + return 0; } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -383,7 +282,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return (0, blame); + return 0; } let spared = queues.classify_work(&|status| { guilty[..count] @@ -396,103 +295,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att }) .unwrap_or(Class::Spared) }); - (cancelled & !(guilty_qids & !collateral) & spared, blame) -} - - -#[kernel::macros::kunit_tests(asahi_g17_recovery_snapshot)] -mod tests { - use super::*; - use core::cell::Cell; - - struct Halted { - recovery: State, - state: Cell, - reads: Cell, - known_keys: bool, - blamed: bool, - } - - impl Halted { - fn new(known_keys: bool) -> Self { - let mut recovery = State::new(); - assert!(recovery.request(0, 38).unwrap()); - Self { recovery, state: Cell::new(1), reads: Cell::new(0), - known_keys, blamed: false } - } - } - - impl Memory for Halted { - fn recovery_state(&self) -> Result { Ok(self.state.get()) } - fn host_recovery(&self) -> Result { Ok(0) } - fn fault_sources(&self) -> Result { - assert_eq!(self.state.get(), 1); - let mut sources = Sources::sample(3, |slot| { - Some((if self.known_keys { ((38 + slot as u64) << 40) | 1 } else { 0 }, 0)) - }); - sources.reason = Some(2); - Ok(sources) - } - fn transition(&self, expected: u32, next: u32) -> Result { - assert_eq!(self.state.get(), expected); - self.state.set(next); - Ok(()) - } - fn clear_timestamps(&self) -> Result { panic!("active render scheduler") } - fn entry(&self, _index: usize) -> Result<(u32, u32)> { - assert_eq!(self.state.get(), 3); - Ok((0, 0)) - } - fn wait_tick(&self) { - assert_eq!(self.state.get(), 2); - self.state.set(3); - } - } - - impl Queues for Halted { - fn render_scheduler_active(&self) -> bool { true } - fn render_pass_started(&self, qid: u8, _stamp: u64) -> Option { - // This is the regression oracle: state 0/2/3 is too late. A - // resumed peer may have started after the fault we are attributing. - assert_eq!(self.state.get(), 1); - self.reads.set(self.reads.get() + 1); - Some(qid == 39) - } - fn publish_recovery_targets(&self) -> Option { None } - fn oldest_spared(&self, _qid: u8) -> Option { None } - fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} - fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } - fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } - fn settle_blamed_render(&mut self, qid: u8, _error: Error) { - assert_eq!(qid, 38); - self.blamed = true; - } - fn settle_cancelled_renders(&mut self, _qids: u128) {} - fn rescan_compute(&mut self) -> Result { Ok(()) } - fn release_retired_compute(&mut self) {} - } - - impl Host for Halted { - fn recovery(&self) -> &State { &self.recovery } - fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } - } - - #[test] - fn pass_evidence_is_frozen_before_the_first_acknowledgement() { - let mut host = Halted::new(true); - service(&mut host, || false).unwrap(); - assert_eq!(host.reads.get(), 2); - assert_eq!(host.state.get(), 0); - assert_eq!(host.recovery.generation(), 1); - assert!(!host.blamed); - } - - #[test] - fn unknown_25g83_keys_still_preserve_firmware_blame() { - let mut host = Halted::new(false); - service(&mut host, || false).unwrap(); - assert_eq!(host.reads.get(), 0); - assert!(host.blamed); - assert_eq!(host.state.get(), 0); - } + cancelled & !(guilty_qids & !collateral) & spared } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs deleted file mode 100644 index d71d3cc81553f..0000000000000 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Fail closed on repeated watchdog/restart traffic, even across client loss. - -use core::sync::atomic::{AtomicU64, Ordering}; - -const WINDOW_NS: u64 = 30_000_000_000; -const LIMIT: u8 = 8; - -/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; -/// timeouts of overloaded clients while other work keeps retiring are not one. -static RETIRED: AtomicU64 = AtomicU64::new(0); -/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. -static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); - -/// Called for every command that retires without error. -pub(crate) fn note_retired() { - RETIRED.fetch_add(1, Ordering::Relaxed); - LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); -} - -pub(super) fn retired() -> u64 { - RETIRED.load(Ordering::Relaxed) -} - -/// Nanoseconds since any command last retired successfully, or `None` before the first. -pub(crate) fn since_retired(now: u64) -> Option { - match LAST_RETIRED_NS.load(Ordering::Relaxed) { - 0 => None, - last => Some(now.saturating_sub(last)), - } -} - -pub(super) struct Storm { - start: Option, - count: u8, - stopped: bool, - progress: u64, -} -impl Storm { - pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false, progress: 0 } - } - /// Like [`Self::note`], but work retired since the previous event starts a new window. - pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { - if !self.stopped && progress != self.progress { - self.progress = progress; - self.start = None; - } - self.note(now) - } - pub(super) fn note(&mut self, now: u64) -> bool { - if self.stopped { return true; } - if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { - self.start = Some(now); - self.count = 0; - } - self.count += 1; - self.stopped = self.count >= LIMIT; - self.stopped - } -} - -#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] -mod tests { - use super::*; - use crate::g17::{preparation::Gate, recovery::State}; - use kernel::prelude::*; - - #[test] - fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { - let gate = Gate::new().unwrap(); - let lease = gate.try_enter().unwrap(); - let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); - gate.close(); - for n in 0..64 { - let stopped = { - let mut state = state.lock(); - state.request(0, -1).unwrap(); - assert!(state.pending()); - let stopped = state.timeout_limit(n * 2_000_000_000); - if stopped { gate.fail(); } // production state -> gate order - stopped - }; - assert_eq!(stopped, n >= u64::from(LIMIT - 1)); - } - // Recovery/stop must release state before joining the retained builder. - assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), - Err(error) if error == ETIMEDOUT)); - assert!(!gate.reopen()); // failure never gives back an owned reservation - drop(lease); - gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); - assert!(!gate.reopen()); - } - #[test] - fn successful_handshakes_do_not_reset_a_restart_storm() { - let mut state = State::new(); - for n in 0..64 { - assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); - } - assert!(state.restart_limit(u64::MAX)); - } - #[test] - fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { - let mut storm = Storm::new(); - for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } - for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } - assert!(storm.note(64 * WINDOW_NS)); - for _ in 0..1024 { assert!(storm.note(u64::MAX)); } - } - #[test] - fn recovery_table_ends_at_its_first_invalid_record() { - use crate::g17::recovery::Entries; - let mut entries = Entries::default(); - assert!(entries.include(1 | (12 << 4))); - assert!(entries.include(1 | 2 | (17 << 4))); - assert!(!entries.include(0)); - assert_eq!(entries.listed, 2); - assert_eq!(entries.guilty, 1u128 << 17); - assert_eq!(entries.cancelled, 0); - } - #[test] - fn replay_dependency_checks_never_wait_for_event_service() { - use crate::g17::dependency::replay_status; - assert_eq!(replay_status(0), Err(EBUSY)); - assert_eq!(replay_status(1), Ok(())); - assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); - assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); - assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index bd9d4c96be14f..307534708e922 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::mmu; +use crate::{hw::t8140, mmu}; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize, slot_keys: u64) -> Result { +pub(super) fn key(slot: usize) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = slot_keys + slot as u64 * size_of::() as u64; + let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize, slot_keys: u64) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as usize, + bindings::MEMREMAP_WB as _, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs deleted file mode 100644 index 7668871376bad..0000000000000 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Whether a pair's retained render groups, retired by firmware without executing, are -//! published again. Only the whole set is replayed, so no replayed group runs after a live -//! successor, and work a recovery blamed is never run again. - -/// One retained group, front first. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Group { - /// Published, not failed and not classified guilty by a recovery. - pub(crate) innocent: bool, - /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): - /// it may have caused the fault. - pub(crate) blamed: bool, - /// Carries the strict abandonment witness. - pub(crate) abandoned: bool, - /// Already republished the maximum number of times. - pub(crate) exhausted: bool, - /// Retired by firmware after executing in full. - pub(crate) ran: bool, - /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after - /// its tiler started, so it may have run partly, or a command of another queue waited on it, - /// and that wait ended when firmware retired the group, before its output existed. - pub(crate) lost: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Decision { - Replay, - /// A group may still execute: look again later. - Wait, - /// Never replay this set; the render timeout settles it. - Refuse, - /// Firmware has retired every group, but a dropped group cannot run again in order: a later - /// group already ran, or the dropped group had started or had a consumer. Fail the set now. - Fail, -} - -/// Groups are read in order and the first one that decides stops the walk, so a caller can -/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its -/// successors, which consume its output, with it. An exhausted successor would break the order. -/// -/// After a restart the firmware retires the groups it had started on a pair without running -/// them, while groups it had not yet started run as usual. Once such a later group has run, a -/// replay of the dropped ones would run them after it; a group that had started cannot run -/// twice; and a group another queue waited on may already have been read before it ran. None of -/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A -/// group that ran ahead of every dropped one is left to in-order retirement. -pub(crate) fn decide(groups: impl Iterator) -> Decision { - let mut front_exhausted = None; - let mut successor_exhausted = false; - let mut dropped = false; - let mut unordered = false; - for group in groups { - if !group.innocent || group.blamed { - return Decision::Refuse; - } - if group.ran { - if !dropped { - return Decision::Wait; - } - unordered = true; - continue; - } - if group.lost { - dropped = true; - unordered = true; - continue; - } - if !group.abandoned { - return Decision::Wait; - } - dropped = true; - match front_exhausted { - None => front_exhausted = Some(group.exhausted), - Some(_) => successor_exhausted |= group.exhausted, - } - } - if unordered { - return Decision::Fail; - } - match front_exhausted { - Some(true) => Decision::Replay, - Some(false) if !successor_exhausted => Decision::Replay, - _ => Decision::Refuse, - } -} - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] -mod tests { - use super::*; - fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } - } - const OK: Group = - Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; - const BLAMED: Group = Group { blamed: true, ..OK }; - const RAN: Group = Group { abandoned: false, ran: true, ..OK }; - const LOST: Group = Group { abandoned: false, lost: true, ..OK }; - #[test] - fn every_group_abandoned_and_innocent_is_replayed() { - assert_eq!(decide([OK].into_iter()), Decision::Replay); - assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); - } - #[test] - fn nothing_retained_is_never_replayed() { - assert_eq!(decide([].into_iter()), Decision::Refuse); - } - #[test] - fn guilty_work_is_never_replayed() { - assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); - assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); - } - #[test] - fn blamed_work_is_never_replayed_even_when_spared() { - // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. - assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); - assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); - assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); - // Also once the replay limit is reached, where an innocent front alone is replayed. - let exhausted = Group { exhausted: true, ..BLAMED }; - assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); - } - #[test] - fn a_live_group_waits_and_stops_the_walk() { - assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); - // Nothing after the live group is looked at, guilty or not. - assert_eq!( - decide([group(true, false, false), group(false, true, false)].into_iter()), - Decision::Wait - ); - let mut seen = 0; - let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); - assert_eq!(decide(groups), Decision::Wait); - assert_eq!(seen, 1); - } - #[test] - fn an_exhausted_successor_refuses_an_exhausted_front_replays() { - assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); - assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); - assert_eq!( - decide([group(true, true, true), group(true, true, true)].into_iter()), - Decision::Replay - ); - } - #[test] - fn a_group_that_ran_after_dropped_ones_fails_the_set() { - // Seen after restarts: the started groups were dropped, the queued ones ran. - assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); - assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); - // Also with an exhausted front, which a replay would only fail later. - assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); - } - #[test] - fn a_group_that_ran_first_is_left_to_retirement() { - assert_eq!(decide([RAN].into_iter()), Decision::Wait); - assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); - } - #[test] - fn a_started_dropped_group_fails_the_set() { - assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); - assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); - } - #[test] - fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { - assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); - assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); - assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d391fbb3001bb..ffaff6d80dbf9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,38 +25,14 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; -use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; +const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; -/// Pending admission keeps exactly retired compute backing available for reuse. -/// The guard spans retries and disappears on publication, cancellation or error. -/// It borrows Shared, so the accounting cannot retain the device after teardown. -struct ComputeDemand<'a>(&'a Arc); - -impl<'a> ComputeDemand<'a> { - fn new(shared: &'a Arc) -> Self { - // Each admitted logical queue has at most one scheduler publisher. - // The execution-context bound is far below the counter's range. - shared.compute_demand.fetch_add(1, Ordering::AcqRel); - Self(shared) - } -} - -impl Drop for ComputeDemand<'_> { - fn drop(&mut self) { - if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { - // Reinstate normal idle retention even if the last publisher was - // cancelled before a kick could produce another event. - self.0.queue_events(); - } - } -} - /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -68,103 +44,13 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: ComputeSlot, -} - -/// A physical compute queue behind its own lock. Publication into a bound queue takes only -/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> -/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is -/// then empty, so the cached reference never extends any firmware-visible lifetime. -#[pin_data] -pub(super) struct ComputeCell { - #[pin] - queue: kernel::sync::Mutex>>, -} - -impl ComputeCell { - fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), - }), - GFP_KERNEL, - ) - } - - /// The installed queue, locked; None once the registry gave it up. - pub(super) fn lock(&self) -> Option> { - QueueGuard::new(self.queue.lock()) - } -} - -/// A locked installed queue. -pub(super) struct QueueGuard<'a> { - queue: core::ptr::NonNull, - _guard: kernel::sync::MutexGuard<'a, Option>>, -} - -impl<'a> QueueGuard<'a> { - fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { - let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); - Some(Self { queue, _guard: guard }) - } -} - -impl core::ops::Deref for QueueGuard<'_> { - type Target = compute::Queue; - fn deref(&self) -> &compute::Queue { - // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. - unsafe { self.queue.as_ref() } - } -} - -impl core::ops::DerefMut for QueueGuard<'_> { - fn deref_mut(&mut self) -> &mut compute::Queue { - // SAFETY: As for `deref`; the lock makes this the only reference. - unsafe { self.queue.as_mut() } - } -} - -/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a -/// cached cell never keeps a removed queue (or its QID's aliases) alive. -struct ComputeSlot(Arc); - -impl ComputeSlot { - fn new() -> Result { - Ok(Self(ComputeCell::new()?)) - } - fn as_deref(&self) -> Option> { - self.0.lock() - } - fn as_deref_mut(&mut self) -> Option> { - self.0.lock() - } - fn is_some(&self) -> bool { - self.0.queue.lock().is_some() - } - fn is_none(&self) -> bool { - !self.is_some() - } - /// Gives the queue up to the caller, which drops it outside the cell lock. - fn take(&mut self) -> Option> { - self.0.queue.lock().take() - } - fn put(&mut self, queue: Option>) { - let old = core::mem::replace(&mut *self.0.queue.lock(), queue); - drop(old); - } -} - -impl Drop for ComputeSlot { - fn drop(&mut self) { - let queue = self.0.queue.lock().take(); - drop(queue); - } + queue: Option>, } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, + pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -175,10 +61,6 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, - /// Completion-ring slot where the next compute scan starts. - completion_cursor: usize, - /// Control word of each completion-ring slot as of the last scan that read it. - completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -186,35 +68,22 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. - #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new( clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, - ) -> impl Init { - Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) - } - - /// `queues` wakes logical-queue creation when a logical queue slot is returned; - /// `renders` wakes render publication when a render pair or lease returns. - pub(super) fn with_waiters( - clusters: u32, - deflake: crate::g17::fw::render::Deflake, - descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, - queues: Option>, - renders: Option>, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { - qids.reserve_initial_render() + qids <- kick::QueueIds::new().chain(|qids| { + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(()) }), buffers <- BufferIds::new(), + accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, - render: render::RenderState::new(clusters, deflake, descriptor_flags)?, + admission: admission::Pool::new()?, + render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -227,21 +96,12 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, - completion_cursor: 0, - completion_seen: { - let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; - let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; - for _ in 0..slots { - seen.push_within_capacity(0).map_err(|_| ENOMEM)?; - } - seen - }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option> { + pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { self.compute .iter() .flatten() @@ -249,24 +109,19 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { - self.compute(owner) - } - - /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. - pub(super) fn compute_cell(&self, owner: u64) -> Option> { - self.compute.iter().flatten().find_map(|entry| { - entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; - Some(entry.queue.0.clone()) - }) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) } /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -274,102 +129,23 @@ impl Registry { Rebind::Fresh }); } - // A binding persists for its logical queue's lifetime while physical - // QIDs allow: take a channel of a closed owner, then a fresh QID, and - // only then the least recently idle channel of one open owner. - if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { - return Ok(rebind); - } - if self.qids.compute_fits(self.admission.queues() as usize) { - return Ok(Rebind::Fresh); - } - let teardown = &self.teardown; - let idle = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.shareable()) - .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) - .min_by_key(|(key, _)| *key) - .map(|(_, qid)| qid); - let Some(qid) = idle else { return Err(ENOSPC); }; - admission.check()?; - let shared = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EIO)? - .share_idle()?; - if !shared { - return Err(ENOSPC); - } - Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) - } - - /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks - /// only whether some channel is free for a new owner. - fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { - if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { - return if queue.spared_pending() { - Readiness::Progress - } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { - // Publication reports the mismatch. - Readiness::Ready - } else if queue.pool_fits(required) || !queue.growing() { - Readiness::Ready - } else { - Readiness::Progress - }; - } - let teardown = &self.teardown; - let available = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| { - (queue.reusable() || queue.shareable() || queue.can_rearm()) - && !queue.context().is_some_and(|bound| teardown.releasing(bound)) - }); - if available || self.qids.compute_fits(self.admission.queues() as usize) { - Readiness::Ready - } else { - Readiness::Channel - } - } - - /// Binds `owner` to a reusable channel whose last owner released it, or - /// names a vacant one to re-arm. None if no channel is reusable. - fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; - let teardown = &self.teardown; - for mut queue in self + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { - admission.check()?; - // The release of the scheduler these records were bound to is armed - // or not yet consumed: the firmware may process it against this - // queue, which therefore must not change owner until it has. - if queue.context().is_some_and(|bound| teardown.releasing(bound)) { - continue; - } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Some(Rebind::Done)); + return Ok(Rebind::Done); } - Ok(vacant.map(Rebind::Rearm)) + Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -380,7 +156,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let mut queue = self + let queue = self .compute .iter_mut() .flatten() @@ -391,18 +167,13 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { - let teardown = &self.teardown; - let mut queue = self + let queue = self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; - // Its last owner may have closed while the backing was rebuilt. - if queue.context().is_some_and(|bound| teardown.releasing(bound)) { - return Err(EAGAIN); - } queue.bind_owner(owner, context.clone()) } @@ -460,7 +231,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let mut queue = self + let queue = self .compute .iter_mut() .flatten() @@ -490,13 +261,12 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for mut queue in self + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -505,23 +275,12 @@ impl Registry { Ok(freed) } - /// Builders and released queues waiting for their exact retirement or pool - /// witness temporarily occupy identities. A failed admission must let those - /// transactions finish rather than fail a healthy successor immediately. - fn compute_admission_error(&self, error: Error) -> Error { - if error == ENOSPC { - EAGAIN - } else { - error - } - } - pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { entry .queue - .as_deref() + .as_ref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -533,19 +292,15 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or_else(|| self.compute_admission_error( - crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; + .ok_or(ENOSPC)?; let identity = self.identities.reserve(1)?.start; - let queues = self.admission.queues() as usize; let id = self .qids - .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) - .map_err(|error| self.compute_admission_error(error))?; - self.note_build_room(); + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: ComputeSlot::new()?, + queue: None, }); Ok(reservation) } @@ -577,7 +332,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue.put(queue.take()); + entry.queue = queue.take(); Ok(()) } @@ -585,7 +340,6 @@ impl Registry { self.reserved(reservation)?; self.qids.cancel_unpublished(reservation.id)?; self.compute[reservation.slot] = None; - self.note_build_room(); Ok(()) } @@ -593,32 +347,20 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_deref()) + .and_then(|entry| entry.queue.as_ref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; - if unpublished { + if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; - self.note_build_room(); } Ok(()) } } -/// What a compute packet still lacks before publication. -#[derive(Copy, Clone, PartialEq, Eq)] -pub(super) enum Readiness { - Ready, - /// A physical channel: every one is busy and no QID fits. - Channel, - /// Retirement-bound capacity or reopened publication. - Progress, -} - /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -631,56 +373,13 @@ pub(super) enum Rebind { Fresh, } - -#[kunit_tests(asahi_g17_uninstalled_cleanup)] -mod uninstalled_cleanup_tests { - use super::*; - use crate::g17::fw::render::Deflake; - use crate::g17::fw::variant::V26A428; - #[test] - fn empty_reservation_prevents_reuse_until_explicit_cancellation() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let first = registry.reserve_compute(1).unwrap(); - assert!(registry.reserved(first).is_ok()); - assert!(registry.reserve_compute(1).is_err()); - let second = registry.reserve_compute(2).unwrap(); - assert_ne!(first.id.qid(), second.id.qid()); - registry.cancel_compute(first).unwrap(); - assert!(registry.reserved(first).is_err()); - let next = registry.reserve_compute(3).unwrap(); - assert_eq!(first.id.qid(), next.id.qid()); - assert!(first.id != next.id); - } -} - /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. -#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, - compute_backlog: core::sync::atomic::AtomicU32, - closed: core::sync::atomic::AtomicBool, - /// The cell of the compute queue this owner last published to under the device mutex, - /// with the shared compute objects mapped into its VM. Direct publication revalidates the - /// binding under the cell lock; a stale cell only sends the packet back to that path. - #[pin] - direct: kernel::sync::Mutex>>, - /// The channel a prepared packet is about to publish into. Its claim keeps the idle - /// channel from being shared away between prepare_job and run_job. - #[pin] - claimed: kernel::sync::Mutex>>, - /// Render pair admission, shared by every logical queue of the session. - admission: Arc, - /// The render pair this owner last published to without the device mutex - /// (zero for none): a hint, revalidated by every lend. - direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -690,63 +389,24 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - // A closed queue keeps its application context until its scheduler - // release is consumed and the context is returned. When only such - // contexts block this one, wait (bounded) for the teardown instead of - // failing a live client; the preparation lease is not held while - // sleeping, and nothing of the new queue is visible yet. - const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; - let started = now_ns(); - let (lease, uat, pool, qos, execution, slot) = loop { - let observed = shared.queues.observe(); - let lease = shared.preparations.enter(true)?; + let lease = shared.preparations.enter(true)?; + let dev = shared.drm()?; + let (uat, pool, qos, execution, slot) = { let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - // Installed compute channels narrow render's QIDs: every open - // logical queue keeps a dedicated render pair within reach. - if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { - return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); - } - match firmware.uat.new_execution_context(vm) { - Ok(execution) => { - break ( - lease, - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) - } - Err(error) if error == ENOSPC => { - // Every open queue holds one context; this one is counted - // in `queues()` already but holds none yet. - let open = firmware.queues.admission.queues().saturating_sub(1); - let closing = firmware.uat.execution_contexts_used().saturating_sub(open); - if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { - return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); - } - drop(slot); - drop(state); - drop(lease); - // Nothing is visible yet: a signalled creator restarts. - if kernel::current!().signal_pending() { - return Err(ERESTARTSYS); - } - shared.queue_events(); - // A closing queue returns its context when its release - // is reclaimed; that reclaim wakes this creator. - let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); - shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; - } - Err(error) => return Err(error), - } + let execution = firmware.uat.new_execution_context(vm)?; + ( + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) }; - let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -754,16 +414,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new( - &alloc, - &pool, - &qos, - execution, - owner_pid, - policy, - shared.renders.clone(), - shared.progress.clone(), - )?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -771,21 +422,14 @@ impl Backend { return Err(EAGAIN); } drop(state); - let admission = slot.pool().clone(); Ok(( - Arc::pin_init( - pin_init!(Self { + Arc::new( + Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), - compute_backlog: core::sync::atomic::AtomicU32::new(0), - closed: core::sync::atomic::AtomicBool::new(false), - direct <- kernel::new_mutex!(None, "g17::Backend::direct"), - claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), - admission, - direct_slot: core::sync::atomic::AtomicU32::new(0), - }), + }, GFP_KERNEL, )?, slot, @@ -794,15 +438,13 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self, admission: &Admission<'_>) -> Result { + fn shared_compute(&self) -> Result { loop { - admission.check()?; - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -821,9 +463,7 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - let _ = self.shared.changed.wait_timeout( - &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); + self.shared.changed.wait(&mut state); } }; let alloc = super::object::Allocator { @@ -834,9 +474,7 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else { built @@ -855,7 +493,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.note_changed(); + self.shared.changed.notify_all(); drop(lease); if result != Err(EAGAIN) { return result; @@ -869,12 +507,10 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, - admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -886,18 +522,14 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - let _ = self.shared.changed.wait_timeout( - &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); + self.shared.changed.wait(&mut state); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else { built @@ -915,30 +547,12 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.note_changed(); + self.shared.changed.notify_all(); result } - /// The caller keeps its preparation lease through this two-phase rollback. - /// An empty registry entry prevents reusing the QID/free-list identity while - /// its old aliases still occupy canonical or client addresses. Recovery joins - /// this lease before removing the session, even if its epoch was invalidated. - fn finish_uninstalled_compute( - &self, retired: Option<(ComputeReservation, KBox)>, - ) -> Result { - let Some((reservation, queue)) = retired else { return Ok(()); }; - drop(queue); - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation)?; - drop(state); - // The slot and its never-published QID are free again. - self.shared.channels.notify(); - Ok(()) - } - - fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; + fn grow_compute(&self, required: usize) -> Result { + use kernel::time::{delay::fsleep, Delta}; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -946,83 +560,45 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let mut growth = None; - let result = (|| loop { - // Growth waits for this queue's retirement and for firmware to consume the pool - // release; both are progress the event worker reports. - let observed = self.shared.progress.observe(); + let started = now_ns(); + loop { let result = (|| { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; if queue.pool_fits(required) { return Ok(None); } - if growth.is_none() { - let notify = queue.begin_growth(&firmware.init, required)?; - // Capture physical identity before any following fallible - // operation. Close can remove the owner-keyed lookup. - growth = queue.growth_token(); - if notify { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; - } + if queue.begin_growth(&firmware.init, required)? { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - self.wait_release(&self.shared.progress, observed, admission)?; - } - Err(error) => return Err(error), - } - })(); - if result.is_err() { - if let Some(token) = growth { - let mut retired = None; - let mut state = self.shared.state.lock(); - if let Some(firmware) = state.as_deref_mut() { - for index in 0..firmware.queues.compute.len() { - let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(aborted) = entry.queue.as_deref_mut() - .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; - if aborted == Some(true) { - // Keep the empty entry and QID reserved until all - // graph aliases have been destroyed outside the lock. - retired = entry.queue.take().map(|queue| (entry.reservation, queue)); - break; - } + Err(error) if error == EAGAIN && lease.is_current() + && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); } - } - drop(state); - // Tentative replacement mappings also belong to this builder. - // Discard them before a fresh QID can acquire a successor. - drop(alias.take()); - drop(pages.take()); - self.finish_uninstalled_compute(retired)?; - self.shared.queue_events(); + Err(error) => return Err(error), } } - result } - fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; - self.admission_open()?; + fn build_compute(&self, required: usize) -> Result { loop { - match self.shared_compute(admission) { + match self.shared_compute() { Err(error) if error == EAGAIN => continue, result => { result?; @@ -1031,14 +607,10 @@ impl Backend { } } loop { - admission.check()?; - // Sampled before any capacity check of this attempt. - let observed = self.shared.channels.observe(); - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; - let (uat, objects, reservation) = { + let (uat, reservation) = { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -1047,44 +619,13 @@ impl Backend { { return Err(EIO); } - self.admission_open()?; - let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { - Err(error) if error == ENOSPC => { - drop(state); - drop(lease); - self.wait_channels(observed, admission, true)?; - continue; - } - result => result?, - }; - match rebound { + match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), + Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = match firmware.queues.reserve_compute(self.owner) { - Ok(reservation) => reservation, - Err(error) if error == EAGAIN => { - // All physical channels are occupied. Retain their - // USC backing for exact-retirement handoff; releasing - // it cannot create a QID and forces fresh allocation. - drop(state); - drop(lease); - self.wait_channels(observed, admission, false)?; - continue; - } - Err(error) if error == ENOSPC => { - let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; - drop(state); - if reclaimed.is_none() { - return Err(error); - } - drop(reclaimed); - continue; - } - Err(error) => return Err(error), - }; - (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), Ok(reservation)) } } }; @@ -1097,32 +638,13 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) - })(); - let (pages, ring) = match built { - Err(error) if error == ENOSPC => { - drop(lease); - self.wait_channels(observed, admission, true)?; - continue; - } - result => result?, - }; + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } - if let Err(error) = self.admission_open() { drop(state); return Err(error); } - // A competing ready owner may have taken/rearmed it while - // backing was built off-lock. Leave rejected backing local - // so it is dropped after the device guard, not in rearm(). - let ready = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.qid() == qid && queue.can_rearm()); - if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -1138,12 +660,10 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } - self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -1151,16 +671,8 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease, admission)?; - let queue = compute::Queue::new( - &alloc, - &objects, - backing, - self.owner, - self.context.clone(), - self.shared.variant.pointers, - self.shared.operand_state, - )?; + self.compute_scheduler(&alloc, &lease)?; + let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -1170,14 +682,12 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) + firmware.queues.install_compute(reservation, &mut built) } }; // The old aliases must disappear before a waiting constructor can @@ -1193,11 +703,9 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result == Err(ENOSPC) { - self.wait_channels(observed, admission, true)?; - continue; + if result != Err(EAGAIN) { + return result; } - if result != Err(EAGAIN) { return result; } } } } @@ -1206,7 +714,6 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { - has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -1242,7 +749,6 @@ impl DeferredBatch { 0 }; Ok(Self { - has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1274,9 +780,6 @@ impl DeferredBatch { drop(release); } } - pub(super) fn vacated_room(&self) -> bool { - self.vacated.len() < self.vacated.capacity() - } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -1296,18 +799,10 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if completion.poisons(*error) { + if *error != ECANCELED && !completion.spared() { completion.status().record(*error); } } - match &value { - Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) - | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => - { completion.note_input_error(*error); }, - Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => - completion.note_source_retired(), - _ => {}, - } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1319,33 +814,11 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); - self.has_source_completions |= completion.sources().is_some(); } Ok(()) } - /// Completions collected so far. - pub(super) fn pending(&self) -> usize { - self.values.len() - } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { - use super::completion::Deferred; - if self.has_source_completions { - for _ in 0..self.values.len() { - let mut changed = false; - for value in &self.values { - if let Deferred::Retired(completion, Ok(_)) - | Deferred::Drained(completion, Ok(_)) = value - { - if let Err(error) = completion.check_source_inputs() { - changed |= completion.note_input_error(error); - } - } - } - if !changed { break; } - } - } - self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1366,15 +839,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - port: &'a super::port::Port, - outer: Option>, + primary: &'a mut super::Coprocessor, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, + accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - Ok(self.port.next_publication()) + self.qids.next_compute_publication() } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1391,204 +864,82 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.port.qos_publish(owner, scheduler, policy) + self.accounting.publish( + &qos::View::new(self.init.qos()?)?, + owner, + scheduler, + policy, + qos::clock, + ) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.port.qos_cancel(publication) + self.accounting + .cancel(&qos::View::new(self.init.qos()?)?, publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(priority, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) + self.primary.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) - } -} - -/// Publication into a bound, installed queue through the port alone. The caller checked the -/// recovery state, full effort and the queue's binding; first installation never comes here. -struct DirectHost<'a> { - port: &'a super::port::Port, - outer: Option>, - generation: u64, -} -impl compute::Host for DirectHost<'_> { - fn next_compute_publication(&mut self) -> Result { - Ok(self.port.next_publication()) - } - fn epoch(&self) -> Result<(u64, u32)> { - Ok((self.generation, self.port.recovery_state()?)) - } - fn prepare_compute_shared(&mut self) -> Result { - Err(EAGAIN) - } - fn qos_publish( - &mut self, - owner: qos::Owner, - scheduler: u64, - policy: super::fw::queue::Policy, - ) -> Result { - self.port.qos_publish(owner, scheduler, policy) - } - fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.port.qos_cancel(publication) - } - fn publish_qid(&mut self, _id: kick::Id) -> Result { - // An installed queue published its identity with its first command. - Ok(()) - } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) - } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } - fn note_submission(&mut self) -> Result { - Ok(()) + super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) } } impl super::Firmware { - #[inline] - fn publish_compute(&mut self, owner: u64, packet: Arc, - dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result - { - if packet.completion.sources().is_some() { - self.publish_compute_inputs(owner, packet, dependencies, deferred) - } else { - self.publish_compute_raw(owner, packet, dependencies, &[], deferred) - } - } - - #[inline(never)] - fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, - dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result - { - if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { - return Err(EAGAIN); - } - let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); - sources.check()?; - let mut merged = super::dependency::Frontier::new(); - for dependency in dependencies { merged.insert(0, *dependency)?; } - let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; - let mut native: KVec = - KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; - for source in &sources.sources { - if source.submission.source_status()? { continue; } - let found = self.queues.compute.iter_mut().flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), - self.recovery.generation()) { - Ok(None) => None, - result => Some(result), - }).ok_or(EAGAIN)??; - let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { - return Err(EAGAIN); - }; - // The completion-wait record orders the consumer behind its - // source; naming the source in the kick entry as well only adds - // a second, slower wait and a predecessor record. - let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; - if let Some(old) = native.iter_mut().find(|old| - old.dependency.qid() == dependency.qid()) { - if sequence > old.sequence { *old = wait; } - } else { - native.push(wait, GFP_KERNEL)?; - } - holds.push(hold, GFP_KERNEL)?; - } - let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), - native.as_slice(), deferred); - if packet.is_published() { - for queue in self.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) - { - for source in &sources.sources { queue.note_input_consumer(source); } - } - sources.set_holds(holds); - super::sync_input::note_published(); - super::sync_input::note_native(native.len()); - } - result - } - - fn publish_compute_raw( + fn publish_compute( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], - native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // reservation; dropping it unused returns the work-ring slot. - let outer = self.port.reserve_compute(packet.context.policy().priority())?; + // capacity check. The same device lock covers queue.publish below. + if !self + .init + .work_ready(packet.context.policy().priority(), DataMaster::Compute)? + { + return Err(EAGAIN); + } let queues = &mut self.queues; - let mut queue = queues + let queue = queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; - // The first command after an owner change waits while the scheduler - // the queue was bound to is being released: a live previous owner may - // have closed after giving up the queue. - if queue - .pending_switch_from() - .is_some_and(|previous| queues.teardown.releasing(previous)) - { - return Err(EAGAIN); - } let mut host = ComputeHost { init: &self.init, - port: &self.port, - outer: Some(outer), + primary: &mut self.primary, effort: &mut self.effort, qids: &mut queues.qids, + accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - if native.is_empty() { - queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) - } else { - queue.publish_with_native_waits(packet, dependencies, native, - &mut host, &mut |event| deferred.push(event)) - } + queue.publish(packet, dependencies, &mut host, &mut |event| { + deferred.push(event) + }) } } impl Registry { - /// Everything ensure_compute establishes for `owner`: the shared compute - /// objects mapped into its VM, and its queue bound to `context` with a - /// pool that fits and no spared witness to release. - fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { - let Some(queue) = self.compute(owner) else { - return Ok(false); - }; - Ok(!queue.spared_pending() - && queue.pool_fits(required) - && queue.matches(owner, queue.qid(), context)?) - } - fn release_compute_witness( &mut self, init: &super::initdata::InitData, - port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(mut queue) = self + let Some(queue) = self .compute .iter_mut() .flatten() @@ -1600,9 +951,10 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } + let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| port.qos_complete(&[owner]), + |owner| self.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1614,7 +966,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for mut queue in self + for queue in self .compute .iter_mut() .flatten() @@ -1641,53 +993,33 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; - // Every queue's pending identities are staged, not only the ones this batch - // retires: the scan passes each new record once, and a queue retired by a later - // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations)?; + requested |= queue.request_observations(&mut registry.observations, masks)?; } - // Completion records are appended in order: fold only the records written since - // the previous scan. A request whose stamp already covers it has its record in the - // ring; if the scan ended before finding one, read the whole ring once. if requested { - let observations = &mut registry.observations; - let seen = &mut registry.completion_seen; - registry.completion_cursor = self.init.fold_compute_completions( - registry.completion_cursor, - seen, - |record| observations.fold(record), - )?; - if observations.covered_pending() { - registry.completion_cursor = self - .init - .resync_compute_completions(seen, |record| observations.fold_any(record))?; - } + self.init + .scan_compute_completions(|record| registry.observations.fold(record))?; } - let port = &self.port; - let retired = deferred.pending(); - let mut freed = false; - for mut queue in registry + let view = qos::View::new(self.init.qos()?)?; + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); - queue.latch_observations(&mut registry.observations)?; + queue.latch_observations(&mut registry.observations, masks)?; queue.retire( masks, generation, state, - |owner| port.qos_complete(&[owner]), + |owner| registry.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; - freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1695,368 +1027,58 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(|mut queue| queue.take_terminal()); + .and_then(compute::Queue::take_terminal); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let port = &self.port; + let view = qos::View::new(self.init.qos()?)?; let registry = &mut self.queues; - for mut queue in registry + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| port.qos_complete(&[owner]), + |owner| registry.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; - freed |= busy && queue.channel_free(); - } - // Wake only the admission waits this batch can end. The fences gate - // scheduler entities alone; their callbacks never take the device mutex. - let shared = &self.primary.state.shared; - if freed { - shared.channels.notify(); - } - if deferred.pending() != retired { - shared.progress.notify(); } Ok(()) } } impl Backend { - fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { - admission.check()?; + fn ensure_compute(&self, scs_pages: u16) -> Result { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; + use kernel::time::{delay::fsleep, Delta}; + const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - loop { - admission.check()?; - let observed = self.shared.progress.observe(); + for _ in 0..SPARED_WAIT_MS { let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; if !pending { - match self.build_compute(required, admission) { - Err(error) if error == EAGAIN => continue, - result => return result, - } - } - // Other owners continue submitting while this owner waits for its - // queue-local drain witness. No new command has been accepted; if the - // admission deadline passes, the spared queue and its retained work stay - // intact until a later admission observes its witness. - admission.extend_for_recovery(); - self.shared.queue_events(); - self.wait_release(&self.shared.progress, observed, admission)?; - } - } - - /// Steady-state publication into this owner's bound channel without the device mutex: only - /// the channel's own lock and the port's small locks are taken, and nothing device-wide is - /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, - /// first installation, input sources, render prefixes, replay and recovery. - fn compute_direct( - &self, - packet: &Arc, - required: usize, - deferred: &mut DeferredBatch, - ) -> Option { - if packet.completion.sources().is_some() - || self.context.render_seen() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - let cell = self.direct.lock().clone()?; - let lease = self.shared.preparations.try_enter().ok()?; - let port = self.shared.port(&lease)?; - if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { - return None; - } - // Full effort is applied by the device-mutex path when it is not already. - if !port.note_submission() { - return None; - } - match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { - Ok(_) => {} - Err(error) if error == EBUSY => return None, - Err(error) => return Some(Err(error)), - } - let mut queue = cell.lock()?; - if !queue.installed() - || queue.spared_pending() - || !queue.pool_fits(required) - || queue.pending_switch_from().is_some() - || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) - || !lease.is_current() - { - return None; - } - let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; - let mut host = DirectHost { - port, - outer: Some(outer), - generation: port.generation(), - }; - match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { - Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, - result => Some(result), - } - } - - /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a - /// work state, retirement-bound room, reopened publication) becomes a fence that the release - /// providing it signals, so publication finds it ready instead of sleeping for it. Errors - /// and anything prepare cannot judge are left to publication. - pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { - if packet.early_result().is_some() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; - loop { - let channels = self.shared.channels.observe(); - let progress = self.shared.progress.observe(); - let (domain, observed) = match self.compute_readiness(packet, required) { - Readiness::Ready => return None, - Readiness::Channel => (&self.shared.channels, channels), - Readiness::Progress => (&self.shared.progress, progress), - }; - self.shared.queue_events(); - match domain.wait(observed) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, + return self.build_compute(required); } + // Other owners continue submitting while this owner gives its + // queue-local drain witness time to arrive. + fsleep(Delta::from_millis(1)); } - } - - fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { - match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { - Err(error) if error == EBUSY => return Readiness::Progress, - Err(_) => return Readiness::Ready, - Ok(_) => {} - } - // The steady state: this owner's channel is bound and installed. - if packet.completion.sources().is_none() { - if let Some(cell) = self.direct.lock().clone() { - let bound = cell.lock().is_some_and(|mut queue| { - let bound = queue.installed() - && !queue.spared_pending() - && queue.pool_fits(required) - && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); - if bound { - queue.claim(); - } - bound - }); - if bound { - self.hold_claim(Some(cell)); - return Readiness::Ready; - } - } - } - match self.sources_published(packet) { - Ok(false) => return Readiness::Progress, - Err(_) => return Readiness::Ready, - Ok(true) => {} - } - let lease = match self.shared.preparations.try_enter() { - Ok(lease) => lease, - Err(error) if error == EAGAIN => return Readiness::Progress, - Err(_) => return Readiness::Ready, - }; - let state = self.shared.state.lock(); - let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; - if !lease.is_current() - || firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 - { - return Readiness::Progress; - } - firmware.queues.compute_readiness(self.owner, Some(&self.context), required) - } - - /// Replaces the prepared packet's channel claim; None releases it after publication, - /// cancellation or close. - pub(super) fn hold_claim(&self, cell: Option>) { - let old = core::mem::replace(&mut *self.claimed.lock(), cell); - let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { - let busy = !queue.channel_free(); - queue.release_claim(); - busy && queue.channel_free() - }); - if freed { - // A channel nobody prepares into any more is shareable now. - self.shared.channels.notify(); - } - } - - /// Called with the device mutex held after a successful publication: remembers the bound - /// channel for direct publication. - fn remember_direct(&self, registry: &Registry) { - let cell = registry.compute_cell(self.owner); - let mut direct = self.direct.lock(); - let current = match (&*direct, &cell) { - (Some(old), Some(new)) => Arc::ptr_eq(old, new), - (None, None) => true, - _ => false, - }; - if !current { - let old = core::mem::replace(&mut *direct, cell); - drop(direct); - drop(old); - } - } - - /// Publishes a dependency-free compute command from its submit ioctl when - /// its queue is already bound and nothing has to wait, as renders do. Any - /// wait leaves the command, unpublished, to its scheduler. - pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { - if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { - return false; - } - if !packet.begin_publication() { - packet.completion.fail_unpublished(ECANCELED); - packet.set_early_result(Err(ECANCELED)); - return true; - } - let mut deferred = match DeferredBatch::publication() { - Ok(deferred) => deferred, - Err(error) => { - if packet.completion.status().get() != 0 { - ::fail_vm(self, error); - } - packet.completion.fail_unpublished(error); - packet.set_early_result(Err(error)); - return true; - } - }; - let result = match self.compute_direct_packet(packet, &mut deferred) { - Some(result) => result, - None => self.compute_inline(packet, &mut deferred), - }; - if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { - deferred.finish(); - packet.abandon_publication(); - return false; - } - let result = self - .finish_publication(packet, result, &mut deferred) - .map(|_| ()); - packet.set_early_result(result); - true - } - - fn compute_direct_packet( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Option { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { - return Some(Err(EINVAL)); - }; - let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { - Ok(required) => required, - Err(error) => return Some(Err(error)), - }; - self.compute_direct(packet, required, deferred) - } - - /// compute_packet's publication section without its waits: every condition - /// it would sleep or retry for returns FALLBACK instead. - fn compute_inline( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Result { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; - if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { - return Err(render::FALLBACK); - } - let lease = self.enter_preparation(false)?; - match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { - Err(error) if error == EBUSY => return Err(render::FALLBACK), - result => { - result?; - } - } - // A lone admission keeps the original latency-sensitive probe. - // Competing publishers otherwise spin on a binding they do not own. - if !self.context.compute_binding_hint() - && self.shared.compute_demand.load(Ordering::Relaxed) > 1 - { - return Err(render::FALLBACK); - } - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() - || self.shared.crashed.load(Ordering::Acquire) - || firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - return Err(render::FALLBACK); - } - firmware.replay_compute(deferred)?; - if !firmware.init.compute_vm_cached(self.context.vm()) - || !firmware.queues.compute_bound(self.owner, &self.context, required)? - { - return Err(render::FALLBACK); - } - let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { - Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { - Err(render::FALLBACK) - } - result => { - if result.is_ok() { - self.remember_direct(&firmware.queues); - } - result - } - } - } - - /// A scheduled dependency may refer to a previous recovery epoch. Do not - /// acquire consumer capacity until every live source is published now. - fn sources_published(&self, packet: &super::job::Packet) -> Result { - let Some(sources) = packet.completion.sources() else { return Ok(true); }; - sources.check()?; - let state = self.shared.state.lock(); - let firmware = state.as_deref().ok_or(ENODEV)?; - if firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { return Ok(false); } - for source in &sources.sources { - if source.submission.source_status()? { continue; } - if !firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.has_input_source(&source.submission, - packet.completion.status(), firmware.recovery.generation())) - { return Ok(false); } - } - Ok(true) + // No new command has been accepted. Keep the spared queue and its + // retained work intact until a later admission observes its witness. + Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -2067,70 +1089,23 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; - if let Some(result) = self.compute_direct(packet, required, deferred) { - return result; - } - // A bound queue needs none of ensure_compute's admission passes. Try - // without them; the publication section below checks the same state. - // Every retry blocks on the release that ends its wait: retirement - // progress (work states, ring room, replays) and the reopening of - // publication after recovery both signal `progress`. - let mut bound = true; - let mut demand = None; - let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); - let check = || { - if packet.cancel_requested() { return Err(ECANCELED); } - self.admission_open()?; - if packet.completion.status().get() != 0 { return Err(EIO); } - Ok(()) - }; - let admission = Admission::new(deadline, &check); - let progress = &self.shared.progress; + let mut recovery_polls = 0; + let mut outer_wait = None; loop { - admission.check()?; - let observed = progress.observe(); - if !self.sources_published(packet)? { - let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { - firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 - }); - if recovering { - admission.extend_for_recovery(); - } - self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; - continue; - } - // All DRM dependencies were satisfied before this scheduler callback. - // Recheck on every retry: pressure may yield an idle binding between - // admission and publication without changing this logical queue. - if !bound { - demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; - } - let lease = self.shared.preparations.enter_until(&admission)?; - // Work states free as this context's earlier commands retire. - match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { - Err(error) if error == EBUSY => { - drop(lease); - self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; - continue; - } - result => { - result?; - } + if packet.completion.status().get() != 0 { + return Err(EIO); } + let lease = self.shared.preparations.enter(false)?; + packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - admission.check()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } if self .shared .crashed @@ -2140,51 +1115,52 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - admission.extend_for_recovery(); + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } firmware.replay_compute(deferred)?; - if !firmware.init.compute_vm_cached(self.context.vm()) - || !firmware.queues.compute_bound(self.owner, &self.context, required)? - { - bound = false; - drop(state); - drop(lease); - continue; - } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), dependencies.as_slice(), deferred, ); - if !packet.is_published() - && (matches!(result, Err(EAGAIN)) - || (matches!(result, Err(EBUSY)) - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) + if matches!(result, Err(EBUSY) | Err(EAGAIN)) + && !packet.is_published() + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0) { - // Recovery or backpressure: nothing is firmware-visible. Other - // queues keep submitting and the event worker retires meanwhile. - if firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - admission.extend_for_recovery(); + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); } + recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } - if result.is_ok() { - self.remember_direct(&firmware.queues); + if matches!(result, Err(EAGAIN)) && !packet.is_published() { + let now = now_ns(); + let since = *outer_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // Backpressure owns no firmware-visible attempt. Other queues + // can submit and the event worker can retire while we sleep. + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; } return result; } @@ -2197,7 +1173,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for mut queue in self + for queue in self .queues .compute .iter_mut() @@ -2223,41 +1199,6 @@ impl super::Firmware { } impl super::Shared { - /// Charge only real watchdog expirations, after the per-job allowance. A - /// changing client or recovery generation cannot mint a fresh storm budget. - pub(super) fn watchdog_timeout(self: &Arc) -> bool { - if self.crashed.load(Ordering::Acquire) { return true; } - let stop = { - let mut state = self.state.lock(); - state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) - }; - if stop { self.lose_device_quietly(); } - stop - } - - /// Workload-triggered stop: halt producers first, then join only for a - /// bounded interval. A stalled borrower leaves stopped firmware and all its - /// mappings owned in state. Only a later joined stop may detach/free them. - pub(super) fn stop_runtime(&self) { - let Some(_stopping) = self.stop.try_lock() else { return; }; - { - let mut state = self.state.lock(); - if let Some(firmware) = state.as_deref_mut() { - if let Some(secondary) = firmware.secondary.as_mut() { - secondary.stop(); - } - firmware.primary.stop(); - } - } - if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { - dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); - return; - } - self.forget_port(); - let firmware = self.state.lock().take(); - if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } - } - /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -2267,13 +1208,10 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); - self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - if let Some(secondary) = firmware.secondary.as_mut() { - secondary.stop(); - } + firmware.secondary.stop(); firmware.primary.stop(); } state.take() @@ -2307,63 +1245,26 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - let mut freed = false; - let mut releasing = false; - for mut queue in self + for queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } - freed |= busy && queue.channel_free(); - releasing |= queue.pool_release_pending(); - } - freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; - let shared = &self.primary.state.shared; - // Edge notifications cover the transitions this driver makes; every - // event pass also rechecks the level, so capacity that appeared through - // another engine (render QIDs, closed logical queues, render-held work - // states) still ends a scheduler wait that has no timeout of its own. - if freed - || (shared.channels.has_waiters() - && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) - { - shared.channels.notify(); - } - // A pool release has no completion event: each observation pass is - // the progress a growing owner waits for. - if releasing || shared.progress.has_waiters() { - shared.progress.notify(); - } - if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.pool_release_stalled(now)) { - return Err(ETIMEDOUT); } + self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(None); } - // While the shared lower window refuses free-list backing, no idle - // retained compute pool stays warm: render and new compute need it more. - let cap = if super::freelist::lower_pressure() { - 0 - } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { - // A retired channel being requested by another runnable publisher - // is not an unused warm pool. Keep its existing backing through - // handoff; real allocation pressure above still forces reclamation. - super::kick::COMPUTE_CHANNELS - } else { - COMPUTE_WARM_CAP - }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - cap, + COMPUTE_WARM_CAP, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -2375,220 +1276,3 @@ impl super::Firmware { } } - -impl Registry { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - self.qids.write_capacity(f)?; - self.admission.write_capacity(f)?; - self.buffers.write_capacity(f)?; - writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; - writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; - self.render.write_capacity(f)?; - let now = now_ns(); - let waits = || self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) - .filter_map(|queue| queue.kill_waiting(now)); - compute_exit::write_kill_waits(f, waits().count(), waits().max().unwrap_or(0))?; - compute_exit::write_render_kills(f, self.teardown.kill_waits(now).count(), - self.teardown.kill_waits(now).max().unwrap_or(0))?; - writeln!(f, "compute_slots used={} total={} reusable={} building={}", - self.compute.iter().flatten().count(), self.compute.len(), - self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), - self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} fresh_fits={}", - self.compute.iter().flatten().count(), - u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; - writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; - for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { - queue.write_capacity(f)?; - } - Ok(()) - } -} - -impl Backend { - /// Compute admission without a channel: give back what pressure allows, then block until a - /// channel is released after `observed` (sampled before the failed check). - fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - self.reclaim_capacity(admission, reclaim_backing)?; - self.wait_release(&self.shared.channels, observed, admission) - } - - /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one - /// admission slice passes: cancellation and logical close have no release to wait for. - fn wait_release( - &self, - domain: &super::availability::Availability, - observed: u64, - admission: &Admission<'_>, - ) -> Result { - use kernel::dma_fence::RawDmaFence; - let slice = admission.slice_ms()?; - if let Some(fence) = domain.wait(observed)? { - // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. - let waited = unsafe { - kernel::bindings::dma_fence_wait_timeout( - fence.raw(), - false, - kernel::time::msecs_to_jiffies(slice) as isize, - ) - }; - if waited < 0 { - return Err(Error::from_errno(waited as i32)); - } - } - admission.check() - } - - /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, - /// under backing pressure, give idle pools back. - fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - admission.check()?; - let mut deferred = DeferredBatch::publication()?; - let result: Result = (|| { - // A short reclaim transaction obeys the same session/epoch gate as - // publication. Never keep this lease while sleeping for capacity. - let lease = self.shared.preparations.try_enter()?; - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { - return Err(EAGAIN); - } - if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { - return Ok(()); - } - // Consumption, not publication or an elapsed timeout, frees backing. - firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - // Pool eviction addresses backing allocation pressure, not channel - // occupancy. Normal idle expiry and the retained-pool cap still run - // in the event worker for either kind of admission wait. - if reclaim_backing { - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - } - if reclaim_backing && !firmware.render_control_backpressured() { - let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; - if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } - } - Ok(()) - })(); - deferred.finish(); - self.shared.queue_events(); - match result { - Err(error) if error == EAGAIN => {} - result => result?, - } - admission.check() - } -} - -#[kunit_tests(asahi_g17_shared_capacity)] -mod shared_capacity_tests { - use super::*; - use crate::g17::fw::render::Deflake; - use crate::g17::fw::variant::V26A428; - - #[test] - fn all_logical_render_owners_fit_beside_shared_compute() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - // With every execution context open, compute keeps only the channels - // that leave each logical queue a dedicated render pair. - for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { - registry.reserve_compute(owner).unwrap(); - } - assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); - assert!(registry.qids.render_fits(slots.len())); - for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); - // Every accepted logical render owner now has its physical pair; no - // published identity was cancelled or retyped to achieve the fit. - } - - #[test] - fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { - use crate::g17::fw::variant::V25G83; - let limit = super::super::kick::queue_limit(V25G83.qid_floor); - assert_eq!(limit, 42); - let mut registry = KBox::init( - Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), - GFP_KERNEL, - ) - .unwrap(); - let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); - let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); - for owner in 1..=u64::from(limit) { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - // The next logical queue is refused when it is created, not left waiting for a pair. - assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); - } - - #[test] - fn four_clients_have_capacity_before_any_owner_closes() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); - for client in 0..4u64 { - for queue in 0..9u64 { - let owner = 1 + client * 9 + queue; - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - } - for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { - registry.reserve_compute(owner).unwrap(); - } - assert_eq!(leases.len(), 36); - } - - #[test] - fn compute_grows_with_free_qids_and_narrows_queue_admission() { - let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); - let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); - for _ in 0..20 { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - // Sixteen open compute clients each keep their own channel. - for owner in 1..=16u64 { - registry.reserve_compute(owner).unwrap(); - } - // Growth stops where open queues plus headroom would lose their pairs. - let mut owner = 17; - while registry.reserve_compute(owner).is_ok() { - owner += 1; - } - assert!(registry.qids.render_fits(slots.len() + 24)); - assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); - } -} - -impl Backend { - /// Logical close is independent of whether an off-lock builder has already - /// installed a physical graph. No constructor can resurrect a closed owner. - fn admission_open(&self) -> Result { - if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } - if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - if !self.context.is_current() { return Err(EFAULT); } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index bf0f989f042a5..3d5f441d8428f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,52 +13,13 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); - } - } - - pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { - match packet.engine() { - Engine::Render => &self.backlog, - Engine::Compute => &self.compute_backlog, + self.backlog.fetch_sub(1, Ordering::AcqRel); } } fn stop_after_error(&self) { - // Scheduler/fence callbacks never synchronously join builders or RTKit - // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - } - - fn rollback_unpublished_compute(&self) -> Result { - // No outer preparation lease survives compute_packet's return. Hold a - // fresh epoch through alias destruction so recovery can join this work. - loop { - let lease = self.shared.preparations.enter(false)?; - let mut retired = None; - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { drop(state); drop(lease); continue; } - let mut binding = None; - if let Some(index) = firmware.queues.compute.iter().position(|entry| { - entry.as_ref().and_then(|entry| entry.queue.as_deref()) - .is_some_and(|queue| queue.owner() == Some(self.owner)) - }) { - let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; - let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? - .rollback_unpublished(self.owner, &self.context)?; - binding = replaced; - if cancel { - retired = entry.queue.take().map(|queue| (entry.reservation, queue)); - } - } - drop(state); - drop(binding); - self.finish_uninstalled_compute(retired)?; - self.shared.channels.notify(); - self.shared.queue_events(); - return Ok(()); - } + self.shared.stop_queues(); } /// Complete all failure fanout before signalling this packet's unpublished @@ -75,11 +36,6 @@ impl Backend { } deferred.finish(); if !packet.is_published() { - if packet.engine() == Engine::Compute { - if self.rollback_unpublished_compute().is_err() { - self.stop_after_error(); - } - } packet.completion.fail_unpublished(error); } Err(error) @@ -90,10 +46,6 @@ impl Backend { } fn release_queue(&self) -> Result { - // A builder may own only an off-lock reservation when close arrives. - // Record intent before scanning queues; every later install/rearm checks it. - self.closed.store(true, Ordering::Release); - self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -108,63 +60,12 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); - // Its idle channel, or its slot and never-published QID, is free. - self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { - fn input_sources(&self, inputs: &[crate::file::SyncItem]) - -> Option> - { - use crate::g17::{fence, sync_input}; - let mut heads = KVec::new(); - let mut sources: KVec = KVec::new(); - let mut publication_waits = KVec::new(); - for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { - let status = job::fence_status(input); - if status > 0 { continue; } - if status < 0 { return None; } - let leaves = sync_input::collect(input)?; - for leaf in &leaves { - if leaf.status > 0 { continue; } - if leaf.status < 0 { return None; } - // SAFETY: Collection owns each leaf until this lookup finishes. - let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; - if submission.source_status().ok()? { continue; } - if !submission.same_vm(self.context.status()) { return None; } - if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { - continue; - } - if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } - let published = { - let state = self.shared.state.lock(); - let firmware = state.as_deref()?; - if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 - { return None; } - firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.has_input_source(&submission, self.context.status(), - firmware.recovery.generation())) - }; - if !published { - if submission.source_status().ok()? { continue; } - // The existing scheduler fence becomes ready after actual - // publication, not GPU completion. No binding loan or - // consumer hardware capacity is acquired while waiting. - publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; - } - sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; - } - heads.push(input.clone(), GFP_KERNEL).ok()?; - } - if sources.is_empty() { return None; } - sync_input::Sources::new(heads, sources, publication_waits).ok() - } - fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -174,11 +75,7 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - // A reverse timeline chain must be submitted before the host signal. - // Validate here, but do not consume physical capacity behind unsignalled - // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) - .map(|_| ()) + Backend::ensure_compute(self, scs_pages) } fn fail_vm(&self, error: Error) { @@ -216,13 +113,8 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - /// Compute and render admission wait for their shared capacity here, as a - /// fence. - fn prepare(&self, packet: &Arc) -> Option { - match packet.engine() { - Engine::Compute => self.prepare_compute(packet), - Engine::Render => self.admit_render(packet), - } + fn prepare(&self, _packet: &Packet) -> Option { + None } fn publish(&self, packet: Arc) -> Result { @@ -250,25 +142,12 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; - if packet.engine() == Engine::Compute { - self.hold_claim(None); - } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } fn timed_out(&self, packet: Arc) -> sched::Status { - // A packet still in publication, for example waiting for capacity or - // for its render pair's graph, has nothing visible to firmware and may - // not even have an installed pair. Its publisher owns its bounded wait - // and result, and it keeps its place in the backlog so that a later - // dependency-free command is not published from its ioctl ahead of it. - // It is not a hang and does not count towards the timeout storm limit. - if !packet.is_published() { - return sched::Status::NoHang; - } self.release_backlog(&packet); - if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -281,9 +160,6 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); - if packet.engine() == Engine::Compute { - self.hold_claim(None); - } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -297,72 +173,22 @@ impl job::Backend for Backend { } } - fn long_job_beside(&self, _packet: &Arc) -> bool { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|other| other.owner() != Some(self.owner) && other.long_job_running()) - } - fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - use crate::g17::timeout::{self, Decision}; - // As for compute, a final retirement pass precedes the attribution, and all completion - // callbacks run after releasing the device mutex. Without a collector the attribution - // goes ahead without that pass, as it did before it existed. - let mut deferred = DeferredBatch::worker().ok(); - let result = (|| -> Result { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(false); - }; - if let Some(deferred) = deferred.as_mut() { - firmware.scan_renders(deferred)?; - } - // Borrowed-graph and queued-retirement deferrals share the renewal bound. - if timeout::transaction_grace(renewals) - && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) - { - return Ok(true); - } - let verdict = firmware.queues.render_timeout_decision(packet, renewals); - let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); - // As compute does: work still blocked behind another VM's older unfinished work - // after the whole grace is innocent, so its failure must stay non-sticky. The pair's - // quarantine extends the class to the pair's other groups. - if decision == Decision::Spare { - packet.completion.classify(crate::g17::recovery::Class::Spared); - } - if let Some(verdict) = verdict { - if let Some(blocker) = verdict.reported_blocker(renewals) { - dev_warn!(self.shared.dev.as_ref(), - "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", - self.owner, verdict.own.vm, verdict.own.published, renewals, decision, - blocker.vm, blocker.published, u8::from(blocker.failed)); - } - } - Ok(decision == Decision::Defer) - })(); - if let Some(deferred) = deferred.as_mut() { - deferred.finish(); - } - matches!(result, Ok(true)) + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) } fn try_early(&self, packet: &Arc) -> bool { - match packet.engine() { - Engine::Render => self.try_render_early(packet), - Engine::Compute => self.try_compute_early(packet), - } + self.try_render_early(packet) } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); + self.backlog.fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed53..dac4ca48a34ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,28 +20,26 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - // Each queue is locked on its own: release this one before visiting the others. - let Some(own) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) - .and_then(|queue| queue.timeout_head()) else { + let Some(queue) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) else { return Ok(Decision::Own); }; - let computes = firmware.queues.compute.iter().flatten() + let Some(own) = queue.timeout_head() else { + return Ok(Decision::Own); + }; + let others = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); - let verdict = timeout::attribute(own, renewals, others); - let decision = verdict.decision; + let decision = timeout::classify(own, renewals, others); if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if let Some(blocker) = verdict.reported_blocker(renewals) { + if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", - self.owner, own.vm, own.published, renewals, decision, - blocker.vm, blocker.published, u8::from(blocker.failed)); + "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", + self.owner, own.published, renewals, decision); } Ok(decision) })(); @@ -52,10 +50,7 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - // Without a collector the watchdog expires again and retries. - let Ok(mut deferred) = DeferredBatch::worker() else { - return Ok(()); - }; + let mut deferred = DeferredBatch::worker()?; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -63,7 +58,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { + let Some(queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -90,7 +85,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { + if let Some(queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 32b9acef94c0f..0d4a29e779b4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, recovery, Firmware}; +use crate::g17::{completion::Deferred, qos, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let port = &self.port; + let view = qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(mut queue) = entry.queue.as_deref_mut() { + if let Some(queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| port.qos_complete(&[owner]), + |owner| self.queues.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; } @@ -46,14 +46,20 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some((packet, owner)) = self.queues.compute[index] + let Some(queue) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) - .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; + let Some(packet) = queue.replay_front() else { + break; + }; + let owner = queue.owner(); + let outer_ready = self.init.work_ready( + packet.context.policy().priority(), + crate::g17::fw::queue::DataMaster::Compute, + )?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -77,14 +83,14 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_replay_dependencies()?; + packet.check_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let mut queue = self.queues.compute[index] + let queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -104,8 +110,6 @@ impl Firmware { } } self.queues.replay_outer_wait = None; - // Publication behind these replays may proceed. - self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3e779bba4fbbf..3ae1d0a609a33 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -15,9 +15,6 @@ use kernel::{prelude::*, sync::Arc}; /// Wake interval while a tiler heap may still shrink: the native check period. const SHRINK_WAKE_NS: u64 = 1_000_000_000; -/// Longest interval between two event passes that run every housekeeping step while -/// completion records alone keep the worker busy. -const HOUSEKEEPING_NS: u64 = 1_000_000; struct Service<'a> { firmware: &'a mut Firmware, @@ -27,30 +24,24 @@ struct Service<'a> { } impl event::Transport for Service<'_> { - fn roles(&self) -> &'static [Role] { - self.firmware.roles() - } fn trace(&mut self, role: Role) { - let firmware = &mut *self.firmware; + let firmware = &mut self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut *self.firmware; - firmware.observe_statistics_power(); + let firmware = &mut self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); - firmware.observe_statistics_power(); } fn drain( &mut self, role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = - channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); + let mut events = channel::Events::new(role); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -78,15 +69,6 @@ impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { self.firmware.queues.acknowledge_compute_exit(cookie); } - fn compute_fault(&mut self, qid: u8, stamp: u32) { - for entry in self.firmware.queues.compute.iter_mut().flatten() { - if let Some(mut queue) = entry.queue.as_deref_mut() { - if queue.qid() == qid { - queue.note_fault(stamp); - } - } - } - } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() } @@ -146,8 +128,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entry(&self, index: usize) -> Result<(u32, u32)> { - self.firmware.init.entry(index) + fn entries(&self) -> Result { + self.firmware.init.entries() } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -218,27 +200,6 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } - fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { - let render = self.firmware.queues.note_render_interrupted(qid, stamp); - if render != recovery::Interrupted::None { - return render; - } - self.firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .find(|queue| queue.qid() == qid) - .map_or(recovery::Interrupted::None, |queue| { - queue.note_interrupted(stamp) - }) - } - fn report_blamed_render(&self, qid: u8) { - self.firmware - .queues - .report_render_recovery(qid, &self.firmware.port); - } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -261,17 +222,19 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let port = &self.port; + let view = crate::g17::qos::View::new(self.init.qos()?)?; let registry = &mut self.queues; - for mut queue in registry + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { - deferred.push(event) - })?; + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; } Ok(()) } @@ -300,11 +263,6 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - self.rings.invalidate_statistics(); - if self.recovery.restart_limit(super::now_ns()) { - pr_warn!("G17: restart storm limit reached\n"); - return Err(ETIMEDOUT); - } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -314,15 +272,7 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - }); - // Publishers without the device mutex stamp their tickets with this - // mirror; they rejoin only after preparation admission reopens. - service - .firmware - .port - .set_generation(service.firmware.recovery.generation()); - service.firmware.rings.invalidate_statistics(); - let result = result?; + })?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -350,7 +300,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_runtime(); + self.stop_queues(); return; } let mut deferred = { @@ -365,10 +315,8 @@ impl crate::g17::Shared { }; deferred }; - crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); - crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -376,15 +324,11 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_runtime(); + self.stop_queues(); } } fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { - // Taken before consuming records: a request made during this pass gets the next one. - let requested = self - .housekeeping - .swap(false, core::sync::atomic::Ordering::AcqRel); let (recover, masks) = { let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -400,10 +344,8 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); - crate::g17::trace::event(1); if recover { - self.preparations - .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations.wait_drained(); } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -446,44 +388,19 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; - // Expiries, drained pairs, exits, pool and heap trimming and teardown give back what - // retirement freed; a completion needs none of them to retire or to signal. While work - // is in flight and nothing waits for capacity, a pass for firmware records alone skips - // them. Any requested pass, recovery, teardown, a waiter, the last retirement and, at - // the latest, the first pass after HOUSEKEEPING_NS run them all. - let housekeeping = requested - || recover - || !in_flight - || firmware.recovery.pending() - || !firmware.queues.teardown.is_empty() - || self.channels.has_waiters() - || self.progress.has_waiters() - || self.queues.has_waiters() - || self.renders.has_waiters() - || now.saturating_sub( - self.housekeeping_ns - .load(core::sync::atomic::Ordering::Relaxed), - ) >= HOUSEKEEPING_NS; - let mut lingers = None; - let mut pools = None; + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + let pools = firmware.service_compute_pools(deferred, now)?; let mut shrinking = false; - if housekeeping { - self.housekeeping_ns - .store(now, core::sync::atomic::Ordering::Relaxed); - if !firmware.recovery.pending() { - lingers = firmware.expire_render_lingers(now)?; - } - firmware.reclaim_drained_renders(deferred)?; - firmware.service_compute_exit(deferred)?; - pools = firmware.service_compute_pools(deferred, now)?; - if !firmware.recovery.pending() { - firmware - .queues - .vacate_evicted_render_pools(&firmware.init, deferred)?; - shrinking = firmware - .queues - .shrink_render_heaps(&firmware.init, now, deferred); - } + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); + } + { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; if !reclaim.is_empty() { @@ -502,7 +419,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + .any(|queue| queue.retirement_polling()); if polling { self.queue_poll(); } @@ -513,14 +430,11 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires, and - // so does an idle render free-list registration. - let idle = [pools, lingers] - .into_iter() - .fold(idle, |idle, expiry| match (idle, expiry) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }); + // A retained compute pool gives its backing back when it expires. + let idle = match (idle, pools) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }; if let Some(delay) = idle { self.queue_idle(delay); } @@ -530,23 +444,17 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || (firmware.secondary.is_some() - && channel::Events::pending(&firmware.init, Role::Secondary)?)) + || channel::Events::pending(&firmware.init, Role::Secondary)?) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. - self.queue_event_records(); + self.queue_events(); } let grow_ready = firmware.render_grow_ready(); drop(state); if grow_ready { self.queue_grow(); } - // Firmware ring room, consumed releases and teardown steps have no - // event of their own: recheck render publishers' conditions every pass. - if self.renders.has_waiters() { - self.renders.notify(); - } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 6657f83e9e738..a47abf2045526 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,7 +6,6 @@ mod exit; mod dependencies; -mod direct; mod grow; mod observe; mod pools; @@ -17,7 +16,6 @@ mod settle; mod submit; pub(super) mod teardown; -pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -26,7 +24,6 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -49,13 +46,7 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, - /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes - /// nothing, so the snapshot can only be stale towards an older switch. - borrowed_switch: Option, event_masks: [u64; 2], - /// A completion event for this pair arrived while it was borrowed. Event - /// passes skip borrowed pairs, so the returning borrower requests another. - missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -64,30 +55,17 @@ struct Entry { published: bool, } -/// Unpublished objects removed together, then destroyed outside the device lock. -pub(super) struct UnusedRenderSpare { - _entry: Option, - _binding: Option, -} - pub(super) struct RenderState { config: Config, entries: KVec>, pools: pools::Pools, grow: grow::Grow, retired: KVec, - /// Graphs of released contexts the firmware may still walk (see `detach_context`): kept - /// until the firmware stops, when this state is dropped. - kept_graphs: KVec, next_pm_generation: u64, } impl RenderState { - pub(super) fn new( - clusters: u32, - deflake: crate::g17::fw::render::Deflake, - descriptor_flags: [u32; 2], - ) -> Result { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -98,14 +76,12 @@ impl RenderState { Ok(Self { config: Config { clusters, - deflake, descriptor_flags, }, entries, pools: pools::Pools::new()?, grow: grow::Grow::new()?, retired: KVec::with_capacity(128, GFP_KERNEL)?, - kept_graphs: KVec::new(), next_pm_generation: 1, }) } @@ -148,14 +124,6 @@ impl RenderState { .ok_or(EIO) } - /// Whether a completion event arrived while the pair in `slot` was - /// borrowed; clears the record. The returned pair is installed again, so - /// the next event pass observes it. - fn take_missed_event(&mut self, slot: u8) -> bool { - self.entry_mut(slot) - .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) - } - fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -189,7 +157,6 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -209,7 +176,6 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -232,155 +198,11 @@ impl RenderState { } impl Registry { - /// Yield unused warming only when admission has run out of physical slots. - /// Previously published graphs keep their existing ownership and retirement - /// rules. The caller destroys returned backing after dropping the device lock. - pub(super) fn reclaim_unused_render_spare( - &mut self, - owner: Option, - require_ids: bool, - ) -> Result> { - // Prefer an empty reservation or an already parked channel. - for with_backing in [false, true] { - for slot in 1..RENDER_SLOTS { - let mut warm = false; - let expected = if let Some(entry) = self.render.entries[slot].as_ref() { - if entry.constructing - || entry.preparing.is_some() - || entry.growing - || entry.pending_error.is_some() - || entry.pending_timeout - { - continue; - } - // Compute can gain identities only from physical channels - // that never entered command publication. Render can reuse - // a retained render identity without changing its lifetime. - if owner.is_none() - && !entry - .reservation - .ids - .iter() - .all(|id| self.qids.can_cancel_unpublished(*id)) - { - continue; - } - if let Some(pair) = entry.pair.as_ref() { - if !with_backing { - continue; - } - if owner.is_some() && pair.reusable()? { - None - } else if pair.can_yield_unused()? { - warm = true; - Some(entry.reservation.owner) - } else { - continue; - } - } else { - if with_backing || entry.context.is_some() { - continue; - } - None - } - } else { - if with_backing || require_ids || owner.is_none() { - continue; - } - None - }; - if !self - .admission - .reassign_unused_spare(slot as u8, expected, owner) - { - continue; - } - if owner.is_none() { - let entry = self.render.entries[slot].take(); - if let Some(entry) = entry.as_ref() { - for id in entry.reservation.ids { - self.qids.cancel_unpublished(id)?; - } - if let Some(pair) = entry.pair.as_ref() { - self.buffers.unreserve_owner(pair.buffer_id())?; - } - self.note_build_room(); - } - return Ok(Some(UnusedRenderSpare { - _entry: entry, - _binding: None, - })); - } - let mut binding = None; - if warm { - let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; - let mut pair = entry.pair.take().ok_or(EIO)?; - self.buffers.unreserve_owner(pair.buffer_id())?; - pair.release_owner(); - let (parked, retired) = KBox::into_inner(pair).park(); - entry.parked = Some(parked); - entry.context = None; - entry.pool = None; - binding = Some(retired); - } - return Ok(Some(UnusedRenderSpare { - _entry: None, - _binding: binding, - })); - } - } - Ok(None) - } - - /// The installed pair in `slot` belongs to `owner` and its context, is - /// neither borrowed nor settling a failure, and its pool has backing: the - /// state in which building the pair would do nothing. - pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { - let Ok(entry) = self.render.entry(slot) else { - return Ok(false); - }; - if entry.pending_error.is_some() - || entry.pending_timeout - || entry.constructing - || entry.preparing.is_some() - || entry.growing - || entry.parked.is_some() - { - return Ok(false); - } - let Some(pair) = entry.pair.as_ref() else { - return Ok(false); - }; - Ok(pair.matches(owner, context)? && !pair.pool().vacant()) - } - - fn has_render_ids(&self, owner: u64) -> bool { - self.render - .entries - .iter() - .enumerate() - .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) - } - - /// Tell admission whether another render pair can still be built. - pub(super) fn note_build_room(&self) { - self.admission.note_build_room(self.qids.render_pair_fits()); - } - - pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } - if self.render.entries[usize::from(slot)].is_some() { - return Ok(()); - } - let built = self.build_render_ids(slot, owner); - self.note_build_room(); - built - } - - fn build_render_ids(&mut self, slot: u8, owner: u64) -> Result { - { + if self.render.entries[usize::from(slot)].is_none() { let identity = self.identities.reserve(2)?.start; let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { @@ -406,9 +228,7 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, - borrowed_switch: None, event_masks: [0; 2], - missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -424,13 +244,6 @@ impl Registry { /// ensure_render_ids cancels a first ID if the second cannot be reserved; /// validation, identity overflow, and cancellation errors still propagate. fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { - // A full queue table refuses a sibling without scanning it on every preparation. - if optional - && self.render.entries.get(usize::from(slot)).is_some_and(Option::is_none) - && !self.qids.render_pair_fits() - { - return Ok(false); - } match self.ensure_render_ids(slot, owner) { Ok(()) => Ok(true), Err(error) if optional && error == ENOSPC => Ok(false), @@ -449,8 +262,12 @@ impl Registry { { return Err(EBUSY); } - // Construction keeps identities reserved. Publication pins them at the - // first firmware-visible command, so unused warming can still yield. + let ids = self.render.entry(slot)?.reservation.ids; + // Physical queue identities survive failed graph construction. Only + // failure to acquire the second identity can return the first one. + for id in ids { + self.qids.publish(id)?; + } let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -557,26 +374,32 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish_pair( + fn qos_publish( &mut self, - owners: [crate::g17::qos::Owner; 2], + owner: crate::g17::qos::Owner, scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result<[crate::g17::qos::Publication; 2]> { - self.firmware.port.qos_publish_pair(owners, scheduler, policy) + ) -> Result { + use crate::g17::qos; + self.firmware.queues.accounting.publish( + &qos::View::new(self.firmware.init.qos()?)?, + owner, + scheduler, + policy, + qos::clock, + ) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - self.firmware.port.qos_cancel(publication) + use crate::g17::qos; + self.firmware + .queues + .accounting + .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { - // Firmware builds without MMIO registration install the pair from its queue - // configuration alone. - if !self.firmware.variant.mmio_pair_registration { - return Ok(()); - } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -596,9 +419,6 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } - fn next_publication(&mut self) -> Result { - Ok(self.firmware.port.next_publication()) - } fn publish_outer_pair( &mut self, priority: u8, @@ -638,9 +458,6 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); - crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { - self.firmware.bridge.regs.g17_core_power() - }); let result = self .firmware .primary @@ -660,20 +477,13 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { + use crate::g17::qos; let queues = &mut self.firmware.queues; - // Each final release is a control record the firmware processes before - // it returns to the work rings. While other pairs keep it busy, a - // registration left idle lingers instead, and the owner's next render - // reuses it. On an otherwise idle device the release is cheaper than - // reusing a registration that went idle, so it is sent at once. - // The pair retiring here is lent out of its entry meanwhile. - let linger = queues.render.entries.iter().flatten().any(|entry| { - entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) - }); - let next = queues.render.pools.completed(pool_id, generation, linger)?; + let next = queues.render.pools.completed(pool_id, generation)?; queues.buffers.can_release(buffer_id)?; + let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - self.firmware.port.qos_complete(&owners)?; + queues.accounting.complete(&view, &owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -706,30 +516,6 @@ impl crate::g17::Firmware { result } - /// Event-worker turn: ends free-list lingers that elapsed or whose owner - /// closed or moved on, publishes their releases, and returns the delay to - /// the next expiry, which the worker turns into a timed wake. - pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { - let render = &mut self.queues.render; - let entries = &render.entries; - let (ended, next) = render.pools.expire_lingers(now, |owner| { - entries.iter().flatten().any(|entry| { - !entry.closed - && entry - .pool - .as_ref() - .is_some_and(|pool| pools::Pools::owner(pool) == *owner) - }) - }); - if ended { - match self.release_render_lists() { - Err(error) if error == EAGAIN => {} - result => result?, - } - } - Ok(next) - } - fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -767,57 +553,6 @@ impl crate::g17::Firmware { Ok(()) } - /// A final retirement pass before a render watchdog attributes its timeout, as compute runs - /// scan_compute() first: a head that finished but whose completion the event worker has not - /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred - /// timeout or error, stay with the worker, which observes and settles them in its order. - pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.pending() - || self - .primary - .state - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Ok(()); - } - for slot in 1..self.queues.render.entries.len() { - if !self.queues.render.entries[slot] - .as_ref() - .is_some_and(|entry| { - entry - .pair - .as_ref() - .is_some_and(|pair| pair.published_in_flight()) - && entry.preparing.is_none() - && !entry.growing - && !entry.pending_timeout - && entry.pending_error.is_none() - }) - { - continue; - } - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - if let Some(pair) = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - { - let qids = pair.qids(); - let stamps = [ - self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), - self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), - ]; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; - } - self.retire_render_slot(slot, None, deferred)?; - self.queues.render.collect_retired(slot)?; - self.settle_render_terminal(slot, deferred)?; - } - Ok(()) - } - pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -869,22 +604,6 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } - /// Marks the interrupted group of the pair that owns `qid`. - pub(in crate::g17) fn note_render_interrupted( - &self, - qid: u8, - stamp: u32, - ) -> crate::g17::recovery::Interrupted { - self.render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid)) - .map_or(crate::g17::recovery::Interrupted::None, |pair| { - pair.note_interrupted(stamp) - }) - } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, @@ -1044,19 +763,9 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { - let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - let bit = masks[qid / 64] & (1u64 << (qid % 64)); - entry.event_masks[qid / 64] |= bit; - hit |= bit != 0; - } - // Event passes skip a pair while preparation, growth or an - // observation has borrowed it. Without a record, a completion - // that arrived meanwhile waits for an unrelated later event or the - // one-millisecond rescan. - if hit && (entry.preparing.is_some() || entry.growing) { - entry.missed_event = true; + entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); } } } @@ -1070,10 +779,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && (crate::g17::queue::render::completion_noted( - entry.reservation.ids.map(kick::Id::qid), - entry.event_masks, - ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) + && entry.reservation.ids.into_iter().all(|id| { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 + }) }) } pub(in crate::g17) fn defer_render_timeout( @@ -1107,105 +816,4 @@ impl Registry { entry.pending_timeout = true; true } - - /// The oldest published group of every render pair, for cross-VM timeout attribution - /// (crate::g17::timeout). - pub(in crate::g17) fn render_timeout_heads( - &self, - ) -> impl Iterator + '_ { - self.render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .filter_map(|pair| pair.timeout_head()) - } - - /// A render watchdog blocked behind another VM's older unfinished work gets the same - /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU - /// stalls on one client, the others' renders then outlive the culprit's teardown instead - /// of all timing out together. - pub(in crate::g17) fn render_timeout_decision( - &self, - packet: &Arc, - renewals: u8, - ) -> Option { - let own = self - .render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.contains_packet(packet)) - .and_then(|pair| pair.timeout_head()); - let computes = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter_map(|queue| queue.timeout_head()); - // Without a head of its own the timeout is its own. - own.map(|own| { - crate::g17::timeout::attribute( - own, - renewals, - Iterator::chain(self.render_timeout_heads(), computes), - ) - }) - } -} - -impl RenderState { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; - writeln!(f, "firmware_linked_render_graphs kept={} leaked={}", self.kept_graphs.len(), - teardown::LEAKED_GRAPHS.load(core::sync::atomic::Ordering::Relaxed))?; - self.pools.write_capacity(f)?; - for (slot, entry) in self.entries.iter().enumerate() { - if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { - context.write_capacity(f, "render", slot)?; - } - } - Ok(()) - } -} - -impl Registry { - /// Give idle private backing to another ready client under capacity pressure. - /// Keep graph/QID/context ownership; the next use rearms the same pool after - /// its consumed release, exactly as eviction-driven reclaim does. - pub(super) fn reclaim_idle_render_pools( - &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, - ) -> Result { - let render = &mut self.render; - render.pools.observe(init)?; - for entry in render.entries.iter().flatten() { - if !deferred.render_backing_room() { break; } - let Some(pool) = entry.pool.as_ref() else { continue; }; - // None means this pool never acquired a firmware registration; - // otherwise require its exact final release to have been consumed. - if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) - || self.admission.owner_active(entry.reservation.owner) { continue; } - // A sibling or replacement may be building off-lock before it has - // installed its pool reference. Its logical owner also excludes us. - let mut idle = true; - for other in render.entries.iter().flatten().filter(|other| { - other.reservation.owner == entry.reservation.owner - || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) - }) { - if other.constructing || other.growing || other.preparing.is_some() - || other.pending_error.is_some() || other.pending_timeout { - idle = false; - break; - } - if let Some(pair) = other.pair.as_ref() { - if !pair.idle_for_context(pair.context())? { idle = false; break; } - } - } - if idle { - if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } - } - } - Ok(()) - } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 277534328406e..b283fb8e7d55c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -14,8 +14,8 @@ use kernel::prelude::*; const RETRY_LIMIT: u16 = 1000; pub(super) struct Grow { - pub(super) requests: KVec, - pub(super) active: Option, + requests: KVec, + active: Option, pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, release_attempts: u16, } @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == context.id() + request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) }) }) }) @@ -228,8 +228,6 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - entry.borrowed_switch = - entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -271,16 +269,13 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; entry.growing = false; - // The grow work item requests an event pass when it finishes. - entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); } firmware.queues.render.grow.active = None; - self.note_changed(); + self.changed.notify_all(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 57543be54e835..31bcb688c8001 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -1,22 +1,21 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Event-driven render snapshots. A pair's completion memory is read and the -//! pair retired in one device-mutex hold, inside the preparation epoch. +//! Event-driven render snapshots outside the device mutex. The preparation +//! epoch and exact pair owner survive every read, cancellation and retry. use super::super::DeferredBatch; +use crate::g17::queue::render::Pair; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { - /// The first slot from `start` whose pair has news to observe: published work in flight, the - /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. - fn next_render_observation(&mut self, start: usize) -> Result> { + fn take_render_observation(&mut self, start: usize) -> Result)>> { if self.recovery.pending() { return Ok(None); } let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; for slot in start..self.queues.render.entries.len() { - let Some(entry) = self.queues.render.entries[slot].as_ref() else { + let Some(entry) = self.queues.render.entries[slot].as_mut() else { continue; }; if entry.preparing.is_some() || entry.growing { @@ -27,8 +26,9 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) - && !pair.stamp_waiting()) + || !qids + .into_iter() + .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) { continue; } @@ -37,22 +37,33 @@ impl crate::g17::Firmware { self.init.stamp(qids[1])?.load(Ordering::Relaxed), ]; core::sync::atomic::fence(Ordering::Acquire); - if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { - return Ok(Some(slot)); - } - // The stamps gate this pair out. If its front executed and was consumed anyway, the - // stamps may never move: after the bound, observe it and retire it on its record. - let pair = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .ok_or(EIO)?; - if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { - return Ok(Some(slot)); - } + let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { + continue; + }; + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet); + return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } Ok(None) } + fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.queues.render.entry_mut(slot as u8)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() as usize != slot + || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + pub(super) fn settle_render_terminal( &mut self, slot: usize, @@ -109,36 +120,57 @@ impl crate::g17::Shared { Err(error) if error == EAGAIN || error == ENODEV => break, result => result?, }; - // Selecting, reading and retiring the front share one hold: a - // separate hold for the read would let publishers and the - // retirement contend twice per observed pair. - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); + let loan = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { + break; + } + firmware.take_render_observation(cursor)? }; - if !epoch.is_current() - || firmware.recovery.pending() - || self.crashed.load(Ordering::Acquire) - { - break; - } - let Some(slot) = firmware.next_render_observation(cursor)? else { + let Some((slot, pair)) = loan else { break; }; if slot != cursor { remaining = crate::hw::t8140::queues::RENDER_DEPTH; retries = 0; } - let observed = firmware.queues.render.entries[slot] - .as_ref() - .and_then(|entry| entry.pair.as_ref()) - .ok_or(EIO)? - .observe_front(); - // Publishers waiting for render capacity recheck once this hold ends. - self.note_changed(); - if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { - // Observation and the growth check run on every pass. - self.queue_event_records(); + let observed = pair.observe_front(); + let mut pair = Some(pair); + let mut state = self.state.lock(); + // Return installed ownership before examining read errors or epoch. + let restored = (*state) + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); + if let Err(error) = restored { + // A failed invariant must not drop firmware-owned storage. Join + // the other preparers only after releasing our own epoch, and + // keep this detached owner until both processors have stopped. + drop(state); + drop(epoch); + self.lose_device_quietly(); + self.stop_queues(); + if let Some(mut pair) = pair { + pair.stopped(&mut |event| { + event.finish(); + Ok(()) + })?; + } + return Err(error); + } + let firmware = (*state).as_deref_mut().ok_or(EIO)?; + self.changed.notify_all(); + if firmware.render_grow_pending() { + self.queue_events(); + } + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; } let result = (|| { let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 1e3354d90dbfb..f7f48e853f225 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool, LINGER_NS}, + freelist::{Lease, Owner, RenderPool}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - pub(super) fn owner(pool: &RenderPool) -> Owner { + fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render(linger)?; + next.complete_render()?; Ok(next) } @@ -169,32 +169,6 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } - /// Ends the linger of every idle registration that `keep` (an open owner - /// still bound to the pool) does not retain or whose linger has elapsed. - /// Returns whether a release became pending, and the next expiry delay. - pub(super) fn expire_lingers( - &mut self, - now: u64, - keep: impl Fn(&Owner) -> bool, - ) -> (bool, Option) { - let mut ended = false; - let mut next: Option = None; - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { - let Some(since) = lease.lingering() else { - continue; - }; - let age = now.saturating_sub(since); - if age >= LINGER_NS || !keep(&lease.owner()) { - lease.end_linger(); - ended = true; - } else { - let delay = LINGER_NS - age; - next = Some(next.map_or(delay, |next| next.min(delay))); - } - } - (ended, next) - } - pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -212,7 +186,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + for lease in self.leases.iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -225,7 +199,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + for lease in self.leases.iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; @@ -265,11 +239,3 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } - -impl Pools { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "render_pool_leases used={} total={} retained={}", - self.leases.iter().flatten().filter(|lease| !lease.released()).count(), - POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 62ca7895a86f8..06bea3c84fa7a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,7 +16,6 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, - rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -29,14 +28,13 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), - rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: firmware.init.pm_metrics_va(), + client: crate::hw::t8140::CONFIG.pm_metrics.va, }, config: firmware.queues.render.config, }) @@ -46,10 +44,6 @@ impl BuildResources { } } -#[expect( - clippy::large_enum_variant, - reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." -)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -78,14 +72,8 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair( - &self, - slot: u8, - first: &Arc, - optional: bool, - sleep: bool, - ) -> Result { - let lease = self.enter_preparation(sleep)?; + fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -145,13 +133,6 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } - // The release of the pair's scheduler is armed or still being - // consumed: the firmware may process it against these queues. - // The teardown parks the pair once the release is witnessed; - // the new owner then installs it with fresh queue records. - if firmware.queues.teardown.releasing(pair.context()) { - return Err(EBUSY); - } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -183,8 +164,6 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - entry.borrowed_switch = - entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -259,11 +238,10 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; Pair::new( - alloc, + &alloc, &resources.objects, - &resources.rings, reservation.ids, slot, self.owner, @@ -274,9 +252,7 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, - resources.config.deflake, resources.config.descriptor_flags, - self.shared.variant, ) })(); let (mut built, error) = match built { @@ -301,7 +277,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -340,9 +316,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - alloc, + &alloc, &resources.objects, self.owner, self.context.clone(), @@ -384,7 +360,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - alloc, + &alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -421,12 +397,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } firmware.render_grow_ready() }; - self.shared.note_changed(); + self.shared.changed.notify_all(); if grow_ready { self.shared.queue_grow(); } @@ -456,9 +429,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - alloc, + &alloc, &resources.objects, self.owner, self.context.clone(), @@ -490,14 +463,10 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } - // That owner may have closed while this binding was prepared. - if firmware.queues.teardown.releasing(pair.context()) { - return Err(EBUSY); - } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); + return Err(ENOSPC); } firmware.queues.teardown.displace( pair.context(), @@ -506,7 +475,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - alloc, + &alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -530,10 +499,6 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -548,7 +513,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.note_changed(); + self.shared.changed.notify_all(); if grow_ready { self.shared.queue_grow(); } @@ -591,12 +556,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { - self.build_render_pair(slot, packet, false, sleep)?; - // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. - if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { - return Ok(()); - } + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { + self.build_render_pair(slot, packet, false)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -607,12 +568,10 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { - // The host lease excludes capacity reclamation until this off-lock - // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling.slot(), packet, true, sleep) { - Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} + match self.build_render_pair(sibling, packet, true) { + Err(error) if error == EBUSY || error == EAGAIN => {} result => result?, } } @@ -624,9 +583,8 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, - sleep: bool, ) -> Result { - let lease = self.enter_preparation(sleep)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -702,9 +660,6 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } if firmware.queues.render.entry(slot)?.closed { firmware .queues @@ -715,7 +670,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.note_changed(); + self.shared.changed.notify_all(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 499e3858fcca1..ce9771778acd4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packets in order; this -//! pass prepares and publishes the same commands again under the device mutex. +//! recovery, fragment end never written) and parks the packet; this pass +//! prepares and publishes the same command again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,91 +27,79 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - // Publish the parked groups front first; a pause keeps the rest parked. - while self.replay_render_front(slot, deferred)? {} - } - Ok(()) - } - - /// Publishes the oldest parked group of `slot`; returns whether it was published. - fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { - let Some((packet, owner, qids, closed, busy, stalled)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - pair.replay_stalled(crate::g17::now_ns()), - )) - }) - else { - return Ok(false); - }; - if stalled { - return Err(ETIMEDOUT); - } - if busy { - return Ok(false); - } - if closed || packet.cancel_requested() || !packet.context.is_current() { - self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) - })?; - return Ok(false); - } - if packet.completion.status().get() != 0 { - self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) - })?; - return Ok(false); - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, + let Some((packet, owner, qids, closed, busy)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + )) + }) + else { + continue; }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), - Err(error) => { + if busy { + continue; + } + if closed || packet.cancel_requested() || !packet.context.is_current() { self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) })?; - return Ok(false); - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - Ok(true) + continue; } - Err(error) if error == EBUSY || error == EAGAIN => Ok(false), - Err(error) => { + if packet.completion.status().get() != 0 { self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) + pair.fail_replay(EIO, &mut |event| deferred.push(event)) })?; - Ok(false) + continue; + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); + } + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + continue; + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + } + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + } } } + Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 71f0bfc052696..2e88848b93216 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,11 +11,7 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - // Without a collector the watchdog expires again and retries: memory - // pressure must not stop the runtime for every client. - let Ok(mut deferred) = DeferredBatch::worker() else { - return Ok(()); - }; + let mut deferred = DeferredBatch::worker()?; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -37,7 +33,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as isize, + kernel::time::msecs_to_jiffies(2000) as _, ) }; if completed > 0 { @@ -79,20 +75,6 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - if error == ETIMEDOUT { - let qids = pair.qids(); - let stamps = [ - self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), - self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), - ]; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - if let Some(pair) = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - { - pair.expire_stamp_wait(packet, stamps, state)?; - } - } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event) @@ -139,14 +121,6 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { - // Unclassified work counts as innocent, and a blamed group must never be replayed. - // Work an earlier recovery spared is collateral here (classify() blames before its - // collateral check) and stays spared, so its failure stays non-sticky; the blame - // itself is recorded apart from the class, and refuses any replay of it. - packet.completion.note_blamed(); - if !packet.completion.spared() { - packet.completion.classify(crate::g17::recovery::Class::Guilty); - } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 970035547dfee..13381b62da86c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,31 +8,13 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; - -/// Internal result of a publication attempted from the submitting ioctl that -/// met contention before anything became visible: the scheduler publishes the -/// command instead. Never returned to userspace. -pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; impl Backend { - /// The submitting ioctl holds the logical queue's mutex and never sleeps - /// for contention or a closed preparation gate: it returns FALLBACK, and - /// the scheduler's worker publishes the command and owns the waits. - pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { - if sleep { - return self.shared.preparations.enter(false); - } - // A closed gate (recovery) is the only case enter() would sleep for. - self.shared.preparations.try_enter().map_err(|error| { - if error == EAGAIN { - FALLBACK - } else { - error - } - }) - } - pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -41,312 +23,6 @@ impl Backend { self.render_with_lease(packet, None, deferred) } - /// Reserve identities before attaching a lease to the packet. An empty - /// logical slot must not hide reusable physical channels when the shared - /// queue-ID namespace is full. - /// Also reports whether the owner's installed pair is ready to take the - /// command, so that the publication skips graph construction. - fn acquire_render_with_ids( - &self, - packet: &Arc, - mut next: Option, - sleep: bool, - ) -> Result<(RenderLease, bool)> { - let pool = &self.admission; - let started = super::super::now_ns(); - let mut deadline_ns = - started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); - let recovery_deadline_ns = started.saturating_add(recovery_wait_ns()); - // A capacity refusal samples the render domain and retries once; only - // a refusal after that sample waits, so no release can be missed. - let mut observed = None; - let mut capacity = WaitClock::default(); - loop { - self.admission_open()?; - if super::super::now_ns() >= deadline_ns { - if deadline_ns < recovery_deadline_ns && self.waiting_for_recovery()? { - deadline_ns = recovery_deadline_ns; - } else { - return Err(EBUSY); - } - } - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let lease = if let Some(lease) = next.take() { - lease - } else { - // Admission sleeps for a pick; the ioctl leaves that to the scheduler. - if !sleep { - return Err(FALLBACK); - } - match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { - // Atomics only: the callback runs under the pool mutex. - if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - Ok(()) - }) { - Ok(lease) => lease, - Err(error) if error == ENOSPC => { - let reclaimed = { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .reclaim_unused_render_spare(Some(self.owner), false)? - }; - if reclaimed.is_none() { - self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; - } - drop(reclaimed); - continue; - } - // The deadline check above decides whether a recovery extends the wait. - Err(error) if error == EBUSY => continue, - Err(error) => return Err(error), - } - }; - let slot = lease.slot(); - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => { - let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; - return Ok((lease, ready)); - } - Err(error) if error == ENOSPC => { - let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; - if reclaimed.is_some() { - drop(state); - drop(reclaimed); - next = Some(lease); - continue; - } - // ensure_render_ids leaves no partial entry on refusal. - if firmware.queues.render.entries[usize::from(slot)].is_some() { - return Err(error); - } - if !lease.cancel_empty(self.owner) { - continue; - } - let reclaimed = firmware - .queues - .reclaim_unused_render_spare(Some(self.owner), true)?; - let ready = firmware.queues.has_render_ids(self.owner); - drop(state); - // A full queue table makes admission reuse a free killed pair instead. - if reclaimed.is_none() && !ready && !pool.killed_pair_free() { - self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; - } - drop(reclaimed); - } - Err(error) => return Err(error), - } - } - } - - /// Scheduler admission of a render packet. A work state of its context or - /// a pair admission it lacks becomes a fence that the return providing it - /// signals, so run_job finds both ready instead of waiting for them. - /// Errors, and anything else publication may wait for, are left to it. - pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { - if packet.early_result().is_some() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - // Each check runs once more after sampling its domain, so the common - // case takes no domain lock and no return between check and wait is lost. - let work = packet.completion.work_state().ok()?; - loop { - match work.acquire(crate::g17::job::Engine::Render) { - Err(error) if error == EBUSY => { - let states = work.returns().observe(); - match work.acquire(crate::g17::job::Engine::Render) { - Err(error) if error == EBUSY => match work.returns().wait(states) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, - }, - Err(_) => return None, - Ok(_) => {} - } - } - Err(_) => return None, - Ok(_) => {} - } - let request = Request::of(packet); - if self.admission.admits_render(self.owner, request) { - return None; - } - let observed = self.shared.renders.observe(); - if self.admission.admits_render(self.owner, request) { - return None; - } - self.shared.queue_events(); - match self.shared.renders.wait(observed) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, - } - } - } - - /// Conditions which end a publication that has not become visible yet. - fn publication_open(&self, packet: &Packet) -> Result { - self.admission_open()?; - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - Ok(()) - } - - /// Blocks on `domain` until a notification after `observed`, or the - /// deadline, which only bounds the wait. Slices of at most 10 ms let the - /// publisher observe cancellation, which has no notification here. - fn wait_domain( - &self, - domain: &crate::g17::availability::Availability, - observed: u64, - deadline_ns: u64, - packet: &Packet, - ) -> Result { - self.publication_open(packet)?; - let remaining = deadline_ns.saturating_sub(super::super::now_ns()); - if remaining == 0 { - return Err(ETIMEDOUT); - } - domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) - } - - /// A recovery is pending, or this owner's pair is spared and draining after one: an - /// innocent publication waits up to RECOVERY_WAIT_MS for it instead of failing. - fn waiting_for_recovery(&self) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_deref().ok_or(ENODEV)?; - Ok(firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - || firmware.queues.admission.spared_quarantined(self.owner)) - } - - /// The deadline of the wait `clock` measures (WaitClock::deadline). - fn wait_deadline(&self, clock: &mut WaitClock, bound_ns: u64) -> Result { - let recovering = self.waiting_for_recovery()?; - Ok(clock.deadline(super::super::now_ns(), recovering, bound_ns)) - } - - /// A pair is borrowed, still settling, waiting for firmware room or for the - /// release of the scheduler it last ran for. Every pair return, retirement - /// and teardown step signals the render domain, and each event pass does - /// while a publisher waits; the deadline is only the failure bound. - fn wait_render_busy( - &self, - clock: &mut WaitClock, - observed: &mut Option, - sleep: bool, - packet: &Packet, - ) -> Result { - if !sleep { - return Err(FALLBACK); - } - let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; - // Sample, then retry once: only a refusal after the sample waits. - let Some(sample) = observed.take() else { - *observed = Some(self.shared.renders.observe()); - return Ok(()); - }; - self.shared.queue_events(); - self.wait_domain(&self.shared.renders, sample, deadline, packet) - } - - /// Render capacity is short: return what idle owners hold, then block until - /// a pair, lease or owner returns. No packet is firmware-visible here, and - /// neither the device mutex nor a preparation lease is held while waiting. - fn wait_render_capacity( - &self, - observed: &mut Option, - clock: &mut WaitClock, - sleep: bool, - ) -> Result { - if !sleep { - return Err(FALLBACK); - } - if self.reclaim_render_capacity()? { - *observed = None; - return Ok(()); - } - let Some(sample) = observed.take() else { - *observed = Some(self.shared.renders.observe()); - return Ok(()); - }; - self.shared.queue_events(); - self.admission_open()?; - let remaining = self - .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? - .saturating_sub(super::super::now_ns()); - if remaining == 0 { - return Err(EBUSY); - } - self.shared - .renders - .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) - } - - /// Gives idle render backing back under capacity pressure. Returns whether - /// anything was returned, in which case the caller retries at once. - fn reclaim_render_capacity(&self) -> Result { - let mut deferred = DeferredBatch::publication()?; - let result: Result = (|| { - let lease = self.shared.preparations.try_enter()?; - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { - return Err(EAGAIN); - } - if firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - return Ok(false); - } - firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - let mut published = 0; - if !firmware.render_control_backpressured() { - (published, _) = firmware.queues.vacate_idle_pools( - &firmware.init, - 0, - super::super::now_ns(), - 0, - )?; - if published != 0 { - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; - } - } - Ok(published != 0 - || !deferred.render.is_empty() - || !deferred.vacated.is_empty() - || !deferred.bindings.is_empty()) - })(); - deferred.finish(); - match result { - Err(error) if error == EAGAIN => Ok(false), - result => result, - } - } - fn render_with_lease( &self, packet: &Arc, @@ -354,7 +30,6 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -362,48 +37,22 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - // A publication from the submitting ioctl (an early lease) never - // sleeps for contention; it falls back to the scheduler instead. - let sleep = early.is_none(); - let mut early = early; - // The pair this owner last published to directly: a lease on it goes - // straight to the direct path, whose lend checks everything the - // identity check would. - let known = self.direct_slot.load(Ordering::Relaxed); - let lease = match early.take() { - Some(lease) => Some(lease), - None if known != 0 => self - .admission - .try_acquire_render(self.owner, Request::of(packet)) - .ok() - .flatten(), - None => None, - }; - let early = match lease { - Some(lease) if u32::from(lease.slot()) == known => { - let slot = lease.slot(); - packet.completion.attach_render_lease(lease)?; - if let Some(result) = self.render_direct(packet, slot, deferred) { - return result; - } - packet.completion.detach_render_lease() + let admission = match early { + Some(lease) => lease, + None => { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + pool.acquire_render(self.owner, Request::of(packet))? } - lease => lease, }; - let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - if ready { - if let Some(result) = self.render_direct(packet, slot, deferred) { - return result; - } - } - let mut busy = WaitClock::default(); - let mut capacity = WaitClock::default(); - let mut state_clock = WaitClock::default(); - let mut recovery_since = None; - let mut seen = None; - let mut states = None; + let mut state_polls = 0; + let mut state_delay = 20i64; + let mut takeover_polls = 0; + let mut recovery_polls = 0; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -416,12 +65,8 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = if sleep { - self.shared.preparations.enter(interruptible)? - } else { - self.enter_preparation(false)? - }; - let result = packet.completion.work_state().and_then(|state| state.acquire(crate::g17::job::Engine::Render)); + let preparation = self.shared.preparations.enter(interruptible)?; + let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result } else { @@ -431,77 +76,51 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if !sleep { - return Err(FALLBACK); - } - // Every node of this context belongs to unretired work: - // sample, retry once, then wait for the next returned one. - let work = packet.completion.work_state()?; - match states.take() { - None => states = Some(work.returns().observe()), - // Work states free as earlier, possibly spared, work retires. - Some(sample) => { - let deadline = self - .wait_deadline(&mut state_clock, super::super::WORK_RING_WAIT_NS)?; - self.wait_domain(work.returns(), sample, deadline, packet)? - } + if state_polls == 1000 { + return Err(ETIMEDOUT); } + state_polls += 1; + fsleep(Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - // A pair found ready under the identity check needs no build; any - // retry revalidates through ensure_render. - let ensured = if core::mem::take(&mut ready) { - Ok(()) - } else { - self.ensure_render(packet, slot, sleep) - }; - match ensured { - Err(error) if error == ENOSPC => { - self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; - continue; - } + match self.ensure_render(packet, slot) { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); continue; } result => result?, } - match self.prepare_render(slot, packet, deferred, sleep) { - Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; - continue; - } + match self.prepare_render(slot, packet, deferred) { Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let (recovering, gate) = { + let recovering = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - ( - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, - self.shared.preparations.snapshot(), - ) + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { - if !sleep { - return Err(FALLBACK); + if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(ETIMEDOUT); } - // Recovery closes the preparation gate before touching - // any queue; the next pass blocks in enter() until it - // reopens. The worker raises the closure. - let since = *recovery_since.get_or_insert(super::super::now_ns()); + recovery_polls += 1; self.shared.queue_events(); - self.shared - .preparations - .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; + fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; - } else if !sleep { - return Err(FALLBACK); + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); } } result => return result, @@ -513,9 +132,19 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { - Ok(Some(lease)) => lease, - _ => return false, + let admission = { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + match firmware + .queues + .admission + .try_acquire_render(self.owner, Request::of(packet)) + { + Ok(Some(lease)) => lease, + _ => return false, + } }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -534,16 +163,6 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); - if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { - // Contention before anything became visible: the ioctl does not - // sleep. Return the lease and the claim; the scheduler publishes - // the command, and later ones wait behind it in the backlog. - let lease = packet.completion.detach_render_lease(); - deferred.finish(); - drop(lease); - packet.abandon_publication(); - return false; - } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -667,37 +286,16 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } - let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; - // Every render publication, with or without the device mutex, holds the - // port's render lock: the render work rings have one host producer at a - // time and attachment overlaps are ordered by publication order. - let port = self.port.clone(); - let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. + // either inner queue becomes visible. All host producers share this lock. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - if let Some(footprint) = packet.footprint.as_ref() { - dependencies.hazard_count = self.queues.admission.publication_barriers( - owner, - slot, - footprint, - &mut dependencies.hazards, - )?; - } + let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; - // An owner change taken from a live owner which closed meanwhile: its - // first command waits until that scheduler's release has been consumed. - if pair - .owner_switch_from(&packet.context) - .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) - { - return Err(EBUSY); - } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; @@ -737,57 +335,3 @@ impl crate::g17::Firmware { result } } - -fn recovery_wait_ns() -> u64 { - u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 -} - -/// One bounded wait (Backend::wait_deadline). -#[derive(Default)] -struct WaitClock { - first: Option, - calm: u64, -} - -impl WaitClock { - /// `bound_ns` after the later of the first call and the last call that saw a recovery - /// pending or this owner's spared pair draining (so the usual bound applies afresh once a - /// recovery ends), and at most RECOVERY_WAIT_MS after the first call. - fn deadline(&mut self, now: u64, recovering: bool, bound_ns: u64) -> u64 { - let first = *self.first.get_or_insert(now); - self.calm = if recovering { now } else { self.calm.max(first) }; - first - .saturating_add(recovery_wait_ns()) - .min(self.calm.saturating_add(bound_ns)) - } -} - -#[kunit_tests(asahi_g17_wait_clock)] -mod tests { - use super::*; - - const BOUND: u64 = 2_000_000_000; - - #[test] - fn a_calm_wait_keeps_its_first_deadline() { - let mut clock = WaitClock::default(); - assert_eq!(clock.deadline(100, false, BOUND), 100 + BOUND); - assert_eq!(clock.deadline(500, false, BOUND), 100 + BOUND); - } - - #[test] - fn the_bound_starts_afresh_once_a_recovery_ends() { - let mut clock = WaitClock::default(); - clock.deadline(100, false, BOUND); - assert_eq!(clock.deadline(1_000, true, BOUND), 1_000 + BOUND); - assert_eq!(clock.deadline(2_000, false, BOUND), 1_000 + BOUND); - } - - #[test] - fn a_recovery_cannot_extend_past_the_recovery_wait() { - let mut clock = WaitClock::default(); - clock.deadline(100, true, BOUND); - let limit = 100 + recovery_wait_ns(); - assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index fffa6d28dd289..759c0c2e43dc0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,15 +17,6 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { - // A pair that changed owner keeps running this scheduler's queues until - // the new owner's first command leaves it, whichever context owns it now. - let scheduler = Some(context.scheduler_va()); - if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { - Some(pair) => pair.switch_in_flight() == scheduler, - None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, - }) { - return Ok(false); - } for entry in self.entries.iter().flatten() { if !entry .context @@ -38,7 +29,7 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { + if !pair.idle_for_context(context)? { return Ok(false); } } @@ -46,14 +37,6 @@ impl RenderState { Ok(true) } - /// Some render pair runs another context's commands. - pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { - self.entries.iter().flatten().any(|entry| { - !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) - && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) - }) - } - pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self @@ -123,7 +106,6 @@ impl RenderState { if !out.has_room(count, 1) { return Err(EBUSY); } - let linked = context.firmware_may_link(); for entry in self.entries.iter_mut().flatten() { if !entry .context @@ -138,17 +120,8 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (mut parked, retired) = KBox::into_inner(pair).park(); - // This park follows the consumed release of `context`. - parked.release_consumed(context.scheduler_va()); - if linked { - // Guard (kill before free): the firmware may still walk this context's work - // list into the graph, so it is not freed or recycled before the firmware stops. - kernel::pr_warn_once!("G17: keeping render graphs of an unkilled context that still links work\n"); - keep_graph(&mut self.kept_graphs, retired); - } else { - *target = Some(retired); - } + let (parked, retired) = KBox::into_inner(pair).park(); + *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. entry.context = None; @@ -163,12 +136,8 @@ impl RenderState { if let Some(id) = self.retired[index].buffer_id() { buffers.unreserve_owner(id)?; } - let retired = self.retired.remove(index).map_err(|_| EIO)?; - if linked { - keep_graph(&mut self.kept_graphs, retired); - } else { - *out.binding_slot()? = Some(retired); - } + let target = out.binding_slot()?; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); } Ok(()) } @@ -193,23 +162,3 @@ impl Registry { Ok(()) } } - -/// Graphs that could not be recorded in `kept_graphs` and were leaked instead, for debugfs. -pub(super) static LEAKED_GRAPHS: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); - -/// Keeps `graph` until the firmware stops; if even that cannot be recorded, it is leaked -/// rather than freed. -fn keep_graph( - kept: &mut KVec, - graph: crate::g17::queue::render::RetiredBinding, -) { - if kept.reserve(1, GFP_KERNEL).is_ok() { - if let Err(error) = kept.push_within_capacity(graph) { - LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); - core::mem::forget(error.0); - } - } else { - LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); - core::mem::forget(graph); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 0222ae4c478c8..8a637e0b3ed01 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -100,8 +100,6 @@ impl ReclaimBatch { struct Service<'a> { render: &'a mut super::render::RenderState, - admission: &'a crate::g17::queue::admission::Pool, - objects: &'a Arc, compute: &'a [Option], buffers: &'a mut BufferIds, init: &'a InitData, @@ -117,8 +115,8 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) + .filter_map(|entry| entry.queue.as_ref()) + .all(|queue| queue.idle_for_context(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) @@ -139,15 +137,8 @@ impl Host for Service<'_> { self.primary.notify(MSG_CONTROL_NOTIFY) } fn detach(&mut self, context: &Arc) -> Result { - let linked = context.retain_linked_work(self.objects); - let (_, _, slots) = self.render.exit_scope(context); - crate::g17::trace::teardown(5, u64::from(context.id()), slots, u64::from(linked)); self.render .detach_context(context, self.buffers, self.reclaim)?; - // The firmware consumed this context's scheduler release after its kill (if any) - // was acknowledged and its pairs went idle: it no longer names these pairs, so - // their kill hold ends here rather than at the next processor stop. - self.admission.release_killed_slots(slots); // detach_context preflights this slot together with every graph slot. // Retain it before Pending removes its reference under the device lock. self.reclaim.context(context) @@ -182,7 +173,6 @@ impl Firmware { self.queues.render.observe_teardown_releases(&self.init)?; let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; let Registry { - admission, render, compute, buffers, @@ -191,8 +181,6 @@ impl Firmware { } = &mut self.queues; let mut service = Service { render, - admission, - objects: &self.pool, compute, buffers, init: &self.init, @@ -202,17 +190,10 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); - let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - { - consumed = true; - } - if consumed { - // A channel whose last scheduler release is consumed can change owner. - self.primary.state.shared.channels.notify(); - } + {} Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index fa2a72a77f060..1a25cd7e3fa3b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -24,45 +24,6 @@ struct Entry { phase: Phase, armed: u64, announce_pending: bool, - render_kill: Option, - /// Since when, and whether reported, a render-only kill has waited for other contexts' - /// compute to pause. - kill_wait: Option<(u64, bool)>, - /// A render-only kill of this context was refused (counted once per context). - kill_refused: bool, -} - -/// The kill of a closed context whose compute side is clean, published so that its render -/// pairs' executed but unstamped commands can be released (see `Pair::firmware_drained`). -/// The scheduler page's killed state, not this record, proves the kill to the pairs. -#[derive(Clone, Copy)] -struct RenderKill { - cookie: u64, - published: u64, - announced: bool, - acknowledged: bool, -} - -impl RenderKill { - fn new(cookie: u64, now: u64) -> Self { - Self { - cookie, - published: now, - announced: false, - acknowledged: false, - } - } - /// Only this kill's exact cookie acknowledges it, once. - fn acknowledge(&mut self, cookie: u64) -> bool { - if cookie == 0 || cookie != self.cookie || self.acknowledged { - return false; - } - self.acknowledged = true; - true - } - fn awaiting(&self, now: u64, window: u64) -> bool { - !self.acknowledged && now.saturating_sub(self.published) < window - } } /// Publication and announcement errors retain the entry; no timeout frees @@ -114,21 +75,6 @@ impl Pending { self.entries.is_empty() } - /// The context's scheduler release is armed or published, and the queues it - /// last ran have not been detached yet. Until then the firmware may still - /// process the release against those queues: none of them may change owner. - pub(crate) fn releasing(&self, context: &Arc) -> bool { - self.find(context) - .is_some_and(|index| self.entries[index].phase != Phase::Deferred) - } - - /// `releasing` for the context that owns scheduler page `scheduler`. - pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { - self.entries.iter().any(|entry| { - entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler - }) - } - fn find(&self, context: &Arc) -> Option { self.entries .iter() @@ -143,9 +89,6 @@ impl Pending { phase, armed: now, announce_pending: false, - render_kill: None, - kill_wait: None, - kill_refused: false, }; self.entries.push(entry, GFP_KERNEL)?; Ok(()) @@ -275,94 +218,6 @@ impl Pending { Ok(None) } - /// A render-only kill of the closed context at `index` was published. - pub(crate) fn render_kill_published(&self, index: usize) -> bool { - self.entries.get(index).is_some_and(|entry| entry.render_kill.is_some()) - } - - /// Record a published render-only kill. Publication is irrevocable: the record stays - /// with the entry until the context's scheduler release is consumed. - pub(crate) fn record_render_kill(&mut self, index: usize, cookie: u64, now: u64) -> Result { - let entry = self.entries.get_mut(index).ok_or(EINVAL)?; - if entry.render_kill.is_some() { - return Err(EINVAL); - } - entry.render_kill = Some(RenderKill::new(cookie, now)); - Ok(()) - } - - /// Ring the doorbell for published render-only kills not yet announced. A failed - /// doorbell is retried on a later pass and never duplicates the record. - pub(crate) fn announce_render_kills(&mut self, mut notify: impl FnMut() -> Result) { - for entry in self.entries.iter_mut() { - if let Some(kill) = entry.render_kill.as_mut() { - if !kill.announced && !kill.acknowledged && notify().is_ok() { - kill.announced = true; - } - } - } - } - - /// Only the exact cookie of an unacknowledged render-only kill is accepted. - pub(crate) fn acknowledge_render_kill(&mut self, cookie: u64) -> bool { - self.entries - .iter_mut() - .filter_map(|entry| entry.render_kill.as_mut()) - .any(|kill| kill.acknowledge(cookie)) - } - - /// A render-only kill awaits its acknowledgement within the polling window. - pub(crate) fn render_kill_polling(&self, now: u64, window: u64) -> bool { - self.entries - .iter() - .any(|entry| entry.render_kill.is_some_and(|kill| kill.awaiting(now, window))) - } - - /// The render-only kill of the entry at `index` waits; how long, and whether this is the - /// first time it crossed `slow`. - pub(crate) fn note_kill_wait(&mut self, index: usize, now: u64, slow: u64) -> (u64, bool) { - let Some(entry) = self.entries.get_mut(index) else { - return (0, false); - }; - let (since, warned) = entry.kill_wait.get_or_insert((now, false)); - let waited = now.saturating_sub(*since); - let crossed = waited >= slow && !*warned; - *warned |= crossed; - (waited, crossed) - } - - /// The render-only kill was published: how long it waited, if it did. - pub(crate) fn take_kill_wait(&mut self, index: usize, now: u64) -> u64 { - self.entries - .get_mut(index) - .and_then(|entry| entry.kill_wait.take()) - .map_or(0, |(since, _)| now.saturating_sub(since)) - } - - /// A render-only kill of the context at `index` was refused: it no longer waits, and this - /// returns whether it is the first refusal for this context. - pub(crate) fn refuse_render_kill(&mut self, index: usize) -> bool { - let Some(entry) = self.entries.get_mut(index) else { - return false; - }; - entry.kill_wait = None; - !core::mem::replace(&mut entry.kill_refused, true) - } - - /// The context at `index` is not waiting for a render-only kill (any more). - pub(crate) fn clear_kill_wait(&mut self, index: usize) { - if let Some(entry) = self.entries.get_mut(index) { - entry.kill_wait = None; - } - } - - /// How long each waiting render-only kill has waited. - pub(crate) fn kill_waits(&self, now: u64) -> impl Iterator + '_ { - self.entries - .iter() - .filter_map(move |entry| entry.kill_wait.map(|(since, _)| now.saturating_sub(since))) - } - /// Poll for a bounded window at close and again at actual release publication. /// Failed announcements remain retryable on later service even after polling /// expires. Time passage never substitutes for a firmware-consumption witness. @@ -372,21 +227,3 @@ impl Pending { }) } } - -#[kernel::macros::kunit_tests(asahi_g17_render_kill)] -mod render_kill_tests { - use super::RenderKill; - - #[test] - fn only_the_exact_cookie_acknowledges_a_render_kill_once() { - let mut kill = RenderKill::new(7, 1_000); - assert!(kill.awaiting(1_000, 10)); - assert!(!kill.acknowledge(0)); - assert!(!kill.acknowledge(8)); - assert!(kill.awaiting(1_009, 10)); - assert!(!kill.awaiting(1_010, 10)); - assert!(kill.acknowledge(7)); - assert!(!kill.acknowledge(7)); - assert!(!kill.awaiting(1_001, 10)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ce003e6b18196..e76dab07f56db 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,100 +6,11 @@ pub(crate) const MAX_RENEWALS: u8 = 8; -/// A borrowed render graph or queued off-lock retirement is not evidence of -/// progress. Share the finite deferral bound with cross-VM attribution. -pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } - -/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. -/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four -/// extra polls give it ten seconds, still one tenth of the legacy scheduler's -/// fallback. This is an execution allowance, not evidence of GPU progress, and -/// is consumed once per scheduler job, independently of cross-VM/replay grace. -pub(crate) const POLL_MS: u32 = 2000; -const EXECUTION_RENEWALS: u8 = 4; - -/// How long an innocent client's publication waits for a recovery to settle and its -/// spared work to drain, instead of failing with EBUSY: the watchdog defers work behind -/// another VM at most this long before it settles it (`classify`; ray work gets the -/// execution renewals on top). -pub(crate) const RECOVERY_WAIT_MS: u32 = - POLL_MS * (1 + EXECUTION_RENEWALS as u32 + MAX_RENEWALS as u32); - -/// A background control/replay poll has no scheduler fence to expire it. -/// Its timestamp belongs to the exact transaction and is never renewed by work. -pub(crate) fn background_stalled(since: Option, now: u64) -> bool { - since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) -} - -/// Input comes only from fully validated commands. Render metadata covers the -/// VDM half (vertex/tessellation/geometry) and the fragment half independently. -/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. -pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { - compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build -} - -/// Firmware dependencies can hold an untagged helper behind a long command on -/// the other engine. Classify the complete validated ioctl before enqueueing -/// anything; no state or allowance leaks into a later submission. -pub(crate) fn select_submission(commands: impl Iterator) -> bool { - commands.into_iter().any(|enabled| enabled) -} - -/// Only the explicit AS-build bit is newly accepted; reserved and unrelated -/// flags remain invalid on G17P. Return None before any command is published. -pub(crate) fn as_build(flags: u32) -> Option { - const AS_BUILD: u32 = 1 << 2; - if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } -} - -pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { - if !published || !enabled || *renewals >= EXECUTION_RENEWALS { - return false; - } - *renewals += 1; - true -} - -/// Polls a published job may be renewed while the GPU keeps retiring work. A job -/// shares the GPU's cores with every other runnable queue, so its wall time grows -/// with the other clients' work: beside four other clients a 1.5 s dispatch took up -/// to 5 s, and lower-priority work waits behind higher-priority work. It has not -/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at -/// thirty seconds, after which a job that still has not finished fails as before. -const SHARED_RENEWALS: u8 = 14; - -/// The GPU counts as live while some command retired this recently. Under heavy -/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires -/// nothing for this long has stopped. -pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; - -/// Whether some command retired within [`LIVENESS_NS`] (`since_retired`). -pub(crate) fn retiring(since_retired: Option) -> bool { - since_retired.is_some_and(|since| since < LIVENESS_NS) -} - -/// Renews a published job's deadline while the GPU is `shared`, at most -/// [`SHARED_RENEWALS`] times. It is shared while some command retires -/// ([`retiring`]), or while a long job of another queue runs beside a long job: -/// two ray traversal dispatches of about 6 s each took 10 to 17 s side by side, and -/// neither retires anything until it ends. -pub(crate) fn renew_shared(published: bool, shared: bool, renewals: &mut u8) -> bool { - if !published || !shared || *renewals >= SHARED_RENEWALS { - return false; - } - *renewals += 1; - true -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. - pub(crate) vm: u64, + pub(crate) vm: usize, pub(crate) published: u64, pub(crate) finished: bool, - /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains - /// it. No later watchdog of its own will settle it. - pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -109,187 +20,23 @@ pub(crate) enum Decision { Spare, } -/// Another VM's older unfinished work that can block `own`: live work before work whose queue -/// already failed, the oldest of each first. -pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { - others - .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) - .min_by_key(|other| (other.failed, other.published)) -} - -/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that -/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind -/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only -/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame -/// published before the culprit, which timed out first, would spare the culprit, and a pair that -/// never drains would spare every later timeout. -pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { - match blocker { - _ if own.finished => Decision::Own, - None => Decision::Own, - Some(_) if renewals < MAX_RENEWALS => Decision::Defer, - Some(blocker) if blocker.failed => Decision::Own, - Some(_) => Decision::Spare, - } -} - -/// A watchdog's decision with the heads it rests on, for its log line. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Verdict { - pub(crate) decision: Decision, - pub(crate) own: Head, - pub(crate) blocker: Option, -} - -impl Verdict { - /// The blocker a watchdog line names: at the first deferral and once the grace has run out. - pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { - self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) - } -} - -pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { - let blocker = blocker(own, others); - Verdict { - decision: decide(own, renewals, blocker), - own, - blocker, +pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + if own.finished || !others.into_iter().any(|other| { + other.vm != own.vm && !other.finished && other.published < own.published + }) { + Decision::Own + } else if renewals < MAX_RENEWALS { + Decision::Defer + } else { + Decision::Spare } } -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] +#[cfg(test)] mod tests { use super::*; - #[test] - fn selects_compute_both_render_halves_and_builds() { - assert!(!select(false, [false; 2], false)); - assert!(select(true, [false; 2], false)); - assert!(select(false, [true, false], false)); - assert!(select(false, [false, true], false)); - assert!(select(false, [true; 2], false)); - assert!(select(false, [false; 2], true)); - } - #[test] - fn helpers_share_only_the_current_submission_allowance() { - assert!(select_submission([false, true, false].into_iter())); - assert!(select_submission([true, false].into_iter())); - assert!(select_submission([false, true].into_iter())); - assert!(!select_submission([false, false].into_iter())); - assert!(!select_submission([].into_iter())); - let mut helper = 0; - for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } - assert!(!renew_execution(true, true, &mut helper)); - assert!(!renew_execution(true, false, &mut 0)); - } - #[test] - fn build_hint_rejects_reserved_and_unknown_flags() { - assert_eq!(as_build(0), Some(false)); - assert_eq!(as_build(1 << 2), Some(true)); - for bit in 0..32 { - if bit == 2 { continue; } - assert_eq!(as_build(1 << bit), None); - assert_eq!(as_build((1 << bit) | (1 << 2)), None); - } - assert_eq!(as_build(u32::MAX), None); - } - #[test] - fn borrowed_render_and_queued_retirement_cannot_defer_forever() { - for count in 0..=u8::MAX { - assert_eq!(transaction_grace(count), count < MAX_RENEWALS); - } - } - #[test] - fn background_control_and_replay_polls_have_an_absolute_deadline() { - let since = 42; - assert!(!background_stalled(None, u64::MAX)); - assert!(!background_stalled(Some(since), since - 1)); - assert!(!background_stalled(Some(since), since + 9_999_999_999)); - for _ in 0..1024 { - assert!(background_stalled(Some(since), since + 10_000_000_000)); - } - assert!(background_stalled(Some(since), u64::MAX)); - } - fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - attribute(own, renewals, others).decision - } - fn head(vm: u64, published: u64, finished: bool) -> Head { - Head { vm, published, finished, failed: false } - } - fn failed(vm: u64, published: u64) -> Head { - Head { vm, published, finished: false, failed: true } - } - #[test] - fn ray_query_execution_is_bounded_at_ten_seconds() { - let mut renewals = 0; - for poll in 1..=4 { - assert!(renew_execution(true, true, &mut renewals)); - assert_eq!(renewals, poll); - } - for _ in 0..256 { - assert!(!renew_execution(true, true, &mut renewals)); - } - assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); - } - #[test] - fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { - for (published, enabled) in [(false, false), (false, true), (true, false)] { - let mut renewals = 0; - assert!(!renew_execution(published, enabled, &mut renewals)); - assert_eq!(renewals, 0); - } - } - #[test] - fn ray_query_and_cross_vm_credits_are_independent_and_finite() { - let mut ray = 0; - let own = head(5, 101, false); - for _ in 0..4 { - assert!(renew_execution(true, true, &mut ray)); - } - for victim in 0..MAX_RENEWALS { - assert!(!renew_execution(true, true, &mut ray)); - assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); - } - assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); - assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); - } - #[test] - fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { - let mut renewals = 0; - for poll in 1..=14u8 { - assert!(renew_shared(true, retiring(Some(4_900_000_000)), &mut renewals)); - assert_eq!(renewals, poll); - } - assert!(!renew_shared(true, true, &mut renewals)); - assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); - } - #[test] - fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { - let mut renewals = 0; - assert!(!renew_shared(true, retiring(Some(LIVENESS_NS)), &mut renewals)); - assert!(!renew_shared(true, retiring(None), &mut renewals)); - assert!(!renew_shared(false, true, &mut renewals)); - assert_eq!(renewals, 0); - } - #[test] - fn a_long_job_beside_another_shares_the_gpu_without_retirements() { - // Two traversal dispatches side by side: nothing retires, yet both progress. - let mut renewals = 0; - let long_job_beside = true; - let shared = retiring(Some(LIVENESS_NS + 1)) || long_job_beside; - for poll in 1..=14u8 { - assert!(renew_shared(true, shared, &mut renewals)); - assert_eq!(renewals, poll); - } - // The thirty-second bound still ends a pair that never finishes. - assert!(!renew_shared(true, true, &mut renewals)); - } - #[test] - fn exhausted_or_invalid_credit_never_wraps() { - for value in 4..=u8::MAX { - let mut renewals = value; - assert!(!renew_execution(true, true, &mut renewals)); - assert_eq!(renewals, value); - } + fn head(vm: usize, published: u64, finished: bool) -> Head { + Head { vm, published, finished } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -305,53 +52,6 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] - fn a_blocker_that_already_failed_defers_then_never_spares() { - // An innocent frame published before the culprit timed out first and failed; its - // quarantined pair keeps its head until the firmware drains it. - let culprit = head(7, 101, false); - for count in 0..MAX_RENEWALS { - assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); - } - assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); - assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); - // A pair that never drains does not spare any later timeout either. - let drained_never = [failed(5, 100), failed(5, 102)]; - for own in [head(7, 105, false), head(9, 200, false)] { - assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); - } - } - #[test] - fn a_live_blocker_still_spares_beside_a_failed_one() { - let own = head(9, 103, false); - let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; - assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); - assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); - // Failed or not, a blocker must be older, unfinished and of another VM. - let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; - assert_eq!(blocker(own, late.into_iter()), None); - assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); - // Among failed blockers the oldest is reported. - let both_failed = [failed(8, 102), failed(5, 100)]; - assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); - } - #[test] - fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { - let own = head(9, 103, false); - let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); - let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; - assert_eq!(verdict, expected); - assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); - assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); - let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); - assert_eq!(last.decision, Decision::Own); - assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); - let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); - assert_eq!(spared.decision, Decision::Spare); - assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); - // Nothing blocks: nothing to report. - assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); - } - #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b0f2861382ad7..da43c10cd3bef 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,41 +8,19 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; -// The persistent proof must retain a deep queue's rotating command buffers. -// Keep the transient parser request set small to bound its stack frame. -const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); -/// Match the page footprint already checked by `Vm::covers_range_locked`. -fn pages((address, size, access): Request) -> Option { - if size == 0 || access == 0 || access & !3 != 0 { - return None; - } - let mask = mmu::UAT_PGSZ as u64 - 1; - let end = address.checked_add(size)?; - let end = end.checked_add(mask)? & !mask; - let start = address & !mask; - Some((start, end.checked_sub(start)?, access)) -} - -fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { - access & needed == needed - && other >= start - && other.checked_add(length).is_some_and(|end| - start.checked_add(size).is_some_and(|limit| end <= limit)) -} - #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; CAPACITY], +struct Requests { + ranges: [Request; MAX_RANGES], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); CAPACITY], + ranges: [(0, 0, 0); MAX_RANGES], count: 0, overflow: false, } @@ -52,55 +30,24 @@ impl Requests { &self.ranges[..self.count] } - fn covers(&self, request: Request) -> bool { - self.as_slice().iter().any(|&range| contains(range, request)) - } - fn insert(&mut self, request: Request) { - let Some(mut request) = pages(request) else { - self.overflow = true; - return; - }; - if self.covers(request) { + if self.as_slice().contains(&request) { return; } - // Merge only equal permissions, so an adjacent writable page never - // grants write access to a read-only page. Restart after each merge - // because it can join a range visited earlier. - let mut index = 0; - while index < self.count { - let range = self.ranges[index]; - let end = request.0 + request.1; - let other_end = range.0 + range.1; - if contains(request, range) - || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) - { - if request.2 == range.2 { - let start = request.0.min(range.0); - request = (start, end.max(other_end) - start, request.2); - } - self.count -= 1; - self.ranges[index] = self.ranges[self.count]; - index = 0; - } else { - index += 1; - } - } - if self.count == CAPACITY { + if self.count == MAX_RANGES { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } - } -/// One logical queue's bounded cache of validated page ranges and permissions. +/// One logical queue's bounded cache of exact address, size and permission tuples. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -117,7 +64,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|&request| self.requests.covers(request)) + .all(|request| self.requests.as_slice().contains(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -132,10 +79,7 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = Requests::new(); - for &request in requests.as_slice() { - self.requests.insert(request); - } + self.requests = requests; } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -146,11 +90,10 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, - variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -164,7 +107,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, variant, &Space::Checked(&range, vm)) + payload.validate(window, &Space::Checked(&range, vm)) } } @@ -200,101 +143,3 @@ impl AddressSpace for Space<'_> { } } } - - -#[kunit_tests(asahi_g17_validation_pages)] -mod tests { - use super::*; - const PAGE: u64 = mmu::UAT_PGSZ as u64; - - fn requests(range: Request) -> Requests { - let mut result = Requests::new(); - result.insert(range); - result - } - - #[test] - fn different_offsets_share_the_already_validated_page() { - let mut proof = Proof::new(); - proof.record(Some(7), requests((PAGE + 4, 4, 3))); - assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); - assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); - assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); - assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); - assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); - } - - #[test] - fn adjacent_equal_permissions_merge_but_holes_do_not() { - let mut ranges = requests((PAGE, 1, 1)); - ranges.insert((3 * PAGE, 1, 1)); - assert_eq!(ranges.count, 2); - assert!(!ranges.covers((2 * PAGE, PAGE, 1))); - ranges.insert((2 * PAGE, 1, 1)); - assert_eq!(ranges.count, 1); - assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); - } - - #[test] - fn adjacent_permissions_never_grant_access_to_another_page() { - let mut proof = Proof::new(); - let mut ranges = requests((PAGE, 1, 1)); - ranges.insert((2 * PAGE, 1, 3)); - proof.record(Some(1), ranges); - assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); - assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); - assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); - } - - #[test] - fn stronger_coverage_removes_only_fully_covered_ranges() { - let mut ranges = requests((PAGE, PAGE, 1)); - ranges.insert((2 * PAGE, PAGE, 2)); - ranges.insert((PAGE, 2 * PAGE, 3)); - assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); - assert!(!ranges.covers((0, PAGE, 1))); - } - - #[test] - fn persistent_cache_retains_a_deep_rotating_working_set() { - let mut proof = Proof::new(); - for index in 0..64 { - proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); - } - assert_eq!(proof.requests.count, 64); - for index in 0..64 { - assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); - } - assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); - } - - #[test] - fn persistent_overflow_keeps_only_a_new_complete_proof() { - let mut proof = Proof::new(); - for index in 0..=MAX_CACHED_RANGES { - proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); - } - assert_eq!(proof.requests.count, 1); - assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); - assert!(proof.covers(Some(7), - &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); - } - - #[test] - fn invalid_rounding_and_full_cache_disable_reuse() { - for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { - assert!(pages(range).is_none()); - let mut proof = Proof::new(); - proof.record(Some(1), requests(range)); - assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); - } - let mut ranges = Requests::new(); - for index in 0..=MAX_RANGES { - ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); - } - assert!(ranges.overflow); - let mut proof = Proof::new(); - proof.record(Some(1), ranges); - assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); - } -} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index f100820a0c75e..3bcf8a9692dcc 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,10 +21,8 @@ use kernel::{ }, // }, error::Result, - new_mutex, prelude::*, sync::aref::ARef, - sync::Mutex, uapi, // }; @@ -41,7 +39,8 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, memacct, mmu, + file, + mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -61,11 +60,6 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, - /// Memory accounting and process attribution of the held backing. - charge: memacct::Charge, - /// Serializes overlapping timestamp writes through every binding of this BO. - #[pin] - pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -80,9 +74,6 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. -/// -/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object -/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -108,7 +99,6 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } - gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -116,7 +106,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result> { +) -> Result<&'a kernel::scatterlist::SGTable> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -125,12 +115,17 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(gem) + { return Err(EINVAL); } Ok(()) } +unsafe impl Send for AsahiObject {} +unsafe impl Sync for AsahiObject {} + /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -216,7 +211,6 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, - charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -235,23 +229,20 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -pub(crate) fn checked_object_size(size: usize) -> Result { - if size == 0 { - return Err(EINVAL); - } +fn checked_object_size(size: usize) -> Result { + if size == 0 { return Err(EINVAL); } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } + fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { - let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - size, + checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, - pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -259,7 +250,6 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, - charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -269,17 +259,12 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. -/// -/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the -/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, - pages_no_reclaim: bool, - commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -291,7 +276,6 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, - pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -299,7 +283,6 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, - charge: memacct::Charge::user_current(commit), }, )?; @@ -325,22 +308,16 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), - charge: args.charge, - timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. - /// - /// Their memory is not counted: an import's backing belongs to, and is charged by, its - /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, - charge: memacct::Charge::none(), }) } @@ -395,33 +372,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { - /// Whether CPU mappings use the cacheable WRITEBACK policy. - pub(crate) fn is_writeback(&self) -> bool { - self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 - } - pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } } -/// Populates and pins a new user object through its DMA mapping, before any CPU view exists. -/// -/// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the -/// freshly zeroed pages to the point of coherency, so that no dirty line can later overwrite data -/// written through a write-combined view. The pin keeps reclaim from creating a new cacheable -/// alias. Only used when the GPU asks for it ([`crate::gpu::Gpu::prepare_new_objects`]). -/// -pub(crate) fn prepare_new_object( - gem: &Object, - dev: &kernel::device::Device, -) -> Result { - // The native backend requires a non-coherent DMA device. Use its DMA mapping - // rather than interpreting DMA addresses as CPU physical addresses. - let _sgt = held_sg_table(gem, dev)?; - Ok(()) -} - /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { @@ -431,26 +386,6 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { Ok(u32::from(gem.madvise(i32::from(dontneed)))) } -/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. -/// Keep ordinary resource allocations and kernel-owned backing on their existing -/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. -pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { - g17 && size >= 256 * 1024 * 1024 -} - -#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] -mod allocation_policy_tests { - use super::*; - - #[test] - fn large_user_backing() { - assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); - assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); - assert!(user_pages_no_reclaim(3_257_113_344, true)); - assert!(!user_pages_no_reclaim(3_257_113_344, false)); - } -} - /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the @@ -461,7 +396,7 @@ mod allocation_policy_tests { /// The DMA mapping only cleans if the device is not marked DMA-coherent (G15's synthesized node /// has no `dma-coherent`). Should it ever be marked so, the pages are cleaned here instead. #[cfg(CONFIG_DRM_ASAHI_G15)] -pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { +pub(crate) fn prepare_new_object(gem: &Object) -> Result { // SAFETY: G15 objects belong to the retained bound platform device; user object // creation occurs only after registration and before a file can outlive its device. let dev: &kernel::device::Device = gem.dev().as_ref().as_ref(); @@ -498,7 +433,9 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } + #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { - fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; -} +extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 3da16c10696e3..a06b5da5ed6b1 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,6 +11,7 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. +use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -35,7 +36,6 @@ use kernel::{ }, macros::versions, new_mutex, - new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,7 +45,6 @@ use kernel::{ }, Arc, Mutex, - SpinLock, UniqueArc, // }, time::{ @@ -86,7 +85,6 @@ use crate::{ queue, regs, stats, - stats_raw, workqueue, // }; @@ -261,30 +259,18 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, - /// Last raw firmware statistics of this device, exported read-only - /// through `stats_raw`. - raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, + #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>, + garbage_contexts: Mutex>>>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { - /// Admission bound checked before copying the userspace command stream. - fn max_cmdbuf_size(&self) -> usize { - usize::MAX - } - /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; - /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. - fn raw_stats_snapshot(&self) -> Option>> { - None - } - /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -299,10 +285,6 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; - /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). - fn supports_soft_faults(&self) -> bool { - crate::module_parameters::fault_control.value() == 0xb - } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -315,14 +297,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; } if self.queue_limits().is_some() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && self.supports_soft_faults() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); + if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags } @@ -330,38 +312,35 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Backend-specific user address aperture; other GPUs keep their geometry's range. fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which - /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against - /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before - /// resolving syncs because the queue installs a fence in every output sync even on failure. + /// Use tracked command publication and sticky VM status on this backend. fn tracked_submission(&self) -> bool { false } - /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. + /// Whether faults may be reported as recoverable to userspace. fn soft_faults(&self) -> bool { true } - /// Whether new user objects must be populated, cleaned to the point of coherency and pinned - /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose - /// default CPU mappings are write-combined). + /// Populate and clean fresh object backing before any WC CPU mapping. fn prepare_new_objects(&self) -> bool { false } - /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also - /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at - /// most a page (EINVAL otherwise). `None` keeps the shared rules. + /// Optional backend-specific upper bound for a VM_BIND batch. fn bind_batch_limit(&self) -> Option { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Admit a client's parameter ABI before creating its VM. + /// A userspace VM handle was closed; release backend-only registry references. + fn vm_closed(&self) {} + /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS + /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 + /// T8122 runtime admits any client. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -371,13 +350,6 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Create a VM charged to a DRM file when the backend bounds retained bindings. - fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { - self.new_vm(kernel_range) - } - /// Release the backend's handle owner after closing user mapping admission. - /// Accepted jobs and firmware resource leases retain their own references. - fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -386,14 +358,13 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result>; - /// Bind a BO range as a firmware- or CPU-written timestamp buffer. + /// Map a BO as a timestamp buffer fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -435,8 +406,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context, with the objects it retains, for invalidation. - fn free_context(&self, ctx: workqueue::RetiredContext); + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); } /// Private generic trait for functions that don't need to escape this module. @@ -781,7 +752,6 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, - mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -846,12 +816,6 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); - // Per-device raw statistics snapshot, shared with the debugfs export. - let raw_stats_snap = Arc::pin_init( - new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), - GFP_KERNEL, - )?; - let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -867,7 +831,6 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, - raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -895,7 +858,6 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), - raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1127,7 +1089,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(u64::from(map.writable)), + readwrite: U64(map.writable as u64), }; }); @@ -1177,14 +1139,20 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let guard = self.dev.registration_guard()?; + let Some(guard) = self.dev.registration_guard() else { + return None; + }; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if let Some(fault) = &info { - dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); } info }) @@ -1351,10 +1319,6 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } - fn raw_stats_snapshot(&self) -> Option>> { - Some(self.raw_stats.clone()) - } - fn as_any(&self) -> &dyn Any { self } @@ -1437,7 +1401,6 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1455,7 +1418,6 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, - usage, )?, GFP_KERNEL, )?) @@ -1465,15 +1427,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(bo.map_range_into_range( + ) -> Result { + bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - )?) + ) } } @@ -1498,7 +1460,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx.data).is_err() { + if self.invalidate_context(&ctx).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1507,9 +1469,6 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } - // The firmware no longer knows the context's queues, so their structures may go now, - // ahead of the context data they point to. - core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1778,7 +1737,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: workqueue::RetiredContext) { + fn free_context(&self, ctx: KBox>) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 2650131a4a293..deffe3fe47d0b 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,7 +13,6 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; -pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -24,7 +23,6 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, - G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs deleted file mode 100644 index c2e7c06f67b7f..0000000000000 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t6040 platforms (M4 Pro). -//! -//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's -//! interface; [`fw::variant::V25G83`] describes where it differs. - -use crate::g17::fw::{ - self, - initdata::{PerfConfig, RegionCConfig, RegisterWindow}, -}; -use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; -use crate::hw::{GpuGen, GpuVariant}; -use crate::mmu::UatQuirks; - -/// Static configuration of the T6040 GPU. -pub(crate) const CONFIG: Config = Config { - chip_id: CHIP_ID, - identity: Identity { - family: 0xa, - // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). - variant: 3, - num_dies: 1, - gpu_variant: GpuVariant::S, - usc_generation: 3, - hal_generation: 200, - }, - uat_ias: 42, - uat_oas: 42, - // One coprocessor carries both firmware roles; `pagetables` holds only its root. - secondary_root_offset: None, - crash_buffers: CrashBuffers::FirmwareData, - // Read by the firmware through the kernel lower root, whose top-level entries it only - // samples when it starts. - pm_metrics: FixedBuffer { - va: 0x10_0008_0000, - size: 0x8000, - }, - windows: ®ISTER_WINDOWS, - reserved_slots: &[], - perf: &PERF, - opps: OPPS, - // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The - // rest is the policy the M4 Pro has run with; none of it was tested otherwise. - uat_quirks: UatQuirks { - reserved_tables: true, - clean_tables: true, - page_tlbi: true, - handoff_lock: false, - handoff_slot: 0xffff_ffff, - native_gpu_permissions: true, - }, - region_c: ®ION_C, - adopt_root_entries: false, - fabric_bridge: false, - // The firmware's per-slot recovery keys are not known for this build; recovery relies on - // the firmware's own blame. - recovery_slot_keys: None, - // Userspace drives this GPU as a G16 part. - gpu_generation: GpuGen::G16, - // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. - firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { - os_version: [26, 6, 2], - name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", - images: &[ - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, - 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, - 0x10, 0xe0, 0x63, 0x87, 0x6d, 0xcd, - ], - }, - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, - 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, - 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, - ], - }, - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, - 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, - 0x37, 0xee, 0x5d, 0x6e, 0x5e, 0x98, - ], - }, - ], - variant: &fw::variant::V25G83, - }]), - // Measured on the 20-core M4 Pro; the alternative below remains experimental. - core_layout: CoreLayout { - cores: Some((2, 20)), - // Admit two clusters of ten slots with eight enabled cores in each cluster. - // Reusing the measured 20-core buffers on it remains experimental. - alternate_core_masks: Some((10, 8)), - operand_state: fw::compute::OperandState::TWO_BY_TEN, - deflake: fw::render::Deflake::TWO_CLUSTERS, - }, -}; - -/// Chip identifier written to the hardware data. -pub(crate) const CHIP_ID: u32 = 0x6040; - -/// Base of the firmware MMIO aperture in the kernel window. -const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; - -const fn window( - slot: usize, - phys: u64, - offset: u64, - size: u32, - unk_18: u64, - unk_20: u32, -) -> RegisterWindow { - RegisterWindow { - slot, - phys, - va: MMIO_APERTURE + offset, - size, - unk_18, - unk_20, - instances: 1, - phys_stride: 0, - } -} - -/// `window` repeated `instances` times, `phys_stride` apart in physical memory. -const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { - RegisterWindow { - instances, - phys_stride, - ..window - } -} - -/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory -/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. -pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ - window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), - instances( - window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), - 4, - 0x200_0000, - ), - window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), - window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), - window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), - window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), - window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), - window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), - window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), - window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), - instances( - window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), - 4, - 0x20_0000, - ), - window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), - window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), - window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), - window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), - window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), - window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), - window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), - window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), - window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), - window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), - window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), - window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), - window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), - window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), -]; - -/// Inert defaults. The 25G83 builder requires the loader's per-chip performance maps. -pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0; 11], - states_b: [0; 11], - relative_a: [0; 11], - relative_b: [0; 11], - sram_min_mv: 810, -}; - -/// Operating points the performance tables reference. -pub(crate) const OPPS: usize = 16; - -/// Region C. -const REGION_C: RegionCConfig = RegionCConfig { - unk_54: 0x0fa0, - unk_78: [1, 0, 0], - max_power_mw: 0, - unk_9c: 0, - unk_a0: 0, - unk_7c4: 1500, - unk_9e0: [ - 1, - 0x0102_0200, - 0x010a_2801, - 0x2802_0101, - 0x0101_010a, - 0x010a_2802, - 0x2801_0101, - 0x0001_010a, - ], -}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index ea0f9cd782e81..143ba720d06f2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,11 +4,10 @@ use crate::g17::{ Config, - CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::{GpuGen, GpuVariant}; +use crate::hw::GpuVariant; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -24,10 +23,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: Some(0x40000), + secondary_root_offset: 0x40000, // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: CrashBuffers::Fixed([ + crash_buffers: [ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -36,31 +35,13 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ]), + ], // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, - recovery_slot_keys: Some(recovery::SLOT_KEYS), - gpu_generation: GpuGen::G17, - windows: ®ISTER_WINDOWS, - reserved_slots: &RESERVED_REGISTER_SLOTS, - perf: &PERF, - opps: OPPS, - uat_quirks: crate::mmu::UatQuirks::DEFAULT, - region_c: &crate::g17::fw::initdata::REGION_C_26A428, - adopt_root_entries: true, - fabric_bridge: true, - firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), - // The layouts the A18 Pro has always run with. - core_layout: crate::g17::CoreLayout { - cores: None, - alternate_core_masks: None, - operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, - deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, - }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -86,8 +67,6 @@ const fn window( size, unk_18, unk_20, - instances: 1, - phys_stride: 0, } } @@ -294,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; + pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -405,9 +384,6 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; - /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the - /// policy word of the user classes is kept (no macOS trace of class 1 yet). - pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -417,11 +393,8 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, - // as many as a pair has command records (the render memory's RECORDS). - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; - pub(crate) const RENDER_DEPTH: u8 = 32; + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ffc57c77afcb2..ca16e18f10fbf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -43,9 +43,6 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, - /// Set before a batch with any of this packet's commands can change firmware-visible state, - /// and never cleared. - published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -81,10 +78,6 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} - /// Called before the first firmware-visible change of a batch that includes this packet. - pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} - /// Whether the firmware may have seen any of this packet's commands. - pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -239,7 +232,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 79071cbc434a0..7a5587ef6ec78 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,39 +15,9 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; -use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; -/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. -/// -/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per -/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second -/// UAT with another policy would change it for both. -static PAGE_TLBI: AtomicBool = AtomicBool::new(false); - -/// Ranges above this many pages are invalidated by ASID in page mode. -const PAGE_TLBI_MAX_PAGES: usize = 2048; - -/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). -pub(crate) fn set_page_tlbi(enabled: bool) { - PAGE_TLBI.store(enabled, Ordering::Relaxed); -} - -/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above -/// [`PAGE_TLBI_MAX_PAGES`]. -fn tlbi_pages(asid: Asid, va: usize, len: usize) { - let start = va >> mmu::UAT_PGBIT; - let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; - if end - start > PAGE_TLBI_MAX_PAGES { - tlbi_asid(asid); - return; - } - for page in start..end { - tlbi_page(asid, page << mmu::UAT_PGBIT); - } -} - /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -71,7 +41,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) (u64::from(asid) << 48) + x = in(reg) ((asid as u64) << 48) ); } } @@ -85,7 +55,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -108,16 +78,12 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } - if PAGE_TLBI.load(Ordering::Relaxed) { - tlbi_pages(asid, va, len); - return; - } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -147,7 +113,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= u64::from(exp) << 44; + val |= (exp as u64) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition @@ -201,10 +167,6 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } - if PAGE_TLBI.load(Ordering::Relaxed) { - tlbi_pages(asid, va, len); - return; - } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -223,7 +185,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = (u64::from(asid) << 48) + let val = ((asid as u64) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs deleted file mode 100644 index 12fc90b23d8eb..0000000000000 --- a/drivers/gpu/drm/asahi/memacct.rs +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU memory attribution, accounting and admission limits. -//! -//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the -//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of -//! the task that faulted it in, but the OOM killer scores a task by the resident set -//! of its mm, and GPU-only pages are mapped into no process. A game holding -//! gigabytes of buffers therefore scores like a tiny process, and so does a -//! compositor; which one an OOM kill hits is left to the other terms of the score. -//! -//! This module closes that gap: -//! -//! - Every user object remembers the mm of the process that created it (an `mmgrab` -//! reference, which keeps only the `mm_struct` itself). While the GPU holds the -//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. -//! They are shmem pages, so this is where `oom_badness()`, the OOM report's -//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages -//! are uncharged when idle reclaim or purge releases the backing and when the -//! object is freed; the mm reference is dropped after the last uncharge, so the -//! counter is back to its own value before `__mmdrop()` checks it. -//! - Each client (DRM file) and the device keep totals of created and held bytes, -//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the -//! `gpu_memory` sysfs file next to `agx_stats`. -//! - A new user object is admitted only while its client stays within a per-client -//! limit and the device within a device-wide limit, both fractions of RAM. -//! A refused allocation fails with `ENOMEM`, which userspace reports as -//! out-of-device-memory, instead of pushing the machine into a global OOM. -//! -//! Imported dma-bufs are charged to their exporter and are not counted here. -//! Kernel objects (firmware structures, per-queue buffers and the global heaps) -//! count towards the device total and its limit, but are attributed to no process. - -use core::sync::atomic::{ - AtomicBool, - AtomicU64, - Ordering, // -}; - -use kernel::{ - mm::{ - Mm, - RssCounter, // - }, - page::PAGE_SIZE, - prelude::*, - sync::{aref::ARef, Arc}, -}; - -use crate::module_parameters; - -/// Device-wide GPU memory totals, in bytes. -/// -/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). -#[repr(C)] -pub(crate) struct Ledger { - /// Size of every live user object (created, not imported). - pub(crate) user_committed: AtomicU64, - /// Size of the live user objects whose backing the GPU currently holds. - pub(crate) user_resident: AtomicU64, - /// Size of the live kernel objects whose backing the GPU holds. - pub(crate) kernel_resident: AtomicU64, - /// User object creations refused by a limit. - pub(crate) refused: AtomicU64, - /// Device-wide limit on user committed plus kernel resident bytes (0: none). - pub(crate) limit: AtomicU64, - /// Per-client limit on committed bytes (0: none). - pub(crate) client_limit: AtomicU64, -} - -// Keep the Rust half of the C sysfs layout contract checked as well. -const _: () = { - assert!(core::mem::offset_of!(Ledger, refused) == 24); - assert!(core::mem::offset_of!(Ledger, client_limit) == 40); - assert!(core::mem::size_of::() == 48); -}; - -/// The ledger of the (single) AGX GPU. -pub(crate) static LEDGER: Ledger = Ledger::new(); - -/// The limits that follow from the amount of RAM and two percentages. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct Limits { - /// Device-wide limit in bytes (0: none). - pub(crate) device: u64, - /// Per-client limit in bytes (0: none). - pub(crate) client: u64, -} - -fn pct_of(total: u64, pct: u32) -> u64 { - match pct { - 0 => 0, - // Above 100% cannot be held anyway; treat it as "no limit". - p if p >= 100 => 0, - p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, - } -} - -/// Computes the limits for `total_ram` bytes of RAM. -/// -/// A percentage of 0 or 100 and above disables that limit. A per-client limit -/// is never larger than the device limit. -pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { - let device = pct_of(total_ram, device_pct); - let mut client = pct_of(total_ram, client_pct); - if device != 0 && (client == 0 || client > device) { - client = device; - } - Limits { device, client } -} - -/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). -/// -/// Returns whether the bytes were added. Concurrent callers cannot overshoot. -fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { - let mut cur = counter.load(Ordering::Relaxed); - loop { - let Some(new) = cur.checked_add(bytes) else { - return false; - }; - if limit != 0 && new.saturating_add(extra) > limit { - return false; - } - match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { - Ok(_) => return true, - Err(actual) => cur = actual, - } - } -} - -fn sub(counter: &AtomicU64, bytes: u64) { - let prev = counter.fetch_sub(bytes, Ordering::AcqRel); - if prev < bytes { - pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); - counter.store(0, Ordering::Relaxed); - } -} - -impl Ledger { - pub(crate) const fn new() -> Self { - Self { - user_committed: AtomicU64::new(0), - user_resident: AtomicU64::new(0), - kernel_resident: AtomicU64::new(0), - refused: AtomicU64::new(0), - limit: AtomicU64::new(0), - client_limit: AtomicU64::new(0), - } - } - - /// Sets the limits from the module parameters and the amount of RAM. - pub(crate) fn configure(&self) { - // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. - let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; - // SAFETY: `si_meminfo()` only fills in the structure it is given. - unsafe { kernel::bindings::si_meminfo(&mut info) }; - let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); - let limits = limits_for( - total, - module_parameters::mem_limit_pct.value(), - module_parameters::client_mem_limit_pct.value(), - ); - self.set_limits(limits); - pr_info!( - "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", - limits.device >> 20, - limits.client >> 20 - ); - } - - /// Admission capacities exported in the optional GET_PARAMS tail. - pub(crate) fn limits(&self) -> Limits { - Limits { - device: self.limit.load(Ordering::Relaxed), - client: self.client_limit.load(Ordering::Relaxed), - } - } - - pub(crate) fn set_limits(&self, limits: Limits) { - self.limit.store(limits.device, Ordering::Relaxed); - self.client_limit.store(limits.client, Ordering::Relaxed); - } - - /// Admits a new user object of `bytes` for `client`. - /// - /// Reserves the bytes against both limits; the returned [`Commit`] gives - /// them back when it is dropped. - pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { - let client_limit = self.client_limit.load(Ordering::Relaxed); - if !try_add_within(&client.committed, bytes, client_limit, 0) { - self.refused.fetch_add(1, Ordering::Relaxed); - return Err(ENOMEM); - } - // Kernel objects share the device limit but are never refused. - let kernel = self.kernel_resident.load(Ordering::Relaxed); - if !try_add_within( - &self.user_committed, - bytes, - self.limit.load(Ordering::Relaxed), - kernel, - ) { - sub(&client.committed, bytes); - self.refused.fetch_add(1, Ordering::Relaxed); - return Err(ENOMEM); - } - Ok(Commit { - ledger: self, - client: client.clone(), - bytes, - }) - } -} - -/// Per-client (DRM file) totals, in bytes. -pub(crate) struct Client { - /// Size of the live user objects this client created. - committed: AtomicU64, - /// Size of those whose backing the GPU currently holds. - resident: AtomicU64, -} - -impl Client { - pub(crate) fn new() -> Result> { - Ok(Arc::new( - Self { - committed: AtomicU64::new(0), - resident: AtomicU64::new(0), - }, - GFP_KERNEL, - )?) - } - - pub(crate) fn committed(&self) -> u64 { - self.committed.load(Ordering::Relaxed) - } - - pub(crate) fn resident(&self) -> u64 { - self.resident.load(Ordering::Relaxed) - } -} - -/// Bytes of a user object reserved against the limits. -pub(crate) struct Commit { - ledger: &'static Ledger, - client: Arc, - bytes: u64, -} - -impl Drop for Commit { - fn drop(&mut self) { - sub(&self.client.committed, self.bytes); - sub(&self.ledger.user_committed, self.bytes); - } -} - -enum Kind { - /// An object the DRM core created (a dma-buf import): charged to its exporter. - None, - /// A driver-owned object. - Kernel(&'static Ledger), - /// A user object, attributed to the process that created it. - User { - commit: Commit, - mm: Option>, - }, -} - -/// The accounting state of one GEM object. -pub(crate) struct Charge { - kind: Kind, - bytes: u64, - /// Whether the bytes are currently counted as held by the GPU. - held: AtomicBool, -} - -// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. -unsafe impl Send for Charge {} -// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. -unsafe impl Sync for Charge {} - -impl Charge { - /// An object that is not counted (a dma-buf import). - pub(crate) fn none() -> Self { - Self { - kind: Kind::None, - bytes: 0, - held: AtomicBool::new(false), - } - } - - /// A kernel object of `bytes`. - pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { - Self { - kind: Kind::Kernel(ledger), - bytes, - held: AtomicBool::new(false), - } - } - - /// A user object admitted by `commit`, attributed to `mm` if there is one. - pub(crate) fn user(commit: Commit, mm: Option>) -> Self { - let bytes = commit.bytes; - let mm = if module_parameters::oom_attribution.value() != 0 { - mm - } else { - None - }; - Self { - kind: Kind::User { commit, mm }, - bytes, - held: AtomicBool::new(false), - } - } - - /// A user object attributed to the current process. - pub(crate) fn user_current(commit: Commit) -> Self { - let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); - Self::user(commit, mm) - } - - fn pages(&self) -> isize { - isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) - } - - fn apply(&self, held: bool) { - match &self.kind { - Kind::None => {} - Kind::Kernel(ledger) => { - if held { - ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); - } else { - sub(&ledger.kernel_resident, self.bytes); - } - } - Kind::User { commit, mm } => { - if held { - commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); - commit - .ledger - .user_resident - .fetch_add(self.bytes, Ordering::AcqRel); - } else { - sub(&commit.client.resident, self.bytes); - sub(&commit.ledger.user_resident, self.bytes); - } - if let Some(mm) = mm { - let pages = self.pages(); - mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); - } - } - } - } - - /// Records that the GPU now holds (or no longer holds) the object's backing. - /// - /// Idempotent: only a change of state is counted. - pub(crate) fn set_held(&self, held: bool) { - if self.held.swap(held, Ordering::AcqRel) != held { - self.apply(held); - } - } -} - -impl Drop for Charge { - fn drop(&mut self) { - // Uncharge before `kind` (and with it the mm reference) is dropped. - self.set_held(false); - } -} - -#[kunit_tests(asahi_memacct)] -mod tests { - use super::*; - - const GIB: u64 = 1 << 30; - const MIB: u64 = 1 << 20; - - #[test] - fn default_limits_follow_ram() { - // The module parameter defaults. - let l = limits_for(8 * GIB, 67, 50); - assert_eq!(l.device, 8 * GIB * 67 / 100); - assert_eq!(l.client, 4 * GIB); - } - - #[test] - fn disabled_and_clamped_limits() { - assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); - assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); - // A client limit above the device limit is clamped to it. - let l = limits_for(8 * GIB, 25, 50); - assert_eq!(l.client, l.device); - // No client limit with a device limit means the device limit. - let l = limits_for(8 * GIB, 50, 0); - assert_eq!(l.client, 4 * GIB); - // No device limit leaves the client limit alone. - let l = limits_for(8 * GIB, 0, 25); - assert_eq!(l, Limits { device: 0, client: 2 * GIB }); - } - - #[test] - fn percentages_do_not_overflow() { - let l = limits_for(u64::MAX, 99, 98); - assert!(l.device > l.client); - assert!(l.device <= u64::MAX / 100 * 99 + 99); - } - - #[test] - fn try_add_respects_limit_and_extra() { - let c = AtomicU64::new(0); - assert!(try_add_within(&c, 6, 10, 0)); - assert!(!try_add_within(&c, 5, 10, 0)); - assert!(try_add_within(&c, 4, 10, 0)); - assert_eq!(c.load(Ordering::Relaxed), 10); - let d = AtomicU64::new(0); - assert!(!try_add_within(&d, 6, 10, 5)); - assert!(try_add_within(&d, 5, 10, 5)); - let e = AtomicU64::new(u64::MAX - 1); - assert!(!try_add_within(&e, 2, 0, 0)); - assert!(try_add_within(&e, 1, 0, 0)); - } - - static ADMIT: Ledger = Ledger::new(); - - #[test] - fn admission_reserves_and_releases() -> Result { - ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); - let a = Client::new()?; - let b = Client::new()?; - let c = Client::new()?; - let a1 = ADMIT.admit(&a, 40 * MIB)?; - // The client limit refuses a's second 40 MiB; nothing stays reserved. - assert!(ADMIT.admit(&a, 40 * MIB).is_err()); - assert_eq!(a.committed(), 40 * MIB); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); - let b1 = ADMIT.admit(&b, 50 * MIB)?; - // The device limit refuses c's 20 MiB and rolls back c's reservation. - assert!(ADMIT.admit(&c, 20 * MIB).is_err()); - assert_eq!(c.committed(), 0); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); - assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); - drop(a1); - assert_eq!(a.committed(), 0); - let c1 = ADMIT.admit(&c, 20 * MIB)?; - drop((b1, c1)); - assert_eq!(b.committed(), 0); - assert_eq!(c.committed(), 0); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); - Ok(()) - } - - static KERNEL: Ledger = Ledger::new(); - - #[test] - fn kernel_objects_count_against_the_device_limit() -> Result { - KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); - let k = Charge::kernel(&KERNEL, 48 * MIB); - k.set_held(true); - let a = Client::new()?; - assert!(KERNEL.admit(&a, 32 * MIB).is_err()); - let c = KERNEL.admit(&a, 16 * MIB)?; - drop(k); - assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); - drop(c); - Ok(()) - } - - static HELD: Ledger = Ledger::new(); - - #[test] - fn held_transitions_are_idempotent() -> Result { - HELD.set_limits(Limits { device: 0, client: 0 }); - let a = Client::new()?; - let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); - assert!(!charge.held.load(Ordering::Relaxed)); - charge.set_held(true); - charge.set_held(true); - assert_eq!(a.resident(), 8 * MIB); - assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); - charge.set_held(false); - charge.set_held(false); - assert_eq!(a.resident(), 0); - charge.set_held(true); - // Dropping a held object uncharges it and returns its commit. - drop(charge); - assert_eq!(a.resident(), 0); - assert_eq!(a.committed(), 0); - assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); - assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); - Ok(()) - } - - #[test] - fn imports_are_not_counted() { - let charge = Charge::none(); - charge.set_held(true); - assert!(charge.held.load(Ordering::Relaxed)); - drop(charge); - } - - static MM: Ledger = Ledger::new(); - - // `mm_alloc()` is only exported to modules in the KUnit namespace. - #[cfg(not(MODULE))] - #[test] - fn held_pages_are_charged_to_the_creating_mm() -> Result { - MM.set_limits(Limits { device: 0, client: 0 }); - // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. - let raw = unsafe { kernel::bindings::mm_alloc() }; - if raw.is_null() { - return Err(ENOMEM); - } - // SAFETY: `raw` is a live mm until the `mmput()` below. - let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); - let a = Client::new()?; - let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - charge.set_held(true); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); - assert_eq!(mm.rss_counter(RssCounter::Anon), 0); - charge.set_held(false); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - charge.set_held(true); - // Freeing a held object uncharges it before dropping its mm reference. - drop(charge); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - drop(mm); - // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then - // checks that every RSS counter is back to zero. - unsafe { kernel::bindings::mmput(raw) }; - Ok(()) - } -} - -#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] -mod exported_limits_tests { - use super::*; - - #[test] - fn exported_capacities_match_admission() { - let ledger = Ledger::new(); - assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); - let limits = limits_for(8 << 30, 67, 50); - ledger.set_limits(limits); - assert_eq!(ledger.limits(), limits); - assert!(limits.client < limits.device); - assert_eq!(limits.client, 4 << 30); - } -} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..2bf4a6823024d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::{size_of, ManuallyDrop}; +use core::mem::size_of; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,6 +29,7 @@ use core::sync::atomic::{ use kernel::{ addr::{ + DmaAddr, PhysicalAddr, // }, c_str, @@ -108,19 +109,20 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; +use pin_init; + const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod shared; mod lifetime; -mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -mod shared; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -154,81 +156,23 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// One or two coprocessor instances that never touch the handoff region (G17). + /// Two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at - /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. - /// TTBAT context 1 starts as an alias of the kernel context. + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: Option, + secondary_root_offset: u64, }, } -/// UAT policy of a SoC where it differs from the A18 Pro's. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct UatQuirks { - /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and - /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of - /// `struct page`. - /// - /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the - /// regions lose `no-map` and this path goes away. - pub(crate) reserved_tables: bool, - // The following settings keep a SoC on the only policy it was tested with. Each one is - // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. - /// Write back every page-table and context-table change: the GPU table walker is not - /// assumed to snoop the CPU caches. - pub(crate) clean_tables: bool, - /// Invalidate ranges one page at a time, never with range TLBI operations. - pub(crate) page_tlbi: bool, - /// Take the handoff lock around context-table edits. Where the firmware leaves its half of - /// the handoff page alone, its lock bytes are not initialized and are never taken. - pub(crate) handoff_lock: bool, - /// Handoff "current slot" written at initialization. - pub(crate) handoff_slot: u32, - /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in - /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. - pub(crate) native_gpu_permissions: bool, -} - -impl UatQuirks { - /// The A18 Pro and earlier policy. - pub(crate) const DEFAULT: UatQuirks = UatQuirks { - reserved_tables: false, - clean_tables: false, - page_tlbi: false, - handoff_lock: true, - handoff_slot: 0, - native_gpu_permissions: false, - }; -} - const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// M4's nonzero wire incarnation of a host-owned root installation. -const fn binding_wire_generation(incarnation: u64) -> u8 { - if incarnation == 0 { - 0 - } else { - (1 + (incarnation - 1) % 255) as u8 - } -} - -/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in -/// context 0; the existing G15 handoff shares them with user contexts. -const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { - if map_kernel_to_user { - root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT - } else { - 0 - } -} - /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -414,28 +358,11 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -pub(crate) struct VmReadGuard(VmExecGuard); - -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -impl VmReadGuard { - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { - Vm::covers_range_locked(&mut self.0, address, size, read, write) - } - - pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { - Vm::read_user_locked(&mut self.0, va, buf) - } -} - /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, - /// Nonzero M4 wire incarnation of the root currently owning this slot. - generation: u8, ttb: u64, } @@ -448,20 +375,7 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>, -} - -/// Copied DMA addresses and the lease covering every alias of this VM/BO association. -/// The lease is dropped only after all aliases have been unmapped and invalidated. -struct SgSegments { - segments: KVVec<(DmaAddress, ResourceSize)>, - _lease: kernel::drm::gem::shmem::SGLease, -} - -impl SgSegments { - fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { - self.segments.iter() - } + segments: Mutex>>, } /// Preallocated resources needed to execute a VM operation. @@ -580,16 +494,22 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - if ctx.dev.is_none() { + let Some(dev) = ctx.dev else { + dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); return Err(ENODEV); - } - // Bind preparation retained the table's backing before taking the VM reservation. - // Never acquire the object reservation from inside the GPUVM map callback. - let segments = op.vm_bo().data().segments.lock(); - let sgt = segments.as_ref().ok_or(EFAULT)?; - for &(dma_address, dma_len) in sgt.iter() { - let mut addr = dma_address as usize; - let mut len = dma_len as usize; + }; + + let sgt = op.obj().sg_table(dev).inspect_err(|e| { + dev_err!( + self.dev.as_ref(), + "sm_step_map() Failed to get SG table: {:?}", + e + ); + })?; + for sgt_entry in sgt.iter() { + // TODO: proper DMA address/length handling + let mut addr = sgt_entry.dma_address() as usize; + let mut len: usize = sgt_entry.dma_len() as usize; if left == 0 { break; @@ -636,7 +556,6 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } - drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -722,8 +641,7 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table - .discard_partial_map(unmap_start..unmap_end)?; + self.page_table.discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -766,12 +684,7 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!( - self.dev, - "MMU: unmap: {:#x}:{:#x}\n", - va.addr(), - va.length() - ); + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -806,7 +719,14 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - self.finish_failed_map(addr, size as usize); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -816,19 +736,6 @@ impl VmInner { } } - /// Complete discarded stores and invalidate both the legacy bound slot and - /// every M4/G17 context naming this root. Call only after releasing PPL. - fn finish_failed_map(&self, addr: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size); - } - fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -883,19 +790,15 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind - /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those - /// contexts before allowing an unmapped object's backing to be released. + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { + if self.uat_inner.firmware == UatFirmware::Handoff { mem::sync(); return; } - let contexts = self - .uat_inner - .lock() - .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); self.tlbi_context_mask(iova, size, contexts); } @@ -923,35 +826,10 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); - // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not - // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; - let uat_inner = self.uat_inner.clone(); - let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); - let discard = if result.is_err() { - // Rollback still changes live leaves and must remain under the PPL lease. - self.page_table.discard_partial_map(node.start()..end) - } else { - Ok(()) - }; - // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. - drop(handoff); if result.is_err() { - self.finish_failed_map(node.start(), node.mapped_size); - } else { - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); - } - if let Err(error) = discard { - if self.uat_inner.m4_adopted { - // The caller retains its node and backing when this latch is set. - self.uat_inner.failed.store(true, Ordering::Release); - } - dev_err!( - self.dev.as_ref(), - "Could not discard failed kernel mapping: {:?}\n", - error - ); + self.rollback_failed_map(node.start(), node.mapped_size as u64); } result } @@ -959,14 +837,7 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node - .bo - .as_ref() - .ok_or(EINVAL)? - .get()? - .data() - .segments - .lock(); + let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -1024,7 +895,8 @@ impl VmInner { } } - // The caller invalidates negative translations after releasing its PPL lease. + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -1032,7 +904,6 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { - device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1103,7 +974,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1111,9 +982,8 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); - inner.publish_slot(id); } - inner.handoff_unlock(); + inner.handoff().unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1154,12 +1024,6 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } - - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn generation(&self) -> u8 { - self.0.binding.lock().generation - } } impl Drop for VmBind { @@ -1200,7 +1064,10 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new(bo: ARef>, owner: ARef>) -> Self { + fn new( + bo: ARef>, + owner: ARef>, + ) -> Self { Self { bo: Some(bo), owner, @@ -1243,90 +1110,29 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's -/// reservation lock, so it must not be dropped where fence producers run. -pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); - -impl TimestampVmap { - pub(crate) fn new(gem: &gem::Object) -> Result { - Ok(Self(gem.owned_vmap::<0>()?)) - } - - fn size(&self) -> usize { - use kernel::drm::gem::BaseObject; - self.0.owner().size() - } - - /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete - /// atomic store; byte stores support unaligned destinations without creating mutable - /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused - /// when the destination is prepared, and ignored here. - pub(crate) fn write(&self, offset: usize, value: u64) { - if offset - .checked_add(size_of::()) - .is_none_or(|end| end > self.size()) - { - return; - } - // Completion runs in task context. Bindings of the same BO share this lock, - // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.0.owner().timestamp_lock.lock(); - let base = self.0.addr.cast::(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. - let pointer = unsafe { base.add(offset) }; - if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: All eight bytes were checked above; the vmap pins the backing. - // A naturally aligned query result becomes available in one complete store. - let destination = unsafe { &*pointer.cast::() }; - destination.store(value.to_le(), Ordering::Relaxed); - } else { - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The complete eight-byte extent was checked above. The owned - // vmap pins its backing; byte atomics permit unaligned and overlapping - // timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(offset + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - } - // A CPU-retired query can be consumed by a later GPU command. WRITEBACK - // objects use a cacheable vmap, while GPU mappings need not snoop that cache. - // Publish the bytes before signalling the completion fence or queue frontier. - if self.0.owner().is_writeback() { - crate::pgtable::clean_poc(pointer, size_of::()); - } - crate::pgtable::poc_barrier(); - } -} - -/// Prepared CPU timestamp access with its own vmap; destruction happens after completion -/// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +/// Prepared CPU timestamp access; destruction happens after completion signalling. pub(crate) struct TimestampMapping { - mapping: TimestampVmap, + mapping: kernel::drm::gem::shmem::VMapOwned, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { - use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: TimestampVmap::new(gem)?, - offset, - }) - } - pub(crate) fn write(&self, value: u64) { - self.mapping.write(self.offset, value); + let base = self.mapping.addr.cast::(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*base.add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(ManuallyDrop>); +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1350,22 +1156,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - self.0.owner.lock_inner().page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Aligned results - /// use a complete atomic store; byte stores support unaligned destinations without - /// creating mutable aliases. An unaligned complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - let gem = self.0._gem.as_ref().ok_or(EINVAL)?; - TimestampMapping::new(gem, self.timestamp_offset(offset)?) - } - - /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete - /// extent lies inside both the mapping and the object. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1376,12 +1174,10 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(start) - } - - /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. - pub(crate) fn timestamp_vmap(&self) -> Result { - TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) + Ok(TimestampMapping { + mapping: gem.owned_vmap::<0>()?, + offset: start, + }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1493,13 +1289,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff_lock(); + uat_inner.handoff().lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff_unlock(); + uat_inner.handoff().unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1589,19 +1385,6 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { - // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected - // failed construction must retain its complete node/backing instead of invoking the - // unrelated G13/G14 FwCtl protocol. - if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { - self.0.uat_inner.failed.store(true, Ordering::Release); - pr_err!( - "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", - self.iova(), - self.size() - ); - return; - } - // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1644,27 +1427,8 @@ impl Drop for KernelMapping { self.size() ); - let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { - Ok(guard) => guard, - Err(error) => { - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after PPL failure: {:?}\n", - error - ); - return; - } - }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { - if self.0.uat_inner.m4_adopted { - self.0.uat_inner.failed.store(true, Ordering::Release); - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after partial unmap failure\n" - ); - return; - } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1673,8 +1437,6 @@ impl Drop for KernelMapping { ); } - drop(upper_handoff); - if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1691,11 +1453,6 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); - drop(_mutation); - drop(owner); - // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have - // been released. Uncertain M4 paths return earlier, retaining the node and its pins. - unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1704,73 +1461,32 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], - /// Firmware-observed incarnations of application roots installed by the VM slot allocator. - binding_generations: [u64; UAT_NUM_CTX], - binding_published: [bool; UAT_NUM_CTX], - /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by - /// successful host publications, never learned by accepting observed hardware words. - owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, - quirks: UatQuirks, } impl UatShared { - /// Takes the handoff lock, where this SoC uses it. - fn handoff_lock(&self) { - if self.quirks.handoff_lock { - self.handoff().lock(); - } - } - - /// Releases the handoff lock taken by [`Self::handoff_lock`]. - fn handoff_unlock(&self) { - if self.quirks.handoff_lock { - self.handoff().unlock(); - } - } - - /// Writes back context-table slot `idx` after it changed, where this SoC needs it. - fn publish_slot(&self, idx: usize) { - if self.quirks.clean_tables { - let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); - pgtable::clean_poc(slot, size_of::()); - pgtable::poc_barrier(); - } - } - /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { - self.ttbs_rgn - .map - .ptr() - .cast::<[SlotTTBS; UAT_NUM_CTX]>() - .as_ref() - } - .unwrap() + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { + fn contexts_naming_root(&self, ttb: u64) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| { - let expected = tagged_root(ttb, *ctx); - let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. - slot.ttb0.load(Ordering::Acquire) == expected - }) + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1787,138 +1503,20 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. - m4_adopted: bool, - /// An uncertain mapping/root is retained; future M4 allocation and publication fail. - failed: AtomicBool, - /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, - /// This SoC's UAT policy. - quirks: UatQuirks, - /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). - reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } -/// A transport owns the sole host-side PPL mutex before touching AP interest. -struct M4Transport<'a> { - shared: Guard<'a, UatShared, MutexBackend>, - start: Instant, - failed: &'a AtomicBool, -} - -impl ppl::Memory for M4Transport<'_> { - fn firmware_interested(&self) -> bool { - self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 - } - fn firmware_turn(&self) -> bool { - self.shared.handoff().turn.load(Ordering::Relaxed) != 0 - } - fn ap_interest(&self, interested: bool) { - self.shared - .handoff() - .lock_ap - .store(u8::from(interested), Ordering::Relaxed); - } - fn give_firmware_turn(&self) { - self.shared.handoff().turn.store(1, Ordering::Relaxed); - } - fn barrier(&self) { - mem::sync(); - } - fn expired(&self) -> bool { - self.start.elapsed() > Delta::from_millis(100) - } - fn relax(&self) { - core::hint::spin_loop(); - } - fn fail(&self) { - self.failed.store(true, Ordering::Release); - } -} - -/// Every AP firmware transaction owns the same mutex, including G16 Window operations. -/// M4 drops its protocol guard before its transport releases the host mutex. The legacy -/// handoff path keeps the preexisting protocol and lock ordering for other generations. -#[must_use = "retain the guard until protected table mutations have completed"] -enum HandoffGuard<'a> { - M4(ppl::Guard>), - Legacy(Guard<'a, UatShared, MutexBackend>), -} - -impl HandoffGuard<'_> { - fn shared_mut(&mut self) -> &mut UatShared { - match self { - Self::M4(guard) => &mut guard.memory_mut().shared, - Self::Legacy(shared) => shared, - } - } -} - -impl Drop for HandoffGuard<'_> { - fn drop(&mut self) { - if let Self::Legacy(shared) = self { - shared.handoff_unlock(); - } - } -} - -/// Opaque host-and-firmware lease for G16 window transactions. No caller may -/// independently manipulate the AP-interest or turn words. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[must_use = "retain the guard throughout the reserved-table window transaction"] -pub(crate) struct M4HandoffGuard<'a> { - _transaction: HandoffGuard<'a>, -} - impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } - fn check_m4(&self) -> Result { - if self.m4_adopted && self.failed.load(Ordering::Acquire) { - Err(EIO) - } else { - Ok(()) - } - } - - /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw - /// AP interest while still holding the mutex and latch M4 failure before releasing it. - fn lock_handoff(&self) -> Result> { - self.check_m4()?; - let shared = self.lock(); - self.check_m4()?; - if !self.m4_adopted { - shared.handoff_lock(); - return Ok(HandoffGuard::Legacy(shared)); - } - let transport = M4Transport { - shared, - start: Instant::::now(), - failed: &self.failed, - }; - match ppl::Guard::acquire(transport) { - Ok(guard) => Ok(HandoffGuard::M4(guard)), - Err(()) => Err(ETIMEDOUT), - } - } - - /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. - fn lock_m4_upper(&self, upper: bool) -> Result>> { - if upper && self.m4_adopted { - self.lock_handoff().map(Some) - } else { - Ok(None) - } - } - /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1946,9 +1544,7 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 { - core::hint::spin_loop() - } + while self.turn.load(Ordering::Relaxed) != 0 {} self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1973,9 +1569,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(slot, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2055,39 +1651,19 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { +fn prefetch_sgt_segments( + dev: &device::Device, + gem: &gem::Object, +) -> Result> { let sgt = crate::gem::held_sg_table(gem, dev)?; + // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { - segments, - _lease: sgt.into_lease(), - }) -} - -/// Reuse cached segments without allocating another copy for each bind. The bind lease was -/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, -/// and its extra count excludes new eviction until preparation/commit finishes. -fn prepare_sgt_segments( - dev: &device::Device, - gem: &gem::Object, - bo: &gpuvm::GpuVmBo, - _bind_lease: &kernel::drm::gem::shmem::SGLease, -) -> Result { - let needs_segments = bo.data().segments.lock().is_none(); - if needs_segments { - // Never enter the object reservation while holding the BO mutex. - let prefetched = prefetch_sgt_segments(dev, gem)?; - let mut segments = bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetched); - } - } - Ok(()) + Ok(segments) } impl Vm { @@ -2110,12 +1686,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, - uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? - .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2141,7 +1717,6 @@ impl Vm { VmBinding { binding: None, bind_token: None, - generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -2178,9 +1753,7 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); - let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { - device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2198,10 +1771,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -2212,18 +1782,15 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout - /// attribution compares and logs VMs by it). + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn id(&self) -> u64 { self.id } @@ -2235,18 +1802,9 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - self.read_lock()?.read_user(va, buf) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - pub(crate) fn read_lock(&self) -> Result { - Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { + let mut inner = lock_vm(&self.gpuvm, None, false)?; let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2316,7 +1874,9 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2331,22 +1891,26 @@ impl Vm { if stride <= block_size { return false; } - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) }) } - /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - // Every mapping change holds the VM reservation, so readers need only that one. The full - // exec lock would also lock every external BO, making each submit O(external BOs). - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return (false, None); + }; let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2398,49 +1962,6 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Best, - ) - } - - /// Lowest-address placement for a monotonic search across several VMs. - pub(crate) fn map_in_range_lowest( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Low, - ) - } - - #[allow(clippy::too_many_arguments)] - fn map_in_range_with_mode( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -2455,30 +1976,32 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; let size = object_range.range(); - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2499,7 +2022,7 @@ impl Vm { 0, range.start, range.end, - mode, + mm::InsertMode::Best, ); let mut node = match result { Ok(node) => node, @@ -2517,16 +2040,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a GEM object into this Vm at a specific address. @@ -2544,32 +2060,36 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(&gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock_vm(/*Some(&gem)*/ None, false) + .exec_lock(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2604,16 +2124,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2631,16 +2144,17 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2653,13 +2167,19 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2702,22 +2222,6 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } - /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn mark_binding_published(&self) -> Result { - let uat = self.gpuvm.lock_inner().uat_inner.clone(); - let binding = self.binding.lock(); - let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; - let mut shared = uat.lock(); - if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) - || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] - { - return Err(EIO); - } - shared.binding_published[slot] = true; - Ok(()) - } - /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2727,9 +2231,7 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2763,35 +2265,22 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; - let transaction_uat = inner.uat_inner.clone(); - let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner - .page_table - .map_pages(iova..end, phys as PhysicalAddr, prot, false); - let discard = if ret.is_err() { - inner.page_table.discard_partial_map(iova..end) - } else { - Ok(()) - }; - drop(handoff); + let ret = inner.page_table.map_pages( + iova..end, + phys as PhysicalAddr, + prot, + false, + ); if ret.is_err() { - inner.finish_failed_map(iova, size); - } else { - inner.tlbi_contexts_naming_root(iova, size); - } - let retain = discard.is_err() && inner.uat_inner.m4_adopted; - if retain { - inner.uat_inner.failed.store(true, Ordering::Release); + inner.rollback_failed_map(iova, size as u64); } + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. drop(mutation); - drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } + let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Unmap everything in an address range. @@ -2833,9 +2322,7 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2866,7 +2353,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2905,8 +2392,7 @@ impl Vm { // The BO's newest sequence is checked under the same lock as binds. If // it is not newer than the close, every mapping belongs to that close // and the per-object list is safe and substantially cheaper to walk. - // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => { return self.drop_mappings_filtered(gem, sequence); } @@ -2999,18 +2485,8 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = match self.uat_inner.lock_handoff() { - Ok(guard) => guard, - Err(_) => { - self.page_table.retain(); - pr_err!( - "MMU: retaining M4 VM {} page tables after PPL failure\n", - self.id - ); - return; - } - }; - let uat_inner = transaction.shared_mut(); + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -3023,10 +2499,9 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); - uat_inner.owned_user_roots[idx] = 0; - uat_inner.publish_slot(idx); } - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -3046,7 +2521,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -3057,56 +2532,34 @@ pub(crate) struct AdoptedRegions { } impl Uat { - /// Common AP and firmware exclusion for G16's reserved-table window. Acquire - /// immutable root addresses before this guard, preserving binding -> UAT lock order. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn lock_m4_handoff(&self) -> Result> { - if !self.inner.m4_adopted { - return Err(EINVAL); - } - Ok(M4HandoffGuard { - _transaction: self.inner.lock_handoff()?, - }) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. - fn map_physical( - range: &Range, - min_size: usize, - reserved_ram: bool, - ) -> Result { + fn map_physical(range: &Range, min_size: usize) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as usize; + raw.flags = kernel::bindings::IORESOURCE_MEM as _; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - let flags = io::mem::MemFlag::WC; - #[cfg(CONFIG_DRM_ASAHI_G15)] - let flags = if reserved_ram { - // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. - if crate::g15::memory_mode(range.start, size as u64)? - == kernel::bindings::MEMREMAP_WB as u32 - { - io::mem::MemFlag::WB - } else { - flags - } + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + let mode = crate::g15::memory_mode(range.start, size as u64)?; + let ram = mode == kernel::bindings::MEMREMAP_WB as u32; + let flags = if ram { + io::mem::MemFlag::WB } else { - flags + io::mem::MemFlag::WC }; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let _ = reserved_ram; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; - // the GPU reaches them through its own translation. + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU + // reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3114,7 +2567,7 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3128,52 +2581,23 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. - #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, true, true) - } - - /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also - /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; - /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn new_adopted_with_user_upper( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn new_adopted_inner( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - reserved_ram: bool, - ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, - m4_adopted: !map_kernel_to_user, - failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, - reserved: None, - quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3182,14 +2606,10 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user, + map_kernel_to_user: true, handoff_rgn, ttbs_rgn, - quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3201,14 +2621,8 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = Vm::new( - dev.into(), - inner.clone(), - geometry.user_range(), - geometry, - None, - 1, - )?; + let kernel_lower_vm = + Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -3235,13 +2649,13 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3305,16 +2719,13 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self - .inner - .lock() - .contexts_naming_root(root, self.inner.m4_adopted)) + Ok(self.inner.lock().contexts_naming_root(root)) } /// Publish a completed alias set before its descriptors become visible. @@ -3343,24 +2754,12 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, true) - } - - /// Binds a VM without waiting for another slot owner to retire. - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn try_bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, false) - } - - fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { - self.inner.check_m4()?; let mut binding = vm.binding.lock(); - self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = module_parameters::robust_isolation.value() != 0; + let isolation = *module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3368,34 +2767,22 @@ impl Uat { None }); - let slot = if wait { - self.slots.get(binding.bind_token)? - } else { - self.slots.try_get(binding.bind_token)? - }; + let slot = self.slots.get(binding.bind_token)?; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = self.inner.lock_handoff()?; - let uat_inner = transaction.shared_mut(); - // On M4, empty bind/close churn must not wrap the wire generation - // back to firmware's last observed identity for a different root. - let previous = uat_inner.binding_generations[idx]; - let incarnation = if !self.inner.m4_adopted - || previous == 0 - || uat_inner.binding_published[idx] - { - previous.checked_add(1).ok_or(EOVERFLOW)? + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT } else { - previous + 0 }; - let ttb1 = - application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); - let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -3404,12 +2791,8 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.binding_generations[idx] = incarnation; - uat_inner.binding_published[idx] = false; - uat_inner.owned_user_roots[idx] = ttb; - binding.generation = binding_wire_generation(incarnation); - uat_inner.publish_slot(idx); - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -3428,35 +2811,6 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } - /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every - /// host root installation/removal; firmware does not write these words. An unexpected - /// observed root is an error, never a new ownership grant. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn check_adopted_bindings(&self) -> Result { - self.inner.check_m4()?; - // Binding locks precede the global UAT lock, as in bind(). - let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; - let inner = self.inner.lock(); - self.inner.check_m4()?; - if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { - return Err(EINVAL); - } - let ttbs = inner.ttbs(); - if ttbs[0].ttb0.load(Ordering::Acquire) != lower - || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID - { - return Err(EIO); - } - for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { - if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || table.ttb1.load(Ordering::Acquire) != 0 - { - return Err(EIO); - } - } - Ok(()) - } - /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -3464,7 +2818,7 @@ impl Uat { return Err(EINVAL); } { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3508,7 +2862,6 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); - inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3518,17 +2871,14 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - self.inner.check_m4()?; - let vm = Vm::new( + Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - )?; - self.inner.check_m4()?; - Ok(vm) + ) } /// Returns the address space geometry of this UAT. @@ -3550,19 +2900,14 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - UAT_PGSZ / size_of::(), - |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) }?; mem::sync(); @@ -3575,16 +2920,11 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - snapshot.len(), - |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) }?; if !matches { return Err(EIO); @@ -3596,19 +2936,15 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let Some(secondary) = self.inner.lock().secondary_ttb1 else { - // A single-instance firmware has no secondary root to mirror. - return Ok(()); - }; + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(reserved, secondary, count, |secondary| { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3625,33 +2961,24 @@ impl Uat { Ok(()) } - /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). - pub(crate) fn invalidate_reserved_tables(&self) { - if let Some(reserved) = self.inner.reserved.as_ref() { - reserved.invalidate(); - } - } - /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff_unlock(); + inner.handoff().unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.publish_slot(KERNEL_ALIAS_CTX); - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3661,30 +2988,21 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner( - dev: driver::AsahiDevice, - firmware: UatFirmware, - quirks: UatQuirks, - reserved: Option>, - ) -> Result> { + fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); - let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - m4_adopted: false, - failed: AtomicBool::new(false), - clean_poc: quirks.clean_tables, - quirks, - reserved, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + clean_poc: false, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3693,14 +3011,10 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, - quirks: shared_quirks, }, "uat_shared" ), @@ -3716,27 +3030,12 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, - quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - mem::set_page_tlbi(quirks.page_tlbi); - let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let reserved = if quirks.reserved_tables { - let mut tables = pgtable::ReservedTables::new(); - for name in [c"pagetables", c"l2-pagetables"] { - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - // SAFETY: The boot loader reserved these regions (no-map) for the firmware's - // page tables; they stay reserved until reboot. - unsafe { tables.add(res)? }; - } - Some(Arc::new(tables, GFP_KERNEL)?) - } else { - None - }; - - let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let inner = Self::make_inner(dev.clone(), firmware)?; + let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3744,11 +3043,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset: Some(offset), - } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), - UatFirmware::PublishedRoots { - secondary_root_offset: None, - } => (UAT_PGSZ, None), + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3799,18 +3098,11 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; - if quirks.clean_tables { - pgtable::clean_poc( - core::ptr::from_ref(inner.handoff()).cast::(), - HANDOFF_SIZE, - ); - pgtable::poc_barrier(); - } + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); @@ -3822,7 +3114,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if matches!(firmware, UatFirmware::PublishedRoots { .. }) { + if secondary_ttb1.is_some() { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3831,12 +3123,8 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } - if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); - pgtable::poc_barrier(); - } - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); @@ -3854,30 +3142,3 @@ impl Drop for Uat { mem::sync(); } } - -impl Vm { - /// Diagnostic reservation occupancy (including guards), not a placement promise. - /// Removal can progress between queries; allocations are excluded by the VM lock. - pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = self.gpuvm.lock_inner(); - let mut next = range.start; - let mut used = 0; - let mut largest = 0; - while next < range.end { - let Some(node) = inner.mm.first_overlap(next..range.end)? else { - largest = largest.max(range.end - next); - break; - }; - largest = largest.max(node.start.saturating_sub(next)); - let end = node.end.min(range.end); - used += end - node.start.max(next); - next = end; - } - Ok((used, largest)) - } -} -impl Uat { - pub(crate) fn execution_contexts_used(&self) -> u32 { - (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() - } -} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index acce5677b9f92..44a1575ecfd2b 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,7 +17,6 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, - _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -58,7 +57,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state before taking the GPUVM exec lock or changing any leaf. + /// Prepares host state without taking DMA reservations or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -81,20 +80,24 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + crate::gem::held_sg_table(gem, dev)?; let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; + { + let mut segments = vm_bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetch_sgt_segments(dev, gem)?); + } + } if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, - _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, @@ -135,7 +138,7 @@ impl Vm { PreparedUserBindOp::Map(map) => { // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is // not locked: the map step may still populate its pages. - let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; match self.mapping_commit() { Ok(commit) => { let (flags, repeat) = if map.single_page { @@ -185,7 +188,7 @@ impl Vm { } PreparedUserBindOp::Unmap(unmap) => { // SAFETY: See `lock_vm()`. - let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; match self.mapping_commit() { Ok(commit) => { let mut ctx = diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 66d507c2d7694..30010be497973 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,13 +79,9 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 - && state.commits == 0 - && !state.blocked - && !state.quarantined - && !state.draining - && !state.closed - && !state.cleanup_failed + state.active == 0 && state.commits == 0 + && !state.blocked && !state.quarantined && !state.draining + && !state.closed && !state.cleanup_failed }) } @@ -343,12 +339,10 @@ impl Vm { drop(previous); } - /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed - /// admission for good). G15 releases its per-VM state after that. + /// Whether userspace has permanently closed this VM's map admission. + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn is_closed(&self) -> bool { - self.lifetime - .as_ref() - .is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -417,10 +411,7 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { - vm: self.clone(), - _residency: residency, - }); + return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); } lifetime.changed.wait(&mut state); } @@ -480,11 +471,7 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch( - &self, - batch: &mut PreparedUserBindBatch, - start: usize, - ) -> Result { + pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -498,8 +485,7 @@ impl Vm { } if batch .ops - .iter() - .skip(start) + .iter().skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -510,21 +496,16 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = - self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state - .unmaps - .iter() - .any(|old| old.iova == unmap.iova && old.size == unmap.size) - { + if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -549,30 +530,16 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state - .unmaps - .iter() - .any(|old| old.iova == iova && old.size == size) - { + if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { - return Err(ENOENT); - } - if !state.blocked { - return Ok(false); - } - if !state.quarantined { - return Err(EBUSY); - } - if !state - .unmaps - .iter() - .any(|old| old.iova == iova && old.size == size) - { + if state.closed { return Err(ENOENT); } + if !state.blocked { return Ok(false); } + if !state.quarantined { return Err(EBUSY); } + if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -667,8 +634,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::take(&mut state.unmaps); - let objects = core::mem::take(&mut state.objects); + let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); + let objects = core::mem::replace(&mut state.objects, KVec::new()); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -704,9 +671,7 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { - status.record(error); - } + if let Some(status) = &self.status { status.record(error); } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs deleted file mode 100644 index 7cdad5e1f3831..0000000000000 --- a/drivers/gpu/drm/asahi/mmu/ppl.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire -//! guard lifetime: firmware exclusion alone does not serialize two AP threads. -//! This core has no kernel dependencies so the production guard can be exercised -//! against deterministic firmware and concurrent host transports. - -pub(super) trait Memory { - fn firmware_interested(&self) -> bool; - fn firmware_turn(&self) -> bool; - fn ap_interest(&self, interested: bool); - fn give_firmware_turn(&self); - fn barrier(&self); - fn expired(&self) -> bool; - fn relax(&self); - /// Latch failure before the transport releases host exclusion on timeout. - fn fail(&self); -} - -/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. -#[must_use = "retain the guard until every protected table mutation has completed"] -pub(super) struct Guard { - memory: T, -} - -impl Guard { - pub(super) fn acquire(memory: T) -> Result { - let timeout = || { - memory.ap_interest(false); - memory.barrier(); - memory.fail(); - Err(()) - }; - memory.ap_interest(true); - memory.barrier(); - while memory.firmware_interested() { - if memory.expired() { - return timeout(); - } - if memory.firmware_turn() { - memory.ap_interest(false); - memory.barrier(); - while memory.firmware_turn() { - if memory.expired() { - return timeout(); - } - memory.relax(); - } - memory.ap_interest(true); - memory.barrier(); - } - memory.relax(); - } - memory.barrier(); - Ok(Self { memory }) - } - - pub(super) fn memory_mut(&mut self) -> &mut T { - &mut self.memory - } -} - -impl Drop for Guard { - fn drop(&mut self) { - self.memory.give_firmware_turn(); - self.memory.barrier(); - self.memory.ap_interest(false); - self.memory.barrier(); - // Only after these stores complete may `memory` release the host mutex. - } -} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 1df852184b31d..5370b87ccf1b5 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,8 +251,6 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} -// SAFETY: Shared callbacks use the VM's residency/reservation locks. The -// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -266,7 +264,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } @@ -275,7 +273,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); + (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -283,36 +281,23 @@ impl VmShrinker { Ok(owner) } - /// Count reclaimable pages. - /// - /// # Safety - /// - /// `raw` must be this owner's live registered shrinker. The core must - /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() + owner.vm.idle_reclaim_count() as _ } - /// Reclaim idle pages. - /// - /// # Safety - /// - /// `raw` must be this owner's live registered shrinker, and `sc` must - /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - // SAFETY: The shrinker core supplies a live initialized control. - let target = unsafe { (*sc).nr_to_scan }; - owner.vm.reclaim_idle_pages(target) + let target = unsafe { (*sc).nr_to_scan } as usize; + owner.vm.reclaim_idle_pages(target) as _ } } @@ -363,7 +348,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) { return 0; } @@ -376,17 +361,11 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Count retained backing on idle VMs, and purgeable contents even if idle - // reclaim already released the table. Never wait on the BO mutex under resv. - let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let advised = object.madv_locked() > 0; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - (advised || object.has_sg_table_locked()) - && object.unleased_pages_releasable_locked() - }; - if object.idle_reclaim_candidate() && (advised || idle) && releasable { + // Purgeable objects count on busy VMs too; idle backing only on idle ones. + if object.idle_reclaim_candidate() + && (object.madv_locked() > 0 || idle) + && object.sole_sg_lease_releasable_locked() + { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -406,7 +385,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) { return 0; } @@ -417,7 +396,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); + let contexts = shared.contexts_naming_root(inner.ttb()); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -429,12 +408,9 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - object.unleased_pages_releasable_locked() - }; - if !releasable { return Ok(true); } + if segments.is_some() && !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -491,7 +467,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) || !self .lifetime .as_ref() @@ -506,7 +482,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); + let contexts = shared.contexts_naming_root(inner.ttb()); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -518,14 +494,9 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - // A competing lease may have made the previous release fail after we - // dropped our segments. Retry that table, but do not count empty backing. - object.has_sg_table_locked() && object.unleased_pages_releasable_locked() - }; - if !releasable { return Ok(true); } + if segments.is_none() || !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index fec7a6b61ad8f..b9e2975c898ff 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: core::ptr::from_ref(gem) as usize, + object: gem as *const gem::Object as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = core::ptr::from_ref(gem) as usize; + let object = gem as *const gem::Object as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 0c9e6f62b5bbe..38a2268137eff 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p.cast::(); + let inner = p as *const u8; // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer.cast::()) }; + let off = unsafe { inner.offset_from(outer as *const u8) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p.cast::(); + let inner = p as *const u8; // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer.cast::()) }; + let off = unsafe { inner.offset_from(outer as *const u8) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,15 +298,11 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc - .ptr() - .ok_or(EINVAL)? - .as_ptr() - .cast::>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -341,14 +337,10 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc - .ptr() - .ok_or(EINVAL)? - .as_ptr() - .cast::>>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if !core::ptr::eq(p.cast::>(), raw) { + if p as *mut T::Raw<'_> != raw as *mut _ { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -357,7 +349,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p.cast::().cast::>(), + raw: p as *mut u8 as *mut T::Raw<'static>, gpu_ptr, alloc, inner, @@ -408,16 +400,16 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; let ret = Self { - raw: p.cast::().cast::>(), + raw: p as *mut u8 as *mut T::Raw<'static>, gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = core::ptr::from_ref::(&*ret.inner); + let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; Ok(ret) } @@ -445,7 +437,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -457,7 +449,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } } } @@ -483,8 +475,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", self.inner)) - .field("alloc", &format_args!("{:?}", self.alloc)) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0f915607cb1f4..b88353a2f7344 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,8 +18,7 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, - sync::Arc, // + prelude::*, // }; use crate::debug::*; @@ -54,20 +53,19 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local -/// CTR_EL0 read can select too large a stride if the task migrates during the loop. +/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). #[inline(always)] fn dcache_line_size() -> usize { - use kernel::bindings; - let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) - | (bindings::SYS_CTR_EL0_Op1 << 16) - | (bindings::SYS_CTR_EL0_CRn << 12) - | (bindings::SYS_CTR_EL0_CRm << 8) - | (bindings::SYS_CTR_EL0_Op2 << 5); - // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine - // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. - let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; - 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. + unsafe { + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags) + ) + }; + 4 << ((ctr >> 16) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -97,6 +95,7 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -117,130 +116,20 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). +/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; -} - -/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached -/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` -/// regions, where the firmware builds its root and kernel-window tables). -pub(crate) struct ReservedTables { - regions: KVec, - starts: KVec, -} - -// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table -// entries are only accessed as atomics. -unsafe impl Send for ReservedTables {} -// SAFETY: As above. -unsafe impl Sync for ReservedTables {} - -impl ReservedTables { - pub(crate) fn new() -> Self { - Self { - regions: KVec::new(), - starts: KVec::new(), - } - } - - /// Adds a reserved region, mapped write-back. - /// - /// # Safety - /// - /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. - pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { - let start = res.start(); - // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = - unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; - self.starts.push(start, GFP_KERNEL)?; - self.regions.push(map, GFP_KERNEL)?; - Ok(()) - } - - /// The CPU address of a page-table page in one of the regions. - fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { - for (start, map) in self.starts.iter().zip(self.regions.iter()) { - let Some(offset) = phys.checked_sub(*start) else { - continue; - }; - let offset = offset as usize; - if offset + UAT_PGSZ <= map.size() { - // SAFETY: The offset and one page lie inside the mapping. - return Some(unsafe { map.ptr().add(offset) }); - } - } - None - } - - /// Whether `phys` is a page of one of the regions. - pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { - self.lookup(phys).is_some() - } - - /// Drops the CPU's cached lines of every region, so that later reads see what the firmware - /// wrote while booting. Lines the host dirtied are written back first, never discarded. - /// - /// This runs once, after the firmware booted. The host has written table entries by then - /// (fixed kernel objects are mapped before the boot), but the T6040, the only SoC with - /// reserved tables, also sets `UatQuirks::clean_tables`: every host table store is cleaned - /// to the PoC as it is made, so this drops only clean lines and loses no host write. - /// - /// The regions are not invalidated again, which assumes the firmware no longer writes the - /// tables the host walks once it has booted. That is not proven by a trace; the J616s has - /// run this way since its GPU came up without a fault traced to a stale table. - pub(crate) fn invalidate(&self) { - for map in self.regions.iter() { - clean_invalidate_poc(map.ptr(), map.size()); - } - poc_barrier(); - } -} - -/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the -/// reserved mapping when the page lies in a reserved region, else through `struct page`. With -/// reserved regions, a page outside them that is not System RAM with a `struct page` (other -/// no-map memory) is refused with `EIO`. -/// -/// # Safety -/// -/// `phys` must be a page-table page that stays allocated while `f` runs. -unsafe fn with_table( - reserved: Option<&ReservedTables>, - phys: PhysicalAddr, - offset: usize, - len: usize, - f: impl FnOnce(*mut u8) -> Result, -) -> Result { - if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { - if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { - return Err(EINVAL); - } - // SAFETY: lookup() checked that the whole page is mapped. - return f(unsafe { base.add(offset) }); - } - let page = if reserved.is_some() { - // SAFETY: The caller guarantees the page stays allocated; borrow_phys() checks that it - // is System RAM with a struct page. - unsafe { Page::borrow_phys(&phys) }.ok_or(EIO)? - } else { - // SAFETY: The caller guarantees an allocated page-table page in RAM. - unsafe { Page::borrow_phys_unchecked(&phys) } - }; - page.with_pointer_into_page(offset, len, f) + unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` -/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the -/// entries concurrently, which is why they are only exposed as atomics. +/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` +/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they +/// are only exposed as atomics. pub(crate) unsafe fn with_root_entries( - reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -248,18 +137,15 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: A root outside the reserved regions must have a struct page. - if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { - return Err(EIO); - } - // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. - unsafe { - with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { - // SAFETY: with_table() checked that `count` entries are mapped, and page table entries - // are naturally aligned. - Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) - }) - } + // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. + let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; + page.with_pointer_into_page(0, count * PTE_SIZE, |p| { + // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page + // table entries are naturally aligned. + Ok(f(unsafe { + core::slice::from_raw_parts(p as *const AtomicU64, count) + })) + }) } const PTE_TYPE_BITS: u64 = 3; @@ -367,9 +253,7 @@ impl Prot { Self { memattr, ..*self } } - /// The protection bits of a leaf with this protection (no address, table type or non-global - /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() } @@ -394,22 +278,6 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } - /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN - /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to - /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. - const fn native_gpu(self) -> Self { - if self.ap != AP_GPU { - return self; - } - let selector = (self.high_bits >> 1) & 3; - Self { - ap: AP_FW_GPU, - high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) - | (((selector + 1) & 3) << 1), - ..self - } - } - pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -428,12 +296,6 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, - /// Ownership could not be relinquished to firmware; physical tables remain until reboot. - retained: bool, - /// Reserved regions holding tables this page table may reach. - reserved: Option>, - /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). - native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -443,6 +305,7 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -454,49 +317,45 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, - retained: false, - reserved: None, - native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been - /// zeroed through the cacheable linear map. + /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb sy`. Call it before anything is mapped and before any context names the root. + /// `dsb osh`. Call it before anything is mapped and before any context names the root. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in - // a reserved region. - unsafe { - with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - }) - }?; + // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. + let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; + root.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency. + /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). #[inline(always)] fn cleans_poc(&self) -> bool { - self.clean_poc - } - - /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form - /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. - pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { - self.native_gpu = native_gpu; - self + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + { + self.clean_poc + } + #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] + { + false + } } /// Borrow a root; track individual child ownership only when other agents @@ -507,7 +366,6 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, - reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -523,9 +381,7 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) - || unsafe { Page::borrow_phys(&ttb) }.is_some(); - if !reachable { + if unsafe { Page::borrow_phys(&ttb) }.is_none() { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -536,23 +392,15 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, - retained: false, - reserved, - native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } - /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim - /// this tree merely because a userspace handle closed or a fence reported an error. - pub(crate) fn retain(&mut self) { - self.retained = true; - } - pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -622,7 +470,6 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; - let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -666,61 +513,55 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - let driver_tables = &mut self.driver_tables; - let oas_mask = self.oas_mask; - let entry = |p: *mut u8| -> Result> { - let uptep = p.cast_const().cast::(); - // SAFETY: with_table() ensures the pointer is valid, and our index is - // aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!( - "UATPageTable::with_pages: alloc PT at {:#x}\n", - pt_page.phys() - ); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; + } + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); + } } - if let Some(tables) = driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } - } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) - } - }; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed tables in RAM or in a reserved region. - pt_addr[level] = unsafe { - with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) - }?; + })?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -756,10 +597,13 @@ impl UatPageTable { count, iova ); - let leaves = |p: *mut u8| -> Result { - let ptep = p.cast_const().cast::(); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_table(). + // checked by with_pointer_into_page(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -767,18 +611,7 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - }; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed tables in RAM or in a reserved region. - unsafe { - with_table( - reserved.as_deref(), - phys, - idx * PTE_SIZE, - count * PTE_SIZE, - leaves, - ) - }?; + })?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -803,9 +636,16 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { + fn free_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - tables.contains(&phys) + match tables.iter().position(|&table| table == phys) { + Some(_) => true, + None => false, + } } else { true }; @@ -813,55 +653,32 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - let oas_mask = self.oas_mask; - let in_reserved = self - .reserved - .as_ref() - .is_some_and(|r| r.contains(parent_phys)); - // SAFETY: Only looks the page up; nothing is accessed through it. - if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { - return Err(EIO); - } - let reserved = self.reserved.as_deref(); - let cleans = self.cleans_poc(); - let detach = |p: *mut u8| -> Result { - // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() - // bounds this access to one PTE. + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & oas_mask & (!UAT_PGMSK as u64); - if current & oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if cleans { + if self.cleans_poc() { clean_poc(p, PTE_SIZE); } Ok(()) - }; - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - unsafe { - with_table( - reserved, - parent_phys, - parent_index * PTE_SIZE, - PTE_SIZE, - detach, - ) - }?; + })?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; tables.swap_remove(index); } - if self.cleans_poc() { - // Complete the parent descriptor clear before its detached child can be reused. - // Callers have already removed this tree from live contexts before freeing tables. - poc_barrier(); - } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -952,15 +769,6 @@ impl UatPageTable { } } - /// The permission bits as this table stores them. - fn stored(&self, prot: Prot) -> Prot { - if self.native_gpu { - prot.native_gpu() - } else { - prot - } - } - pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -980,7 +788,6 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); - let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -1007,7 +814,6 @@ impl UatPageTable { iova_range, prot ); - let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -1055,9 +861,6 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { - if self.retained { - return; - } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index 20396f2e77225..a68352828cfbc 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.firmware_mapping()?, + mapping: mapping.clone(), offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index aa5dc1fba4723..0872a50ce5d46 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,9 +67,7 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { - return Err(EINVAL); - } + if layouts != [0; 2] { return Err(EINVAL); } Ok(()) } @@ -153,7 +151,6 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, - usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -194,7 +191,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("aurora") + c_str!("asahi") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -265,14 +262,10 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = - Some(kernel::module::this_module::()); + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { - dev: self.dev.clone(), - id: self.id, - } + QueueTimeout { dev: self.dev.clone(), id: self.id } } fn prepare(job: &mut Self) -> Option { @@ -364,7 +357,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - guard.registration_data_with(|reg_data| { + return guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -439,7 +432,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }) + }); } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -468,7 +461,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("aurora_fence"); +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -488,7 +481,6 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -526,21 +518,12 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("aurora_sched"), + c_str!("asahi_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; - let gpu_context = Arc::new( - workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, - GFP_KERNEL, - )?; - // Every command of this queue points the firmware at the notifier, and the notifier links - // into the notifier list of the work queues. Keep it as long as they are kept, until the - // context is invalidated, instead of freeing it with the last command. - gpu_context.retain(KBox::new(notifier.clone(), GFP_KERNEL)?)?; - let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -554,12 +537,15 @@ impl Queue::ver { inner: QueueInner::ver { dev: dev.clone(), ualloc, - gpu_context, + gpu_context: Arc::new( + workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, + GFP_KERNEL, + )?, + buffer, notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, - usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), @@ -567,7 +553,7 @@ impl Queue::ver { }; // Rendering structures - let tvb_blocks = module_parameters::initial_tvb_size.value(); + let tvb_blocks = *module_parameters::initial_tvb_size.value(); ret.inner.buffer.ensure_blocks(tvb_blocks)?; @@ -637,11 +623,8 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -652,10 +635,8 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i] = microseq::Attachment { + attachments.list[i as usize] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -773,33 +754,10 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - let payload_size = header.size as usize; - match u32::from(header.cmd_type) { - uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render - .ts_frag_end_copies - .iter() - .any(|ts| ts.handle != 0 || ts.offset != 0) - { - return Err(EINVAL); - } - } - uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute - .ts_end_copies - .iter() - .any(|ts| ts.handle != 0 || ts.offset != 0) - { - return Err(EINVAL); - } - } - _ => cmdbuf.skip(payload_size), - } + cmdbuf.skip(header.size as usize); nr_commands += 1; - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -891,7 +849,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match u32::from(header.cmd_type) { + let queue_job = match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -921,7 +879,7 @@ impl Queue for Queue::ver { } } - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index e8b845cba989f..fb22569ea5fdf 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,15 +13,12 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, - c_str, + bindings, c_str, device::Core, + devres::Devres, error::{from_err_ptr, to_result}, io::{ - mem::{ - DevresIoMem, - IoMem, // - }, + mem::IoMem, Io, IoBase, // }, @@ -225,10 +222,7 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get( - pdev.as_ref().as_raw(), - index.try_into()?, - ))? + from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -241,10 +235,7 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { - dev: pdev.into(), - mbox, - }) + Ok(Self { dev: pdev.into(), mbox }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -260,11 +251,7 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!( - self.dev.as_ref(), - "Could not stop GPU coprocessor: {:?}\n", - error - ); + dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); } } } @@ -566,17 +553,14 @@ impl Resources<'_> { #[derive(Clone)] pub(crate) struct G17Resources { dev: ARef, - sgx: Arc>, + sgx: Arc>>, } impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new( - sgx_req.iomap_sized::()?.into_devres()?, - GFP_KERNEL, - )?; + let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; Ok(G17Resources { dev: pdev.into(), @@ -709,11 +693,6 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } - /// Raw core power state, for tracing only. - pub(crate) fn g17_core_power(&self) -> u32 { - self.sgx_read32::() - } - /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { @@ -722,6 +701,7 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + } /// Split the packed per-cluster core masks into one mask per cluster. @@ -752,7 +732,11 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); + dev_err!( + dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); return Err(EIO); } @@ -760,10 +744,7 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision( - dev: &platform::Device, - id_version: u32, -) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -777,29 +758,3 @@ fn gpu_revision( } }) } - -/// Device-managed SGX mapping retained by the G16 firmware session. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[derive(Clone)] -pub(crate) struct G16Resources { - sgx: Arc>, -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl G16Resources { - pub(crate) fn new(pdev: &platform::Device>) -> Result { - let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - Ok(Self { - sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, - }) - } - pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - (&*sgx).as_view().try_read32(offset) - } - #[allow(dead_code)] - pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - (&*sgx).as_view().try_write32(value, offset) - } -} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7132e6de41c09..9f0baa4350d93 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include @@ -18,7 +16,7 @@ #include "sysfs.h" -/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ +/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -34,85 +32,12 @@ struct asahi_stats_snapshot { u64 jobs; }; -/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ -struct asahi_gpu_memory { - u64 user_committed; - u64 user_resident; - u64 kernel_resident; - u64 refused; - u64 limit; - u64 client_limit; -}; - struct asahi_stats_attribute { struct device_attribute attr; - struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; - const struct asahi_gpu_memory *memory; bool export_enabled; - asahi_stats_read_fn read; - asahi_stats_suspend_fn suspend; - struct notifier_block pm_notifier; }; -/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ -static const char *asahi_stats_quality(u32 status) -{ - return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; -} - -static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) -{ - struct asahi_qualified_stats snap; - ssize_t n = 0; - - stats->read(stats->snapshot, &snap); - n += sysfs_emit_at(buf, n, "utilisation_status %s\n", - asahi_stats_quality(snap.util_status)); - if (snap.util_status == 1) { - n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); - n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); - n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); - } else { - n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" - "util_renderer_percent unsupported\n" - "util_device_percent unsupported\n"); - } - n += sysfs_emit_at(buf, n, "pstate_status %s\n", - asahi_stats_quality(snap.pstate_status)); - if (snap.pstate_status == 1) - n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); - else - n += sysfs_emit_at(buf, n, "pstate unsupported\n"); - /* No A/B selector, packet duration, measured power or temperature. */ - n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" - "achieved_frequency_hz unsupported\n" - "busy_ns unsupported\n" - "power_mw unsupported\n" - "temperature_raw unsupported\n"); - return n; -} - -static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) -{ - struct asahi_stats_attribute *stats = - container_of(nb, struct asahi_stats_attribute, pm_notifier); - - switch (action) { - case PM_SUSPEND_PREPARE: - case PM_HIBERNATION_PREPARE: - case PM_RESTORE_PREPARE: - stats->suspend(stats->snapshot, true); - break; - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - stats->suspend(stats->snapshot, false); - break; - } - return NOTIFY_OK; -} - static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -123,8 +48,6 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); - if (stats->read) - return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -149,56 +72,20 @@ static ssize_t agx_stats_show(struct device *dev, return n; } -/* - * GPU memory held by the driver, in bytes: user objects created (committed) - * and held by the GPU (resident), held kernel objects, the limits that admit - * user objects (0: none) and the number of refused creations. - */ -static ssize_t gpu_memory_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct asahi_stats_attribute *stats = - container_of(attr, struct asahi_stats_attribute, memory_attr); - const struct asahi_gpu_memory *mem = stats->memory; - ssize_t n = 0; - - n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", - (unsigned long long)READ_ONCE(mem->user_committed)); - n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", - (unsigned long long)READ_ONCE(mem->user_resident)); - n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", - (unsigned long long)READ_ONCE(mem->kernel_resident)); - n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", - (unsigned long long)READ_ONCE(mem->limit)); - n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", - (unsigned long long)READ_ONCE(mem->client_limit)); - n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", - (unsigned long long)READ_ONCE(mem->refused)); - - return n; -} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle, - asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) + int export_enabled, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || - (!!read != !!suspend)) + if (!dev || !handle || (export_enabled && !snapshot)) return -ENODEV; - BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); - BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); - BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); - BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); - BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -214,36 +101,9 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; - stats->read = read; - stats->suspend = suspend; - if (suspend) { - stats->pm_notifier.notifier_call = asahi_stats_pm_notify; - ret = register_pm_notifier(&stats->pm_notifier); - if (ret) { - kfree(stats); - return ret; - } - } - - sysfs_attr_init(&stats->memory_attr.attr); - stats->memory_attr.attr.name = "gpu_memory"; - stats->memory_attr.attr.mode = 0444; - stats->memory_attr.show = gpu_memory_show; - stats->memory = memory; ret = device_create_file(dev, &stats->attr); if (ret) { - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); - kfree(stats); - return ret; - } - - ret = device_create_file(dev, &stats->memory_attr); - if (ret) { - device_remove_file(dev, &stats->attr); - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -253,46 +113,6 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, } EXPORT_SYMBOL_GPL(asahi_sysfs_register); -/* - * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores - * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores - * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs - * about 1 ms, so userspace may lengthen it on AC power. - */ -static u32 *asahi_idle_word; - -static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); -} - -static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - u32 ms; - - if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) - return -EINVAL; - WRITE_ONCE(*asahi_idle_word, ms); - return count; -} - -static DEVICE_ATTR_RW(idle_off_delay_ms); - -int asahi_sysfs_register_idle(struct device *dev, u32 *word) -{ - int ret; - - if (!dev || !word) - return -ENODEV; - asahi_idle_word = word; - ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); - if (ret) - asahi_idle_word = NULL; - return ret; -} -EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); - void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -300,20 +120,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; - /* Both notifier removal and file removal join readers of the Rust Arc. */ - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ - device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); - if (asahi_idle_word) { - device_remove_file(dev, &dev_attr_idle_off_delay_ms); - asahi_idle_word = NULL; - } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index d43f482903e55..5969a1718925f 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,31 +3,10 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H -#include - struct device; -/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ -struct asahi_qualified_stats { - u32 util_status; - u32 pstate_status; - u32 tiler; - u32 renderer; - u32 device; - u32 pstate; - u64 util_host_ns; - u64 pstate_host_ns; - u64 util_samples; - u64 pstate_timestamp; -}; - -typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); -typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); - int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle, - asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); -int asahi_sysfs_register_idle(struct device *dev, u32 *word); + int export_enabled, void **handle); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 7c34f0627b020..079623d1ef210 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,111 +3,24 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; +use crate::stats::StatsSnapshot; use core::ffi::{c_int, c_void}; -type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); -type SuspendStats = unsafe extern "C" fn(*const c_void, bool); - extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const c_void, + snapshot: *const StatsSnapshot, export_enabled: c_int, - memory: *const Ledger, handle: *mut *mut c_void, - read: Option, - suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } -extern "C" { - fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; -} - -/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). -pub(crate) fn register_idle( - dev: *mut kernel::bindings::device, - word: *mut u32, -) -> kernel::error::Result { - // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes - // it with READ_ONCE/WRITE_ONCE. - let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { - Err(kernel::error::Error::from_errno(ret)) - } else { - Ok(()) - } -} - /// Register only after initialization succeeds, keeping the snapshot Arc alive. -/// -/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, -) -> kernel::error::Result<*mut c_void> { - register_with_callbacks(dev, snapshot, export_enabled, None, None) -} - -/// The caller retains the G17 snapshot Arc until unregister joins both sysfs -/// readers and the PM notifier. The callbacks never power the GPU up. -pub(crate) fn register_g17( - dev: *mut kernel::bindings::device, - snapshot: *const StatsSnapshot, - export_enabled: bool, -) -> kernel::error::Result<*mut c_void> { - register_with_callbacks( - dev, - snapshot, - export_enabled, - Some(read_g17), - Some(suspend_g17), - ) -} - -/// Copy the locked qualified snapshot to the C formatter's local output. -/// -/// # Safety -/// -/// `snapshot` must name a live StatsSnapshot retained by the registration. -/// `out` must be aligned and writable for one Readout for this call. -unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { - // SAFETY: C calls only while registration retains the snapshot Arc, and - // supplies a writable, aligned output for this call. - let snapshot = unsafe { &*snapshot.cast::() }; - let view = snapshot - .g17_reader - .as_ref() - .map_or_else(Readout::default, |reader| reader.read()); - // SAFETY: As above; the C output is initialized in one locked read. - unsafe { out.write(view) }; -} - -/// End the telemetry epoch at either side of system sleep. -/// -/// # Safety -/// -/// `snapshot` must name a live StatsSnapshot. The caller must join this -/// callback before releasing the registration's owning Arc. -unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { - // SAFETY: The notifier is unregistered and joined before the Arc is dropped. - if let Some(reader) = unsafe { &*snapshot.cast::() } - .g17_reader - .as_ref() - { - reader.suspend(suspended); - } -} - -fn register_with_callbacks( - dev: *mut kernel::bindings::device, - snapshot: *const StatsSnapshot, - export_enabled: bool, - read: Option, - suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -115,12 +28,9 @@ fn register_with_callbacks( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot.cast(), + snapshot, c_int::from(export_enabled), - &crate::memacct::LEDGER, &mut handle, - read, - suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs deleted file mode 100644 index 9f13436b4caf5..0000000000000 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ /dev/null @@ -1,220 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Prepared timestamp destinations shared by native command completion paths. - -use crate::mmu; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use crate::{file, gem}; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use core::ops::Range; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use core::sync::atomic::{fence as barrier, Ordering}; -use kernel::{prelude::*, sync::Arc}; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use kernel::{uapi, xarray}; - -/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the -/// buffer's lifetime so that commands need no mapping of their own. -pub(crate) struct FirmwareBuffer { - mapping: Arc, - cpu: mmu::TimestampVmap, -} - -/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. -/// CPU-retired timestamps retain their backing without consuming global firmware address space. -pub(crate) enum Buffer { - /// The last reference may be a command's, dropped where fence producers run, while - /// releasing the CPU mapping takes the reservation lock: the deferral runs the release - /// on the cleanup queue. - Firmware(crate::cleanup::Deferred), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Cpu { - // The GEM pin keeps the object's held SG table alive through CPU publication. - gem: gem::KernelMappingPin, - range: Range, - }, -} - -impl Buffer { - pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - let cpu = mapping.timestamp_vmap()?; - Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { - mapping: Arc::new(mapping, GFP_KERNEL)?, - cpu, - })?)) - } - - /// Retain a CPU-written byte range without publishing any firmware address. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { - let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; - if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { - return Err(EINVAL); - } - let gem = gem::KernelMappingPin::new(&bo.gem); - let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - let _ = gem::held_sg_table(&gem, parent.as_ref())?; - Ok(Self::Cpu { gem, range }) - } - - pub(crate) fn size(&self) -> usize { - match self { - Self::Firmware(buffer) => buffer.mapping.size(), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { range, .. } => range.end - range.start, - } - } - - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - match self { - Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { gem, range, .. } => { - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { - return Err(ERANGE); - } - mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) - } - } - } - - /// The object offset of the timestamp at `offset` of a firmware buffer, for - /// [`Self::write_timestamp`]. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { - match self { - Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => Err(EINVAL), - } - } - - /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], - /// through the buffer's own CPU mapping. - pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { - match self { - Self::Firmware(buffer) => buffer.cpu.write(offset, value), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => {} - } - } - - /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. - pub(crate) fn firmware_mapping(&self) -> Result> { - match self { - Self::Firmware(buffer) => Ok(buffer.mapping.clone()), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => Err(EINVAL), - } - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { - let offset = usize::try_from(offset).map_err(|_| EINVAL)?; - let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; - if aligned && offset % size_of::() != 0 { - return Err(EINVAL); - } - if end > size { - return Err(if aligned { EINVAL } else { ERANGE }); - } - Ok(offset) -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -struct Destination { - // Retain the bound range and backing along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl Destination { - fn resolve( - objects: Pin<&xarray::XArray>>, - timestamp: uapi::drm_asahi_timestamp, - aligned: bool, - ) -> Result> { - if timestamp.handle == 0 { - return Ok(None); - } - let object = objects - .lock() - .get(timestamp.handle.try_into()?) - .ok_or(ENOENT)? - .clone(); - let file::Object::TimestampBuffer(mapping) = object; - let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { - _mapping: mapping, - cpu, - })) - } - - fn write(&self, ticks: u64, frequency_hz: u64) { - const NSEC_PER_SEC: u64 = 1_000_000_000; - let ns = (ticks / frequency_hz) - .wrapping_mul(NSEC_PER_SEC) - .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); - self.cpu.write(ns); - barrier(Ordering::Release); - } -} - -/// Timestamp mappings resolved and retained before any command is enqueued. -#[cfg(CONFIG_DRM_ASAHI_G16)] -pub(crate) struct Destinations(Option; 4]>>); - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl Destinations { - /// G16 retains its eight-byte destination alignment and EINVAL contract. - pub(crate) fn resolve_aligned( - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], - ) -> Result { - Self::resolve_inner(objects, timestamps, true) - } - - fn resolve_inner( - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], - aligned: bool, - ) -> Result { - if timestamps.is_empty() || timestamps.len() > 2 { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; - } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) - } - - /// Publish measured timestamps before signalling completion or releasing dependent work. - pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { - for (destination, ticks) in self.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks, frequency_hz); - } - } - } - - fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = self - .0 - .as_deref() - .map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() - } -} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index 6e5540caeac3f..b51b027eb573c 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,13 +96,6 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } -/// A value that can be populated from an untrusted byte record. -/// -/// # Safety -/// -/// Every bit pattern must be valid for Self. Its byte representation must -/// contain no uninitialized bytes (including padding) when produced by Default, -/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -124,9 +117,7 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { - core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) - }; + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; dst.copy_from_slice(&src[..size]); self.offset = end; diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 4b613b1cc7f7a..c00da634972e5 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,16 +87,8 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; - - /* A bulk pin must not run direct reclaim once per folio when - * the driver can instead return an allocation failure. Clearing - * DIRECT_RECLAIM also makes memcg charges fail without reclaim - * or invoking the OOM killer. Preserve placement and kswapd. - */ - if (shmem->pages_no_reclaim) - gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); - mapping_set_gfp_mask(obj->filp->f_mapping, gfp); + mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | + __GFP_RETRY_MAYFAIL | __GFP_NOWARN); } return 0; @@ -491,23 +483,10 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, } EXPORT_SYMBOL_GPL(drm_gem_shmem_vunmap_locked); -/** - * drm_gem_shmem_create_with_handle - Allocate an object with the given size and - * returns a GEM handle - * @file_priv: DRM file structure to create the dumb buffer for - * @dev: DRM device - * @size: Size of the object to allocate - * @handle: Returns the GEM handle on success - * - * Allocates an shmem GEM buffer using drm_gem_shmem_create() and returns - * a GEM handle to it. - * - * Returns: - * Zero on success, or an error code otherwise. - */ -int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle) +static int +drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle) { struct drm_gem_shmem_object *shmem; int ret; @@ -526,7 +505,6 @@ int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, return ret; } -EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_handle); /* Update madvise status, returns true if not purged, else * false or -errno. @@ -634,10 +612,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return !drm_gem_is_imported(obj) && !obj->dma_buf && + return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == !!shmem->sgt && + refcount_read(&shmem->pages_use_count) == 1 && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1002,7 +980,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (vma_is_cow_mapping(vma)) + if (is_cow_mapping(vma->vm_flags)) return -EINVAL; #ifdef CONFIG_MM_SUBPAGE @@ -1037,7 +1015,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vma_get_page_prot(vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); @@ -1097,10 +1075,6 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; - /* Purged contents cannot be restored by rebuilding an empty table. */ - if (shmem->madv < 0) - return ERR_PTR(-EINVAL); - drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 8392f9e4a0109..d25bca5352aa6 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,13 +37,6 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; - /** - * @pages_no_reclaim: Disable direct reclaim during backing population. - * Set before drm_gem_shmem_init(). Drivers that can fail a large pin - * request cheaply may opt in; background reclaim remains enabled. - */ - bool pages_no_reclaim; - /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -152,9 +145,7 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && - !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == !!shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } @@ -168,10 +159,6 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); -int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle); - extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 418522205ee6f..d311b92620b86 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,20 +212,6 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; - - /** - * @memory_limit: Device-wide limit on user committed plus kernel - * resident bytes. Zero means no driver limit. This is a capacity, - * not current free memory; other clients share it. - */ - __u64 memory_limit; - - /** - * @client_memory_limit: Per-DRM-file limit on user committed bytes. - * Zero means no driver limit. Userspace must zero-initialize the reply - * buffer: older kernels leave these optional tail fields untouched. - */ - __u64 client_memory_limit; }; /** @@ -362,66 +348,6 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, - - /** - * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept - * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. - */ - DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, - - /* - * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports - * them; this kernel never does. They are listed so that the numbers are - * not given to anything else, and userspace reads every kernel's - * features the same way. Bits 25 and 27 are also reserved for that - * runtime. - */ - - /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ - DRM_ASAHI_FEATURE_RENDER_RESOURCES = (1UL) << 17, - - /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2: M5: versioned render storage. */ - DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 = (1UL) << 18, - - /** @DRM_ASAHI_FEATURE_SPARSE_TILES: M5: DRM_ASAHI_BIND_SPARSE_TILE. */ - DRM_ASAHI_FEATURE_SPARSE_TILES = (1UL) << 19, - - /** @DRM_ASAHI_FEATURE_COMPUTE_MATRIX: M5: compute matrix instructions. */ - DRM_ASAHI_FEATURE_COMPUTE_MATRIX = (1UL) << 20, - - /** @DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: M5: task and mesh storage. */ - DRM_ASAHI_FEATURE_TASK_MESH_STORAGE = (1UL) << 21, - - /** @DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: M5: per-render shader writes. */ - DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES = (1UL) << 22, - - /** - * @DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: M5: compressed - * multisample depth. - */ - DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION = (1UL) << 23, - - /** - * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a - * render's tiling half. - */ - DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, - - /** - * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed - * multisample stencil. - */ - DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, - - /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ - DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, - - /** - * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one - * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs - * include the earlier points of that timeline without adding GPU waits. - */ - DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -445,10 +371,7 @@ enum drm_asahi_feature { * The driver publishes an accepted job's failure before signaling its fence. * Thus userspace must query after a successful DRM syncobj wait/poll before * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. A job cancelled (ECANCELED) because - * another VM's recovery discarded it, an input did not signal in time, or - * its queue was closed, is not a failure of this VM: its fence carries the - * error, error stays 0, and its output is undefined. + * an execution-success indication. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. @@ -703,12 +626,6 @@ enum drm_asahi_bind_flags { * It is intended as a mechanism to accelerate sparse. */ DRM_ASAHI_BIND_SINGLE_PAGE = (1L << 3), - - /** - * @DRM_ASAHI_BIND_SPARSE_TILE: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_SPARSE_TILES). This kernel rejects it. - */ - DRM_ASAHI_BIND_SPARSE_TILE = (1L << 4), }; /** @@ -870,24 +787,6 @@ enum drm_asahi_cmd_type { * Followed by (possibly multiple) @drm_asahi_attachment payloads. */ DRM_ASAHI_SET_COMPUTE_ATTACHMENTS = 4, - - /** - * @DRM_ASAHI_SET_RENDER_RESOURCES: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_RENDER_RESOURCES). This kernel rejects it. - */ - DRM_ASAHI_SET_RENDER_RESOURCES = 5, - - /** - * @DRM_ASAHI_SET_COMPUTE_RESOURCES: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_COMPUTE_RESOURCES). This kernel rejects it. - */ - DRM_ASAHI_SET_COMPUTE_RESOURCES = 6, - - /** - * @DRM_ASAHI_SET_RENDER_RESOURCES_V2: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2). This kernel rejects it. - */ - DRM_ASAHI_SET_RENDER_RESOURCES_V2 = 7, }; /** @@ -907,8 +806,7 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails - * with -EACCES. + * Reserved for future extension. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -982,16 +880,6 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, - - /** - * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue - * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. - * At most one is allowed. All outputs receive completion of this submit - * and the earlier anchor points. - * Other outputs must not alias the anchor. No execution dependency is - * added. Calls updating this private anchor must be externally serialized. - */ - DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** @@ -1726,17 +1614,6 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), - - /** - * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, - * including compute-based transfers. Requires BOUNDED_AS_BUILD. - * This is an execution hint, not proof of progress or permission to - * access additional memory. The kernel validates the command normally - * and caps its execution allowance at ten seconds (four extra 2s polls). - * Other commands in the same submission receive the same allowance so - * ordered helpers cannot time out while waiting for the build. - */ - DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..099e64b3ce5fd 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,8 +30,6 @@ #include #include -#include -#include #include #include #include @@ -39,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -67,12 +64,9 @@ #include #include #include -#include #include #include -#include #include -#include #include #include #include @@ -99,18 +93,13 @@ #include #include #include -#include #include #include #include -#include #include #include -#include -#include #include #include -#include #include #include #include @@ -124,8 +113,6 @@ #include #include #include -#include -#include #include #include #include @@ -146,7 +133,6 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; -const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; @@ -157,7 +143,6 @@ const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; -const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; @@ -167,17 +152,8 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; -const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; -const u32 BINDINGS_IIO_COLORTEMP = IIO_COLORTEMP; -const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; -const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; -const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; -const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; -const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; -const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; -const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index d9933c4595682..b5540997bd20d 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,18 +23,6 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } -__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, - long value) -{ - add_mm_counter(mm, member, value); -} - -__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, - int member) -{ - return get_mm_counter_sum(mm, member); -} - __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3b76d329688db..4d6674f245049 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: Some(drm::File::::show_fdinfo_callback), + show_fdinfo: None, fbdev_probe: None, major: T::INFO.major, @@ -207,8 +207,7 @@ impl UnregisteredDevice { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = - drm::gem::create_fops(crate::module::this_module::().as_ptr()); + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// @@ -252,7 +251,7 @@ impl UnregisteredDevice { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { + unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; @@ -301,26 +300,6 @@ impl Device { self.dev.get() } - /// Reports that device execution is unavailable and the recovery method is unknown. - /// - /// May sleep: call in task context outside fence signalling and driver locks. - /// A registration guard keeps the parent bound throughout event delivery; a device - /// still being probed or already unplugged cannot deliver the event. - pub fn report_wedged(&self) -> Result { - // SAFETY: Every device context satisfies the weaker Normal context invariants. - let normal = unsafe { self.assume_ctx::() }; - let _registration = normal.registration_guard().ok_or(ENODEV)?; - let raw = self.as_raw(); - // SAFETY: The referenced device and its registered primary node remain live under - // the registration guard. A probe failure has already been rejected above. - if unsafe { (*raw).primary.is_null() } { - return Err(ENODEV); - } - // SAFETY: The device/node are live, 0 asks DRM to report an unknown recovery method, - // and a NULL task means no userspace task is attributed as guilty. - crate::error::to_result(unsafe { bindings::drm_dev_wedged_event(raw, 0, ptr::null_mut()) }) - } - /// # Safety /// /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 81f0a90e4a517..8a56f00203f56 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,23 +12,6 @@ use crate::{ }; use core::marker::PhantomData; -/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. -/// -/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub struct MemoryStats { - /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). - pub shared: u64, - /// Memory private to the client (part of `drm-total-*`). - pub private: u64, - /// Memory currently resident (`drm-resident-*`). - pub resident: Option, - /// Resident memory userspace marked purgeable (`drm-purgeable-*`). - pub purgeable: Option, - /// Memory in use by active submissions (`drm-active-*`). - pub active: Option, -} - /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -39,16 +22,6 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; - - /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. - fn memory_stats(&self) -> Option { - None - } - - /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. - /// Engine names must be stable, unique and contain no whitespace or colons. - /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. - fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -75,12 +48,6 @@ impl File { self.0.get() } - /// Whether this file holds the device's DRM master (the compositor or display server). - pub fn is_current_master(&self) -> bool { - // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. - unsafe { bindings::drm_is_current_master(self.as_raw()) } - } - /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -132,59 +99,6 @@ impl File { 0 } - /// The `show_fdinfo` callback of a `struct drm_driver`. - pub(crate) extern "C" fn show_fdinfo_callback( - printer: *mut bindings::drm_printer, - raw_file: *mut bindings::drm_file, - ) { - // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose - // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the - // file, so `postclose_callback()` cannot run concurrently. - let file = unsafe { File::::from_raw(raw_file) }; - file.inner().engine_stats(|name, ns| { - // SAFETY: The DRM core owns printer throughout this callback; the format and - // name are NUL-terminated, and the vararg types match %s and %llu. - unsafe { - bindings::drm_printf( - printer, - c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), - name.as_char_ptr(), - ns as core::ffi::c_ulonglong, - ) - }; - }); - let Some(stats) = file.inner().memory_stats() else { - return; - }; - - let mut supported = 0; - let mut raw = bindings::drm_memory_stats { - shared: stats.shared, - private: stats.private, - resident: 0, - purgeable: 0, - active: 0, - }; - if let Some(resident) = stats.resident { - raw.resident = resident; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; - } - if let Some(purgeable) = stats.purgeable { - raw.purgeable = purgeable; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; - } - if let Some(active) = stats.active { - raw.active = active; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; - } - - // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` - // and the region name outlive the call. - unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) - }; - } - /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 83eca852139be..fb7e5c38e5483 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,12 +82,6 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, - /// Fail backing population when it would need direct reclaim. - /// - /// Background reclaim and ordinary allocation placement are preserved. - /// Intended for large, fallible pin requests, not essential kernel backing. - pub pages_no_reclaim: bool, - /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -100,13 +94,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, - pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); +const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -122,12 +116,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// One cleanup registration for the C SG-table cache; never holds a table borrow. - sgt_cleanup: ManuallyDrop>>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, #[pin] - /// Serializes cleanup registration and table acquisition. + /// Lock for protecting initialization of `sgt_res`. sgt_lock: Mutex<()>, - /// Counted SG leases exclude backing eviction; the high bit excludes new leases. + /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. sgt_access: AtomicUsize, #[pin] inner: T, @@ -195,13 +189,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_cleanup` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -279,59 +273,67 @@ impl Object { self.make_vmap() } - /// Acquire a revocable lease on this object's DMA scatter-gather table. + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. /// - /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. - /// The table borrow cannot outlive the lease. The device must be this object's parent. - /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. - pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // Release and purge set the reclaim bit only while they hold this object's reservation. - // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. - while let Err(e) = self.acquire_sgt_access() { - if e != EBUSY { - return Err(e); - } - drop(DmaResvGuard::new(self)); - } - // Every failure after incrementing the count drops this lease. - let lease = SGLease { obj: self.into() }; - - // SAFETY: Import attachment ownership is immutable after publication. - let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // Imports belong to the exporter. Never register them for Rust devres cleanup, - // rebuild their tables, or release their backing. - // SAFETY: The attachment table stays valid until this object is freed. + // SAFETY: `import_attach` is set before an imported object is published and does not + // change until the object is freed. + if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // The SG table of an imported object is the dma-buf attachment's mapping, which the + // exporter made for this device at import time. It belongs to the attachment, and + // `drm_gem_shmem_release()` hands it back to the exporter through + // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` + // would unmap and free it as if these helpers had built it. + // + // SAFETY: `sgt` of an imported object is set before the object is published and stays + // valid and unmodified until the object is freed, which `'a` cannot outlive. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - sgt - } else { - // Register cleanup once, but cache the actual table only in C's shmem->sgt. - // Reclaim clears that pointer. No Rust reference to it survives without a lease. + // SAFETY: See above. + return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); + } + + // The borrowed reference can outlive any lock, so pin the backing permanently. + self.acquire_sgt_access(true)?; + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. let _guard = self.sgt_lock.lock(); - if self.sgt_cleanup.as_ref().is_none() { - self.sgt_cleanup.populate(Devres::new( - dev, - Ok::<_, Error>(SGTableCleanup { obj: self.into() }), - )?); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; } - self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; - // SAFETY: The live object and lease exclude backing release. The helper locks - // the reservation and either returns the cached table or rebuilds it. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } }; - Ok(SGTable { - // SAFETY: The lease excludes release; the Bound device excludes devres revocation. - table: unsafe { scatterlist::SGTable::from_raw(sgt) }, - lease, - }) + Ok(sgt_res.access(dev)?) } /// Create a new shmem-backed DRM object of the given size. @@ -347,7 +349,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -358,10 +360,6 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; - // SAFETY: The object is initialized but not published. The C initializer - // reads this policy when setting up the backing mapping's GFP mask. - unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; - // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -405,7 +403,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -431,18 +429,23 @@ impl Object { } } - fn acquire_sgt_access(&self) -> Result { + fn acquire_sgt_access(&self, permanent: bool) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } + let next = if permanent { + state | SGT_PERMANENT + } else { + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } + state + 1 + }; match self.sgt_access.compare_exchange_weak( state, - state + 1, + next, Ordering::AcqRel, Ordering::Acquire, ) { @@ -463,34 +466,11 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } - /// Whether the object still has a cached device SG table. - /// - /// # Safety - /// - /// The caller holds the object's DMA reservation lock. - pub unsafe fn has_sg_table_locked(&self) -> bool { - // SAFETY: Reservation ownership serializes the C table pointer. - !unsafe { (*self.as_raw_shmem()).sgt }.is_null() - } - - /// Whether backing without a Rust SG lease can be reclaimed or purged. - /// - /// Also accepts backing whose SG table was already released by idle reclaim. - /// - /// # Safety - /// - /// The caller holds the object's DMA reservation lock. - pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 0 && - // SAFETY: The caller establishes the reservation-lock contract. - unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } - } - /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on - /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. + /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that + /// borrow may outlive any lock. /// /// # Safety /// @@ -536,9 +516,7 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } - || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } - { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { return Err(EBUSY); } self.sgt_access @@ -728,72 +706,63 @@ where { } -/// An owned backing lease, suitable for retaining copied DMA segments. +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. /// -/// Dropping the last lease permits reclaim after device translations have been invalidated. -/// This does not itself provide access to the table: table access also requires a Bound device. -#[must_use = "retain until all device uses of the backing have ended"] -pub struct SGLease { - obj: ARef>, -} - -impl Drop for SGLease { - fn drop(&mut self) { - let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); - debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); - } -} - -/// A leased SG-table borrow. References obtained from it cannot outlive the lease. -#[must_use = "retain while using the SG table or transfer into an owned lease"] -pub struct SGTable<'a, T: DriverObject> { - table: &'a scatterlist::SGTable, - lease: SGLease, +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, } -impl Deref for SGTable<'_, T> { +impl Deref for SGTableMap { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - self.table - } -} - -impl SGTable<'_, T> { - /// Retain the backing after copying the DMA addresses, without retaining a table borrow. - pub fn into_lease(self) -> SGLease { - self.lease + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } } } -/// Device-unbind cleanup registration for locally allocated SG tables. -/// -/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may -/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. -struct SGTableCleanup { - obj: NonNull>, -} - -impl Drop for SGTableCleanup { +impl Drop for SGTableMap { fn drop(&mut self) { - // SAFETY: The registration is dropped before its GEM object. + // SAFETY: `obj` is always valid via our type invariants let obj = unsafe { self.obj.as_ref() }; let _lock = DmaResvGuard::new(obj); - // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. - // Reclaim may already have freed the table; the C free helper requires a non-null one. - unsafe { - if !(*obj.as_raw_shmem()).sgt.is_null() { - bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); - } - } + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; } } -// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. -unsafe impl Send for SGTableCleanup {} -// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. -unsafe impl Sync for SGTableCleanup {} +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -870,14 +839,6 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); - let dev: &device::Device = fdev.as_ref(); - // SAFETY: This new faux device is private to the fixture. Use direct DMA with a - // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. - unsafe { - let raw = dev.as_raw(); - (*raw).coherent_dma_mask = u64::MAX; - (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; - } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -924,225 +885,19 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.try_read8(offset).unwrap(), expected); + assert_eq!(vmap.read8(offset), expected); } Ok(()) } - #[test] - fn sg_lease_counting() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let second = obj.sg_table(dev.as_ref().as_ref())?; - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); - drop(first); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); - drop(second); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); - Ok(()) - } - - #[test] - fn release_idle_without_mappings() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. - unsafe { obj.release_idle_pages_locked()? }; - // SAFETY: Reservation ownership serializes these fields. - let empty = unsafe { - ( - (*obj.as_raw_shmem()).sgt.is_null(), - (*obj.as_raw_shmem()).pages.is_null(), - ) - }; - assert!(empty.0); - assert!(empty.1); - // SAFETY: Still no device mappings or leases; an already idle release succeeds too. - unsafe { obj.release_idle_pages_locked()? }; - Ok(()) - } - - #[test] - fn release_refused_while_leased() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no actual device mappings were installed. - assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); - // SAFETY: Reservation held; no actual device mappings were installed. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - drop(bind); - // SAFETY: Reservation held; the mapping lease is now the sole owner. - assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); - // SAFETY: Reservation held; the remaining lease must still exclude release. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - drop(mapping); - // SAFETY: Reservation held; no device mapping or SG lease remains. - unsafe { obj.release_idle_pages_locked()? }; - Ok(()) - } - - #[test] - fn purge_madvised_after_last_lease() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - assert!(obj.madvise(1)); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no real device access. - assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); - drop(mapping); - // SAFETY: Reservation held; all device uses and SG leases have ended. - unsafe { obj.purge_locked()? }; - // SAFETY: Reservation ownership serializes the backing fields and advice. - let (empty_sgt, empty_pages, madv) = unsafe { - ( - (*obj.as_raw_shmem()).sgt.is_null(), - (*obj.as_raw_shmem()).pages.is_null(), - obj.madv_locked(), - ) - }; - assert!(empty_sgt); - assert!(empty_pages); - assert_eq!(madv, -1); - } - assert!(!obj.madvise(0)); - assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); - Ok(()) - } - - #[test] - fn sg_rebuild_after_release() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - obj.vmap::<4>()?.as_view().write32(0x12345678, 0); - for _ in 0..3 { - let table = obj.sg_table(dev.as_ref().as_ref())?; - assert!(table.iter().count() > 0); - drop(table); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; all leases have dropped, with no device mappings. - unsafe { obj.release_idle_pages_locked()? }; - // SAFETY: Reservation ownership serializes the table pointer. - assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); - } - assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); - } - // Drop with a rebuilt table exercises the surviving devres cleanup registration too. - drop(obj.sg_table(dev.as_ref().as_ref())?); - Ok(()) - } - - #[test] - fn purge_after_idle_release() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device access or SG leases remain. - unsafe { obj.release_idle_pages_locked()? }; - } - assert!(obj.madvise(1)); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device access or SG leases remain. - assert!(unsafe { obj.unleased_pages_releasable_locked() }); - // SAFETY: Reservation held; no device access or SG lease remains. - unsafe { obj.purge_locked()? }; - // SAFETY: Reservation ownership serializes the advice and table pointer. - let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; - assert_eq!(madv, -1); - assert!(empty); - Ok(()) - } - - #[test] - fn purge_refuses_kernel_vmap() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - let vmap = obj.vmap::<4>()?; - assert!(obj.madvise(1)); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; the vmap must prevent the backing from being freed. - assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); - // SAFETY: Reservation held; the vmap must exclude idle release too. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - } - drop(vmap); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device, vmap or SG users remain. - unsafe { obj.purge_locked()? }; - Ok(()) - } - - #[test] - fn sg_reclaim_excludes_new_leases() -> Result { - let (_dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); - assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); - obj.sgt_access.store(SGT_USERS, Ordering::Release); - assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); - obj.sgt_access.store(0, Ordering::Release); - Ok(()) - } - - #[test] - fn sg_import_attachment_is_not_owned() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mut table = MaybeUninit::::zeroed(); - let mut attachment = MaybeUninit::::zeroed(); - // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay - // live until the lease drops and are detached before the real object destructor runs. - unsafe { - (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); - (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); - } - let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { - let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held. Capture results before restoring the fixture, so a - // failed assertion cannot leave a mock attachment for the GEM destructor. - unsafe { - let held = obj.release_idle_pages_locked(); - drop(lease); - ( - no_cleanup, - held, - obj.release_idle_pages_locked(), - obj.purge_locked(), - (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), - ) - } - }); - // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. - unsafe { - (*obj.as_raw()).import_attach = ptr::null_mut(); - (*obj.as_raw_shmem()).sgt = ptr::null_mut(); - } - let (no_cleanup, held, unleased, purge, unchanged) = result?; - assert!(no_cleanup); - assert_eq!(held, Err(EBUSY)); - assert_eq!(unleased, Err(EINVAL)); - assert_eq!(purge, Err(EBUSY)); - assert!(unchanged); - Ok(()) - } + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. - // A wrong device must fail before registering cleanup or taking any SG lease. + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -1153,10 +908,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // A failed lookup must not bind the cleanup registration to the wrong device. + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + // TODO: Someday, we should test that creating an sg_table here still succeeds. Ok(()) } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index e43d5ae131c85..4764d7b68f2a7 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,59 +151,6 @@ impl Mm { } } -/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). -/// -/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report -/// show for a process. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum RssCounter { - /// Resident file-backed pages (`MM_FILEPAGES`). - File, - /// Resident anonymous pages (`MM_ANONPAGES`). - Anon, - /// Swapped-out anonymous pages (`MM_SWAPENTS`). - Swap, - /// Resident shared-memory pages (`MM_SHMEMPAGES`). - Shmem, -} - -impl RssCounter { - fn as_raw(self) -> core::ffi::c_int { - let member = match self { - Self::File => bindings::MM_FILEPAGES, - Self::Anon => bindings::MM_ANONPAGES, - Self::Swap => bindings::MM_SWAPENTS, - Self::Shmem => bindings::MM_SHMEMPAGES, - }; - member as core::ffi::c_int - } -} - -// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these -// methods are safe even if `mm_users` is zero. -impl Mm { - /// Adds `pages` (which may be negative) to one of the resident-set counters. - /// - /// This is how a driver attributes pages it holds on behalf of a process, but which are not - /// mapped into it, to that process. Every page added must be subtracted again before the - /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad - /// rss-counter state. - #[inline] - pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { - // SAFETY: The pointer is valid since self is a reference, and the counters are initialized - // for as long as the `mm_struct` is. - unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; - } - - /// Returns the exact current value of one of the resident-set counters. - #[inline] - pub fn rss_counter(&self, counter: RssCounter) -> usize { - // SAFETY: The pointer is valid since self is a reference, and the counters are initialized - // for as long as the `mm_struct` is. - unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } - } -} - // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer. From a8f092b08ba07ebf988d48418001565203f93d26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 19:13:02 -0400 Subject: [PATCH 009/392] drm/asahi: sync the G17 driver with aurora-silicon at d3cf19be Bring drivers/gpu/drm/asahi up to the aurora-silicon/linux state at d3cf19be, the base of the G17 performance series that follows, merged three-way on top of integration/aurora with the 7.2.10 merge point 44d5afc7 as base. integration/aurora's own work is kept: the G15 backend, the M3 T8122 runtime, UAT adoption, deferred-close binding sequences (#214) and the 7.2.11 shared-code adaptations. The G16 (T8132) backend is not carried. This carries the G17 work merged on the aurora side since 7.2.10, including the T6040 (M4 Pro) and 25G83 firmware support and the render replay/recovery series by Jonathan Goyvaerts, the watchdog, replay and backing-population changes by Ryan Murray, the tracked VM lifetime work by Eryk Wieliczko, and the G17 queue, capacity and timestamp changes by DjDeveloperr (aurora-silicon/linux #382, #391, #397, #400). Adaptations to 7.2.11: - module parameters return references; THIS_MODULE replaces kernel::module::this_module; a local raw_try_init replaces the newer pin-init helper; of_device_table! takes the MODULE_OF_TABLE name. - regs.rs keeps G17's sgx window as Devres, as 7.2 has no DevresIoMem. - The shared feature flags keep G17's opt-out of soft faults. - G15 and the M3 runtime use the shared timestamp::Buffer and the reserved-table aware root walk; omacom's G15 vm_closed hook becomes vm_destroyed, next to the G17 vm_closed. - Crate and helper pieces the driver needs: the shmem SG lease and pages_no_reclaim (drm_gem_shmem_helper), DRM file memory stats, fdinfo and is_current_master, and the mm RSS counter. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/asahi_core.rs | 14 + drivers/gpu/drm/asahi/driver.rs | 13 +- drivers/gpu/drm/asahi/file.rs | 153 ++- drivers/gpu/drm/asahi/g15/command.rs | 3 +- drivers/gpu/drm/asahi/g15/mod.rs | 6 +- drivers/gpu/drm/asahi/g15/uat.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 64 +- drivers/gpu/drm/asahi/g17/buffer/target.rs | 61 +- drivers/gpu/drm/asahi/g17/capacity.rs | 88 ++ drivers/gpu/drm/asahi/g17/command.rs | 50 +- drivers/gpu/drm/asahi/g17/completion.rs | 130 ++- drivers/gpu/drm/asahi/g17/context.rs | 45 +- drivers/gpu/drm/asahi/g17/context/identity.rs | 9 + drivers/gpu/drm/asahi/g17/data_segment.rs | 56 ++ drivers/gpu/drm/asahi/g17/dependency.rs | 8 + drivers/gpu/drm/asahi/g17/event.rs | 8 +- drivers/gpu/drm/asahi/g17/feed.rs | 23 +- drivers/gpu/drm/asahi/g17/freelist.rs | 141 ++- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 77 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 157 ++- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 681 +++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/queue.rs | 196 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 303 +++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 133 +++ drivers/gpu/drm/asahi/g17/initdata.rs | 202 ++-- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 238 +++++ drivers/gpu/drm/asahi/g17/job.rs | 71 +- drivers/gpu/drm/asahi/g17/kick.rs | 381 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 469 +++++++-- drivers/gpu/drm/asahi/g17/object.rs | 112 ++- drivers/gpu/drm/asahi/g17/power.rs | 123 +++ drivers/gpu/drm/asahi/g17/preparation.rs | 153 +++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 474 ++++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 456 +++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 3 + .../drm/asahi/g17/queue/compute/retirement.rs | 180 +++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 36 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 518 +++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 111 ++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 43 +- .../drm/asahi/g17/queue/render/publication.rs | 34 +- .../drm/asahi/g17/queue/render/retirement.rs | 8 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/recovery.rs | 35 +- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 84 ++ drivers/gpu/drm/asahi/g17/recovery/slot.rs | 8 +- drivers/gpu/drm/asahi/g17/replay.rs | 109 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 605 +++++++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 102 +- .../drm/asahi/g17/runtime/compute_control.rs | 18 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 19 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 306 +++++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 7 +- .../drm/asahi/g17/runtime/render/observe.rs | 6 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 8 + .../drm/asahi/g17/runtime/render/prepare.rs | 77 +- .../drm/asahi/g17/runtime/render/replay.rs | 148 +-- .../drm/asahi/g17/runtime/render/settle.rs | 16 +- .../drm/asahi/g17/runtime/render/submit.rs | 218 +++- .../drm/asahi/g17/runtime/render/teardown.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- drivers/gpu/drm/asahi/g17/teardown.rs | 15 + drivers/gpu/drm/asahi/g17/timeout.rs | 257 ++++- drivers/gpu/drm/asahi/g17/validation.rs | 5 +- drivers/gpu/drm/asahi/gem.rs | 68 +- drivers/gpu/drm/asahi/gpu.rs | 59 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t6040.rs | 180 ++++ drivers/gpu/drm/asahi/hw/t8140.rs | 37 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 5 +- drivers/gpu/drm/asahi/mem.rs | 42 +- drivers/gpu/drm/asahi/memacct.rs | 549 ++++++++++ drivers/gpu/drm/asahi/mmu.rs | 943 ++++++++++++++---- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 79 +- drivers/gpu/drm/asahi/mmu/m3.rs | 2 +- drivers/gpu/drm/asahi/mmu/ppl.rs | 71 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 69 +- drivers/gpu/drm/asahi/mmu/shared.rs | 4 +- drivers/gpu/drm/asahi/object.rs | 2 +- drivers/gpu/drm/asahi/pgtable.rs | 409 +++++--- drivers/gpu/drm/asahi/queue/common.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 20 +- drivers/gpu/drm/asahi/regs.rs | 64 +- drivers/gpu/drm/asahi/sysfs.c | 106 +- drivers/gpu/drm/asahi/sysfs.h | 3 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 21 +- drivers/gpu/drm/asahi/timestamp.rs | 185 ++++ drivers/gpu/drm/asahi/util.rs | 14 + drivers/gpu/drm/drm_gem_shmem_helper.c | 20 +- include/drm/drm_gem_shmem_helper.h | 11 +- include/uapi/drm/asahi_drm.h | 34 +- rust/bindings/bindings_helper.h | 1 + rust/helpers/mm.c | 12 + rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/file.rs | 69 ++ rust/kernel/drm/gem/shmem.rs | 487 ++++++--- rust/kernel/mm.rs | 53 + 104 files changed, 10248 insertions(+), 1518 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs create mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs create mode 100644 drivers/gpu/drm/asahi/g17/replay.rs create mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs create mode 100644 drivers/gpu/drm/asahi/memacct.rs create mode 100644 drivers/gpu/drm/asahi/mmu/ppl.rs create mode 100644 drivers/gpu/drm/asahi/timestamp.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0fbe6e728773d..8b782a6f8ffa7 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -25,6 +25,7 @@ mod gpu; mod hw; mod initdata; mod mem; +mod memacct; mod m3; mod microseq; mod mmu; @@ -35,6 +36,7 @@ mod regs; mod slotalloc; mod stats; mod sysfs_exports; +mod timestamp; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -172,5 +174,17 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, + mem_limit_pct: u32 { + default: 67, + description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", + }, + client_mem_limit_pct: u32 { + default: 50, + description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", + }, + oom_attribution: u8 { + default: 1, + description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d..9b9b8fc4ccd15 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -103,7 +103,6 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -153,6 +152,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -233,6 +236,7 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); + crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -419,6 +423,11 @@ impl AsahiPlatformDriver { return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + } Ok(AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, @@ -433,7 +442,7 @@ impl AsahiPlatformDriver { impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a..c627bd9c6b8a0 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + memacct, + gpu, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -218,7 +223,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -236,6 +241,8 @@ pub(crate) struct File { /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, + /// GPU memory this client created and holds. + mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -255,9 +262,20 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } @@ -270,26 +288,37 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + crate::util::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + crate::util::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + crate::util::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); + addr_of_mut!((*slot).mem).write(mem); Ok(()) }) } @@ -341,7 +370,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +389,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -383,7 +415,7 @@ impl File { // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { @@ -402,15 +434,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +456,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +527,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +618,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -589,7 +641,7 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); + reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -632,8 +684,24 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = gem::user_pages_no_reclaim( + size, + reg_data.gpu.as_any().is::(), + ); + let gem = gem::new_object( + device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + )?; if reg_data.gpu.prepare_new_objects() { gem::prepare_new_object(&gem)?; } @@ -713,11 +781,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +802,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +820,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1268,27 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + return Err(EINVAL); + } + // SAFETY: capable() only reads the calling task's credentials (ioctl process context). + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + && !file.is_current_master() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1260,6 +1350,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1385,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1369,7 +1462,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026..b343aa3fe4e5b 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -360,7 +360,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 95f04114ba874..48e6afa948c98 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -632,7 +632,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_destroyed(&self) { vm::closed(self) } @@ -658,7 +658,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 7ab3a1919eaec..1d63fb83b9fa1 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -99,7 +99,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea0..8f3a03fbb8dba 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } @@ -1119,3 +1131,51 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } + +impl BufferIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", + self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, + self.entries.iter().filter(|entry| entry.owner).count(), + self.entries.iter().filter(|entry| entry.references != 0).count()) + } +} +impl MetricsIds { + pub(crate) fn used(&self) -> u32 { + self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() + - (128 - fw::METRICS_SLOTS as u32) + } +} + +#[kunit_tests(asahi_g17_buffer_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_build_and_fault_retirement_return_parameter_buffers() { + let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); + for _ in 0..BUFFER_IDS { + clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); + } + assert_eq!(ids.allocate_render(), Err(ENOSPC)); + for (index, id) in clients.into_iter().enumerate() { + if index % 2 == 0 { + // Failed constructor never installed the graph owner. + ids.release(id).unwrap(); + } else { + ids.reserve_owner(id).unwrap(); + ids.release(id).unwrap(); + ids.acquire_owner(id).unwrap(); + // Close cannot return a still-owned failed command. + ids.unreserve_owner(id).unwrap(); + assert!(!ids.entries[id as usize].free()); + ids.release(id).unwrap(); + } + } + assert!(ids.entries.iter().all(|entry| entry.free())); + assert_eq!(ids.returned_count, BUFFER_IDS); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c93..64c88da8eca14 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at +/// the default parameter-buffer sizes. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs new file mode 100644 index 0000000000000..bcbbb2e7e875d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Name capacity refusals at their source, before a common submit or fence +//! failure handler loses their provenance. Counters are device-lifetime +//! independent diagnostics, never admission or ownership state. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Site { + ComputeReserve, + PhysicalQids, + RenderSlots, + LogicalQueues, + ComputeSlots, + QosIds, + ComputeItems, + RetiredRenderBindings, + RenderPoolIds, + RenderWorkSlots, + ParameterBuffers, + MetricsIds, + LowerVa, + UpperVa, + AliasVa, + ExecutionContexts, +} + +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; +static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; + +/// First and power-of-two refusals per site keep a pressure storm bounded in +/// the log. This does not change whether the caller retries or rejects work. +#[track_caller] +pub(super) fn refused(site: Site) -> Error { + let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); + if count.is_power_of_two() { + let at = core::panic::Location::caller(); + pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); + } + ENOSPC +} + +/// Read-only diagnostics hold the live device session while sampling its pools. +/// They never wake firmware, reclaim a resource, or change admission state. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), + u8::from(self.0.crashed.load(Ordering::Acquire)))?; + let uat = { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1")?; + firmware.queues.write_capacity(f)?; + firmware.pool.write_capacity(f)?; + writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; + firmware.uat.clone() + }; + // Mapping destruction can issue firmware flushes while holding VM + // locks. Never take a VM/TTBAT lock with the device mutex held here. + // The retained UAT owns valid allocator metadata even during shutdown. + writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let mut upper = uat.geometry().kernel_range(); + upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + for (name, vm, range) in [ + ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), + ("dynamic_upper_bytes", uat.kernel_vm(), upper), + ] { + match vm.capacity(range.clone()) { + Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, + Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, + } + } + for (site, count) in [ + Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, + Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, + Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, + ].iter().zip(REFUSALS.iter()) { + writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 3dfa760969224..c65f7de6ac915 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,6 +12,8 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; +const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); + const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -121,7 +123,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -131,7 +133,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if header.cmd_type as u32 + let payload = if u32::from(header.cmd_type) == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -238,7 +240,7 @@ impl UscWindow { } require_range( space, - self.base + (tagged_offset & !7) as u64, + self.base + u64::from(tagged_offset & !7), 4, Access::Read, ) @@ -306,12 +308,14 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { + long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -323,6 +327,19 @@ pub(crate) enum Validated { }, } +impl Validated { + pub(crate) fn long_job(&self) -> bool { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, + } + } + pub(crate) fn allow_long_job(&mut self) { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, + } + } +} + impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -334,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -407,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -440,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -447,6 +470,8 @@ impl Payload { } } Ok(Validated::Render { + long_job: super::timeout::select(false, + ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -454,8 +479,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 + let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; + if cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -470,14 +495,19 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { + long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -489,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231db..4f5b87bc853dc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -28,7 +28,7 @@ pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, + _mapping: Arc, cpu: mmu::TimestampMapping, } @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,84 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + +/// Republications of one command whose earlier publications firmware dropped unexecuted. +pub(crate) const MAX_REPLAYS: u8 = 3; + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +252,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -185,11 +264,18 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, + /// A recovery blamed this command. Unlike `class`, which a later + /// classification can change, this never clears. + blamed: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { + #[expect( + clippy::too_many_arguments, + reason = "Completion construction takes distinct queue, context, and lifetime owners." + )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -216,7 +302,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -235,6 +321,7 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), + blamed: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -278,14 +365,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -305,7 +392,7 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < 3).then_some(count + 1) + (count < MAX_REPLAYS).then_some(count + 1) }) .is_ok(); if accepted { @@ -335,7 +422,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -384,6 +471,20 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } + pub(crate) fn guilty(&self) -> bool { + self.class.load(Ordering::Acquire) == 2 + } + + /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, + /// so it is never published again. + pub(crate) fn note_blamed(&self) { + self.blamed.store(true, Ordering::Release); + } + + pub(crate) fn blamed(&self) -> bool { + self.blamed.load(Ordering::Acquire) + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -401,6 +502,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -411,7 +518,7 @@ impl Completion { /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(); + lease.quarantine(self.spared()); } } @@ -442,8 +549,11 @@ impl Completion { feed.note_render_pass(ticks[2], ticks[3]); } if let Err(error) = result { + // Spared work cancelled after another VM's recovery is not this client's device loss. if self.spared() { - self.status.report_failure(error); + if error != ECANCELED { + self.status.report_failure(error); + } } else if error != ECANCELED { self.status.record(error); } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44bee..c93ab25d55ce3 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; Ok(QosLease { @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -194,12 +196,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -387,7 +393,8 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context.work.lock().render.get(pair).ok_or_else(|| + super::capacity::refused(super::capacity::Site::RenderWorkSlots)) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -406,3 +413,37 @@ impl Drop for WorkStateLease { } } } + +impl QosIds { + pub(crate) fn used(&self) -> u32 { + (*self.used.lock() & !3u128).count_ones() + } +} + +#[kunit_tests(asahi_g17_qos_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_client_construction_returns_every_qos_id() { + let ids = QosIds::new().unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); + for _ in 0..126 { + clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 126); + assert!(matches!(ids.acquire(), Err(ENOSPC))); + drop(clients); + assert_eq!(ids.used(), 0); + assert_eq!(*ids.used.lock(), 3); + } + } +} + +impl Context { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { + writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", + scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 6c48367e2d686..595c7154d42ed 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -69,3 +69,12 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } + +impl WorkStates { + pub(super) fn used(&self) -> usize { + self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() + } +} +impl RenderSlots { + pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } +} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs new file mode 100644 index 0000000000000..02bcc66de8f51 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/data_segment.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2026 The Gravity Linux Contributors +// +// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and +// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. + +//! Crash log buffers in the GPU firmware's data segment. +//! +//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it +//! by physical address inside its data segment, the `fw-data` reserved region, which the boot +//! loader keeps for the loaded firmware until reboot. + +use kernel::{ + c_str, device, + io::{ + self, + IoSysMap, + Region, // + }, + prelude::*, +}; + +/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. +pub(super) struct DataSegmentLog { + mem: io::mem::Mem, + offset: usize, + size: usize, +} + +impl DataSegmentLog { + /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which + /// must lie inside the `fw-data` region of `dev`. + pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { + return Err(EINVAL); + } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + Ok(Self { mem, offset, size }) + } + + /// The CPU view of the buffer. + pub(super) fn buf(&mut self) -> Result> { + // SAFETY: map() checked that the range lies inside the mapping. + let base = unsafe { self.mem.ptr().add(self.offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5abd494eeb1f5..0bc1d5024566e 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -17,6 +17,14 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// A replay's original host inputs must still be signalled. Treat an unmet +/// input as a failed replay, never as a wait under the device mutex. +pub(crate) fn replay_status(status: i32) -> Result { + if status == 0 { Err(EBUSY) } + else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else { Ok(()) } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c81..9508a827be4b1 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 4cd05fde16581..0045dff413922 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -147,6 +147,10 @@ impl Requests { } } + /// Add a policy to the pinned requests. + /// + /// # Safety + /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -179,7 +183,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } } @@ -193,7 +197,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + bindings::FREQ_QOS_MIN_DEFAULT_VALUE }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -211,7 +215,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } applied } @@ -235,7 +239,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } fn clear(mut self: Pin<&mut Self>) { @@ -364,6 +368,13 @@ impl Feed { } } + /// Process a cpufreq policy notification. + /// + /// # Safety + /// + /// `block` must be this pinned Feed's registered notifier, and `data` must + /// point to the live policy supplied by the cpufreq notifier core. + /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -387,12 +398,12 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as _; + return bindings::NOTIFY_DONE as i32; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as _ + bindings::NOTIFY_OK as i32 } fn now() -> u64 { diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc..3b31c3cead064 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) } } impl Drop for RenderId { @@ -69,19 +69,84 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -100,7 +165,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - fn page_list_va(&self) -> u64 { + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -144,14 +209,17 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, @@ -522,6 +590,18 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } + self.write_pages(&pages)?; + self.pages = Some(pages); + Ok(()) + } + /// Preserve both sets of pages on failure. The caller owns the consumed + /// release and returns old pages for destruction outside the device mutex. + pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { + self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; + core::mem::swap(&mut self.pages, pages); + Ok(()) + } + fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -540,9 +620,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); - self.pages = Some(pages); Ok(()) } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -742,3 +822,30 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } + +impl RenderIds { + pub(crate) fn used(&self) -> u32 { + self.used[0].load(Ordering::Acquire).count_ones() + + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() + } +} + +#[kunit_tests(asahi_g17_render_id_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn rejected_pool_construction_returns_all_render_ids() { + let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); + for _ in 0..127 { + pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 127); + assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); + drop(pools); + assert_eq!(ids.used(), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index ec6797637bdbe..9d430d50db365 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c4cf6960db296..524132c4ae7be 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -498,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: first as u8, + first: u8::from(first), }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0633da7f43541..0cb7304e30677 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u16::MAX as u64; - let cliques = (layout >> 24) & u8::MAX as u64; - let user = (layout >> 32) & u16::MAX as u64; - let driver = (layout >> 48) & u16::MAX as u64; + let depth = layout & u64::from(u16::MAX); + let cliques = (layout >> 24) & u64::from(u8::MAX); + let user = (layout >> 32) & u64::from(u16::MAX); + let driver = (layout >> 48) & u64::from(u16::MAX); if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,13 +152,42 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster of six cores (A18 Pro). + pub(crate) const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + pub(crate) const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +312,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +336,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -347,14 +379,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +396,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 1f25bd0fc1c0d..3092e599caa84 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -591,31 +608,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va - .checked_add(self.size as u64)? + let start = va & !mask; + let span = va + .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +657,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +678,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -676,14 +704,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +761,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +791,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) @@ -1169,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1197,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1213,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1224,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1234,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1285,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1319,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1364,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1411,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 0000000000000..c237896a49fb1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index e07fbfb485a10..4f3a70671eb87 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -297,7 +297,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, }) } @@ -332,7 +332,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | ((self.count - 1) as u64) << Self::COUNT_SHIFT + | u64::from(self.count - 1) << Self::COUNT_SHIFT } } @@ -444,8 +444,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | (args.qid as u64) << Self::QID_SHIFT - | (args.priority as u64) << Self::PRIORITY_SHIFT; + | u64::from(args.qid) << Self::QID_SHIFT + | u64::from(args.priority) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -478,9 +478,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: mask as u64 - | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT - | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + dependency_control: u64::from(mask) + | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT + | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index b6bb6aaea860f..1cb46fce2d602 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -30,7 +33,7 @@ pub(crate) mod qos; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; + unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 48b110a4193e3..5708a2251a062 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -94,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -115,26 +121,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +152,62 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + /// Offsets of the two host words a [`PointerLayout`] may name. + const WORD_40: usize = core::mem::offset_of!(Self, word_40); + const WORD_50: usize = core::mem::offset_of!(Self, word_50); + + /// Whether `offset` is one of the host words a [`PointerLayout`] may name. + const fn is_host_word(offset: usize) -> bool { + offset == Self::WORD_40 || offset == Self::WORD_50 + } + + /// Whether `layout` names host words of the block, and two different ones when it has a + /// recovery target. Every firmware variant's layout is checked at build time. + pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { + match layout.recovery_target { + None => Self::is_host_word(layout.producer), + Some(target) => { + Self::is_host_word(layout.producer) + && Self::is_host_word(target) + && target != layout.producer + } + } + } /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Result { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target)?.store(u32::MAX, Ordering::Relaxed); + } + Ok(block) + } + + /// The host word at `offset`, which must be one a [`PointerLayout`] may name. + fn word(&self, offset: usize) -> Result<&AtomicU32> { + match offset { + Self::WORD_40 => Ok(&self.word_40), + Self::WORD_50 => Ok(&self.word_50), + _ => Err(EINVAL), } } @@ -174,12 +216,14 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { + let producer = self.word(layout.producer)?; self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + producer.store(base, Ordering::Relaxed); + Ok(()) } } @@ -271,12 +315,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [args.policy.priority() as u32; 2], + priority: [u32::from(args.policy.priority()); 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: args.policy.policy() as u32, + policy: u32::from(args.policy.policy()), unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -307,12 +351,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [policy.priority() as u32; 2]; + let priority = [u32::from(policy.priority()); 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != policy.policy() as u32 { - self.policy = policy.policy() as u32; + if { self.policy } != u32::from(policy.policy()) { + self.policy = u32::from(policy.policy()); } } } @@ -480,7 +524,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - binding.fragment_qid as u32, + u32::from(binding.fragment_qid), ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -491,14 +535,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: args.qid as u16, - install: args.install as u16, + qid: u16::from(args.qid), + install: u16::from(args.install), unk_1c: 0, - priority: args.policy.priority() as u16, + priority: u16::from(args.policy.priority()), unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed_valid: u32::from(args.completion_seed.is_some()), completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -506,13 +550,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: args.qos_update as u32, - qos_slot: args.qos_slot as u16, + qos_update: u32::from(args.qos_update), + qos_slot: u16::from(args.qos_slot), unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: args.context_update as u16, + context_update: u16::from(args.context_update), unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -525,6 +569,94 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp, in addition to the kick-entry dependency. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u64, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id, + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -564,7 +696,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: qid as u16, + qid: u16::from(qid), count: 1, priority, timestamp: timestamp.get(), @@ -603,7 +735,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: qid as u16, + qid: u16::from(qid), previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 85a2385b421b8..0e270d40392a8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,53 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// One cluster (A18 Pro). + pub(crate) const ONE_CLUSTER: Deflake = Deflake { + sub: [0x20, 0x2a0], + viewport: true, + }; + /// Two clusters (M4 Pro). + pub(crate) const TWO_CLUSTERS: Deflake = Deflake { + sub: [0xa0, 0x820], + viewport: false, + }; +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +147,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +309,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +364,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -357,12 +411,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = pass.width as u64; - let height = pass.height as u64; + let width = u64::from(pass.width); + let height = u64::from(pass.height); let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / pass.utile_width as u64; - let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles_x = TILE_SIZE / u64::from(pass.utile_width); + let utiles_y = TILE_SIZE / u64::from(pass.utile_height); let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -411,7 +465,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -441,19 +495,19 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u32::MAX as u64, + false => spec & u64::from(u32::MAX), }; - let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); Ok(Self { - utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | pass.samples_log2 as u64, + utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | u64::from(pass.samples_log2), tile_config, layer_mode, - tib_blocks: (pass.sample_size as u64 - * pass.utile_width as u64 - * pass.utile_height as u64 + tib_blocks: (u64::from(pass.sample_size) + * u64::from(pass.utile_width) + * u64::from(pass.utile_height) * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, @@ -491,9 +545,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let deflake = user_offset32(r.deflake_va)? as u64; - let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; - let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let [deflake_20, deflake_2a0] = args.deflake.sub; + let deflake = u64::from(user_offset32(r.deflake_va)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -561,7 +616,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -573,7 +628,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -581,6 +636,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +766,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -609,8 +792,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, p.merge_upper_x as u64), - RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), + RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -625,15 +808,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, p.depth_clear as u64), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15301, u64::from(p.depth_clear)), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -682,11 +865,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -716,7 +899,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -742,13 +925,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -760,13 +943,13 @@ fn partial_load_registers( RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -902,7 +1085,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,7 +1103,11 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { let mut trimmed = [registers[0]; TA_REGISTERS]; @@ -945,7 +1133,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -969,7 +1160,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = fragment_busy as u32; + self.fragment_busy = u32::from(fragment_busy); self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -980,11 +1171,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1365,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,7 +1448,12 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; @@ -1289,12 +1493,15 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = (args.ordinal == 0) as u32; + desc.first_kick = u32::from(args.ordinal == 0); desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 0000000000000..2f7b598733368 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, +} + +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, + init_ack: InitAck::Message, + power_on_done: true, + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, + lead_commands: false, + programs: super::render::RegisterPrograms::G17P, +}; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; + +const _: () = { + assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); +}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09e..10af261aeb3ac 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -121,7 +123,7 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - self.record as *const _ as *const u8, + core::ptr::from_ref(self.record).cast::(), self.destination, core::mem::size_of_val(self.record), ); @@ -155,6 +157,15 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -174,6 +185,8 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, + /// GPU clusters, as the GPU reports them. + clusters: u32, } impl BuildEnv<'_> { @@ -235,6 +248,11 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +265,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -363,7 +385,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); } Ok(()) })?; @@ -502,7 +524,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -522,7 +544,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -601,7 +623,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -694,7 +716,12 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + clusters: u32, + active_cores: u32, + variant: &super::fw::Variant, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -703,9 +730,13 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + clusters, }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; + if variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } + self.map_register_windows(&env, cfg.windows)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -713,31 +744,41 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -939,12 +980,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for slot in 0..2 { + for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - completion_rings[slot] = abi::CompletionRing { + *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -988,13 +1029,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1002,7 +1043,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1050,8 +1091,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1107,7 +1148,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1150,7 +1191,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1180,37 +1221,65 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + /// The GPU's operating points: `count` of them, each with one voltage cell for the whole + /// GPU or one per cluster. + fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + let cells = opps.first().map_or(0, |opp| opp.cells); + if cells != 1 && u32::try_from(cells) != Ok(clusters) { + dev_err!( + dev.as_ref(), + "GPU operating points have {} voltage cells, expected 1 or {}\n", + cells, + clusters + ); + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1218,7 +1287,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1247,9 +1316,14 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1261,7 +1335,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1336,7 +1410,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1351,8 +1425,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1363,20 +1441,20 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; @@ -1391,21 +1469,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1417,17 +1495,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1437,7 +1519,7 @@ impl super::recovery::Memory for InitData { fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); let mut entries = super::recovery::Entries::default(); for index in 0..256 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 0000000000000..85aa6468fa311 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6d..61b8c3b7b4737 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -143,23 +143,40 @@ impl Packet { } } + /// Replay holds the device mutex. Its original inputs were satisfied before + /// first publication; never sleep on a fence that needs event service here. + pub(crate) fn check_replay_dependencies(&self) -> Result { + for fence in &self.inputs { + super::dependency::replay_status(fence_status(fence))?; + } + for fence in &self.firmware { check_status(fence)?; } + Ok(()) + } + /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { + let polls = if self.command.long_job() { 5 } else { 1 }; + let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); + loop { + if self.cancel_requested() || self.completion.status().get() != 0 { + return Err(ECANCELED); + } + if fence_status(fence) != 0 { break; } + if super::now_ns() >= deadline { return Err(ECANCELED); } + // SAFETY: The retained fence owns the pointer throughout the bounded wait. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::time::msecs_to_jiffies(10) as isize, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited > 0 { break; } } check_status(fence)?; } @@ -179,6 +196,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -201,7 +225,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |_| 1), + result.map_or_else(|error| error.to_errno(), |()| 1), Ordering::Release, ); } @@ -242,7 +266,8 @@ pub(crate) trait Backend: Send + Sync { None } - /// Lazily create or rebind this logical queue's physical compute graph. + /// Validate compute sizing without reserving a physical channel. Queued + /// commands may wait on a host signal submitted after this ioctl returns. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -271,6 +296,7 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, + execution_renewals: u8, packet: Arc, } @@ -287,12 +313,14 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, + execution_renewals: 0, packet: self.packet.clone(), } } @@ -301,9 +329,14 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } + if super::timeout::renew_execution( + job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { - job.victim_renewals += 1; + if deferred { + job.victim_renewals = job.victim_renewals.saturating_add(1); } deferred } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86d..c337a8b298f19 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -17,8 +24,24 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Minimum render-channel capacity protected from compute-only allocation. +// Protect one tiling/fragment pair for each logical execution context, plus +// the two unpublished bootstrap reservations. The remaining ten compute +// channels are shared after exact retirement, rather than pinned per client. +const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +/// Logical queues that can each hold a render pair: what the render reserve leaves beside the +/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone +/// (every execution context with no floor). +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + +const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); + assert!(COMPUTE_CHANNELS >= 4); + assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); +}; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -47,19 +70,42 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - compute_publication: u64, + initial_render: [Option; 2], + publication: u64, leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - compute_publication: 0, + initial_render: [None; 2], + publication: 0, leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -83,16 +129,56 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + // The QIDs below the floor are never the host's; the reserve shrinks + // with them, so the compute channel count stays the same. + let total = QID_COUNT - self.floor; + let reserve = COMPUTE_RENDER_RESERVE - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| { + matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) + }) + .count(); + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let replacement = usize::from(initial.is_some()); + if total - self.leased + replacement + <= reserve.saturating_sub(render - replacement) + { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); + } } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) + .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { qid: index as u8, owner, @@ -102,14 +188,19 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. - pub(crate) fn next_compute_publication(&mut self) -> Result { - self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.compute_publication) + /// Ordered under the device mutex; no atomic operation on the submission path. Compute and + /// render publications share it, so a watchdog can order work across both engines. + pub(crate) fn next_publication(&mut self) -> Result { + self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.publication) } /// Call after preflighting the complete installation, before any address @@ -127,6 +218,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -140,6 +238,82 @@ impl QueueIds { } } +#[kunit_tests(asahi_g17_qid_accounting)] +mod accounting_tests { + use super::*; + + fn table() -> KBox { + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() + } + + #[test] + fn render_identities_satisfy_the_compute_reserve() { + let mut ids = table(); + for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + assert_eq!(ids.leased, QID_COUNT); + } + + #[test] + fn compute_cannot_consume_the_render_minimum() { + let mut ids = table(); + for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + ids.cancel_unpublished(tiling).unwrap(); + assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + } + + #[test] + fn initial_reservations_yield_but_published_ids_do_not() { + let mut ids = table(); + ids.reserve_initial_render().unwrap(); + for owner in 3..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); + let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); + assert_ne!(first.qid(), second.qid()); + assert!(ids.initial_render.iter().all(Option::is_none)); + assert_eq!(ids.leased, QID_COUNT); + ids.publish(first).unwrap(); + assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT); + ids.cancel_unpublished(second).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT - 1); + } + + #[test] + fn failed_second_render_id_returns_only_the_unpublished_first() { + let mut ids = table(); + for owner in 1..QID_COUNT as u64 { + let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + ids.publish(id).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); + ids.cancel_unpublished(first).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); + assert_eq!(replacement.qid(), first.qid()); + assert_ne!(replacement, first); + assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -147,10 +321,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -186,6 +444,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -220,14 +481,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -255,14 +547,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -272,19 +576,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -297,7 +601,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -353,3 +657,20 @@ impl Queue { }) } } + +impl QueueIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let mut engines = [0usize; 3]; + let mut published = 0; + for entry in self.entries.iter().flatten() { + engines[match entry.id.engine { + DataMaster::Tiling => 0, + DataMaster::Fragment => 1, + DataMaster::Compute => 2, + }] += 1; + published += usize::from(entry.published); + } + writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", + self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9da90d2b0843a..2999daec47015 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -23,6 +23,7 @@ mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; +mod data_segment; mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; @@ -35,13 +36,15 @@ mod ids; pub(crate) mod hazard; mod initdata; mod job; +mod capacity; mod kick; mod object; -mod power; +pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; +mod replay; mod runtime; pub(crate) mod status; mod teardown; @@ -149,12 +152,63 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, + /// Layouts of the command buffers that follow the GPU's core layout. + pub(crate) core_layout: CoreLayout, +} + +/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. +pub(crate) struct CoreLayout { + /// The cluster and enabled core counts the layouts were measured on, which the GPU must + /// report at probe; `None` where the layouts hold for every configuration of the SoC. + pub(crate) cores: Option<(u32, u32)>, + /// A compute queue's operand-state buffer. + pub(crate) operand_state: fw::compute::OperandState, + /// A render's deflake buffer. + pub(crate) deflake: fw::render::Deflake, +} + +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -197,6 +251,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -207,7 +264,15 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData(data_segment::DataSegmentLog), } impl rtkit::Buffer for CrashLog { @@ -216,7 +281,10 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData(log) => log.buf(), + } } } @@ -225,8 +293,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -305,13 +373,23 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { - return Err(EINVAL); + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); } + // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( + &data.shared.dev, + iova, + size, + )?), }) } } @@ -324,7 +402,7 @@ struct Coprocessor { running: bool, rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -338,27 +416,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), + }; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -404,8 +487,9 @@ impl Coprocessor { Pin::new(&mut self.rtkit).boot() } - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { let mut rtkit = Pin::new(&mut self.rtkit); rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; rtkit.as_mut().start_endpoint(EP_DOORBELL)?; @@ -413,7 +497,10 @@ impl Coprocessor { EP_FIRMWARE, MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), )?; - self.state.wait_for_ack() + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. @@ -452,6 +539,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -472,15 +567,21 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + hold: power::Hold, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -490,7 +591,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -500,6 +603,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -519,6 +631,11 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { + // An idle hold keeps the cores powered past registration; it releases them itself. + if self.hold.active() { + self.registration_power_release_pending = false; + return Ok(()); + } power::release(self) } @@ -548,14 +665,49 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - match self.effort.idle(now_ns, in_flight) { - power::IdleAction::None => Ok(None), + let effort = match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => None, power::IdleAction::Clear => { self.apply_effort(0)?; - Ok(None) + None + } + power::IdleAction::Recheck(delay) => Some(delay), + }; + Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }) + } + + /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A + /// refused control (recovery in progress) is retried on a later check. + fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { + const RETRY_NS: u64 = 1_000_000; + let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); + for _ in 0..2 { + match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + power::HoldAction::None => return None, + power::HoldAction::Recheck(delay) => return Some(delay), + power::HoldAction::Assert => { + if self.set_idle_control(false).is_err() { + return None; + } + self.hold.asserted(); + } + power::HoldAction::Release => { + if self.set_idle_control(true).is_err() { + return Some(RETRY_NS); + } + self.hold.released(); + return None; + } } - power::IdleAction::Recheck(delay) => Ok(Some(delay)), } + None + } + + fn set_idle_control(&mut self, enabled: bool) -> Result { + power::Control::set_idle(self, enabled) } } @@ -610,6 +762,11 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, + /// Layout of renders' deflake buffers for this GPU's cores. + deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -742,12 +899,20 @@ impl Shared { } fn drain_reclaims(self: &Arc) { + let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); + reclaimed = true; + } + if reclaimed { + // Context release returns its execution root off-lock. The next + // device pass can now detach idle compute aliases and their QoS + // leases, without depending on a new client's submission/event. + self.queue_events(); } } @@ -826,6 +991,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _capacity: Pin>>, } impl Gpu { @@ -836,6 +1002,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -860,12 +1027,28 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + if cfg + .core_layout + .cores + .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + { + dev_err!( + dev.as_ref(), + "Unsupported GPU core layout: {} clusters with {} cores\n", + id.num_clusters, + active_cores + ); + return Err(ENODEV); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant, + operand_state: &cfg.core_layout.operand_state, + deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -884,6 +1067,11 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", capacity::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -892,6 +1080,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + _capacity: capacity, }, GFP_KERNEL, )?; @@ -901,6 +1090,42 @@ impl Gpu { Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -932,6 +1157,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -948,11 +1174,20 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -964,9 +1199,17 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + hold: power::Hold::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: shared.variant, + queues <- runtime::Registry::new( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + ), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -977,45 +1220,95 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, + shared.clusters, + shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; + } + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if shared.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1081,12 +1374,20 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; - params.gpu_generation = hw::GpuGen::G17 as u32; + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + let variant = self.shared.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1115,9 +1416,18 @@ impl gpu::Gpu for Gpu { true } + /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not + /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 + /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads + /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case + /// failed; all pass with soft faults off). + fn supports_soft_faults(&self) -> bool { + false + } + fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1156,12 +1466,26 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.shared.variant, + )?; let policy = scheduling_policy(priority)?; + let admission_started = now_ns(); let (backend, slot) = loop { match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, + Err(error) if error == ENOSPC => { + // Closed logical queues can still own execution/QoS IDs + // until the scheduler release is consumed and reclaimed. + // Backend construction has rolled back all temporary leases. + if now_ns().saturating_sub(admission_started) + >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } result => break result?, } }; @@ -1185,27 +1509,28 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { + ) -> Result { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; let mut addresses = firmware.uat.geometry().kernel_range(); addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( + crate::timestamp::Buffer::firmware(bo.map_range_into_range( firmware.uat.kernel_vm(), range, addresses, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so -/// MEDIUM arrives as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH +/// arrives as 1, MEDIUM as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { + 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae..f1b22ad5d7331 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,7 +78,17 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + Placement::In(range, align) => { + let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start + && range.end <= crate::hw::t8140::dynamic::LOWER.end { + super::capacity::Site::LowerVa + } else { + super::capacity::Site::UpperVa + }; + this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + if error == ENOSPC { super::capacity::refused(site) } else { error } + })? + } }; this.mapping = Some(mapping); Ok(this) @@ -113,6 +123,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -159,6 +172,20 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) + } + + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -341,6 +368,76 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = (if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + ) + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + ) + }).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) + } else { error })?; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + ).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) + } else { error })?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware @@ -577,3 +674,16 @@ impl Pool { } } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let inner = self.inner.lock(); + for family in 0..POOL_FAMILIES { + let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; + writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, + inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), + entries.iter().flatten().filter(|entry| entry.poisoned).count())?; + } + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 96b363bc27447..7fef15084c50a 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,9 +3,19 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. +use core::sync::atomic::AtomicU32; + pub(super) const FULL_EFFORT: u32 = 0x10000; const IDLE_HOLD_NS: u64 = 16_000_000; +/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after +/// the last job. Not measured on other firmware; at or below it the hold stays off. +pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; + +/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU +/// cores powered until the GPU has been idle this long ([`Hold`]). +pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -17,6 +27,7 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, + submissions: u64, } impl Effort { @@ -25,12 +36,19 @@ impl Effort { streaming: false, idle_since: None, applied: 0, + submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; + self.submissions = self.submissions.wrapping_add(1); + } + + /// Count of published submissions, for [`Hold::step`]. + pub(super) fn submissions(&self) -> u64 { + self.submissions } pub(super) fn applied(&self) -> u32 { @@ -62,6 +80,83 @@ impl Effort { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum HoldAction { + None, + /// Send the keep-powered idle policy. + Assert, + /// Send the power-off idle policy again. + Release, + Recheck(u64), +} + +/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job +/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer +/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off +/// policy once no work has been submitted or in flight for that long. Checked from the event +/// worker, which runs on every completion, well inside the firmware's 2 ms. +pub(super) struct Hold { + active: bool, + seen: u64, + since: Option, +} + +impl Hold { + pub(super) const fn new() -> Self { + Self { + active: false, + seen: 0, + since: None, + } + } + + pub(super) fn active(&self) -> bool { + self.active + } + + pub(super) fn step( + &mut self, + now_ns: u64, + in_flight: bool, + submissions: u64, + hold_ms: u32, + ) -> HoldAction { + let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; + let fresh = submissions != self.seen; + self.seen = submissions; + if in_flight || fresh { + self.since = None; + } + if !self.active { + return if enabled && (in_flight || fresh) { + HoldAction::Assert + } else { + HoldAction::None + }; + } + if in_flight { + return HoldAction::None; + } + let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; + let since = *self.since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= hold_ns { + HoldAction::Release + } else { + HoldAction::Recheck(hold_ns - idle) + } + } + + pub(super) fn asserted(&mut self) { + self.active = true; + } + + pub(super) fn released(&mut self) { + self.active = false; + self.since = None; + } +} + /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -137,3 +232,31 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] +mod tests { + use super::*; + + #[test] + fn holds_until_idle_for_the_delay_then_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); + // A new submission restarts the idle period. + assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + + #[test] + fn the_default_delay_never_holds_and_lowering_it_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); + assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 8563beda979e6..0297c619b4f93 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -20,6 +20,34 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; +/// One ready packet's monotonic compute-admission budget. The callback checks +/// packet cancellation, logical close, VM status and device loss even when no +/// capacity/recovery event arrives. Wakes and nested constructors never renew it. +pub(super) struct Admission<'a> { + deadline: u64, + check: &'a dyn Fn() -> Result, +} + +impl<'a> Admission<'a> { + pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { + Self { deadline, check } + } + + pub(super) fn check(&self) -> Result { + (self.check)()?; + if super::now_ns() >= self.deadline { return Err(EBUSY); } + Ok(()) + } + + /// Ignore condvar remaining-time reports: only the monotonic deadline counts. + pub(super) fn slice_ms(&self) -> Result { + self.check()?; + let remaining = self.deadline.saturating_sub(super::now_ns()); + if remaining == 0 { return Err(EBUSY); } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) + } +} + #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -92,6 +120,27 @@ impl Gate { } } + /// Packet admission may wait for recovery only within its original budget. + /// Called with no device mutex or existing preparation lease. + pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let _ = self.changed.wait_timeout( + &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); + } + } + fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -129,6 +178,17 @@ impl Gate { self.changed.notify_all(); } + /// Runtime failure/recovery must not join an indefinitely stalled builder. + /// Timeout retains every graph and reservation; it never authorizes freeing. + pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + if super::now_ns() >= deadline { return Err(ETIMEDOUT); } + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); + } + Ok(()) + } + /// Join builders with no device, queue or mapping lock held. pub(super) fn wait_drained(&self) { let mut wait = self.wait.lock(); @@ -184,3 +244,96 @@ impl Drop for Lease { } } } + +#[kunit_tests(asahi_g17_growth_gate)] +mod growth_gate_tests { + use super::*; + #[test] + fn recovery_can_join_between_growth_admissions() { + let gate = Gate::new().unwrap(); + let outer = gate.try_enter().unwrap(); + // The growth handoff must drop this lease before enter can wait. + drop(outer); + gate.close(); + assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); + gate.wait_drained(); + assert!(gate.reopen()); + let growth = gate.enter(false).unwrap(); + assert!(growth.is_current()); + drop(growth); + } +} + +#[kunit_tests(asahi_g17_compute_deadline)] +mod compute_deadline_tests { + use super::*; + + #[test] + fn spent_budget_refuses_even_an_open_gate() { + let gate = Gate::new().unwrap(); + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn closed_gate_expires_without_any_recovery_wake() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(super::super::now_ns() - start < 500_000_000); + gate.wait_drained(); + assert!(gate.reopen()); + } + + #[test] + fn cancellation_without_capacity_or_recovery_events() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || { + if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }; + let admission = Admission::new(start + 2_000_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn recovery_and_binding_retry_churn_cannot_renew_budget() { + let gate = Gate::new().unwrap(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + loop { + // Re-entry models a channel taken between admission/publication. + // Recovery joins each dropped lease, then wakes the same budget. + gate.close(); + gate.changed.notify_all(); + gate.wait_drained(); + assert!(gate.reopen()); + match gate.enter_until(&admission) { + Ok(lease) => drop(lease), + Err(error) => { assert_eq!(error, EBUSY); break; } + } + } + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn packet_status_and_cancel_precede_available_capacity() { + let gate = Gate::new().unwrap(); + for error in [ECANCELED, EIO] { + let check = || Err(error); + let admission = Admission::new(u64::MAX, &check); + assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f..ebdc79d2b2dd3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -47,12 +47,25 @@ struct InFlight { const QUARANTINE_WAIT_MS: u32 = 1000; +/// A wake never renews admission's budget. Short observations also notice a +/// packet cancellation that has no corresponding pair-release notification. +fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { + let remaining = deadline_ns.saturating_sub(now_ns); + if remaining == 0 { + return Err(EBUSY); + } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) +} + struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, + /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail + /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. + spared: u64, kill_hold: u64, released: u64, epoch: u64, @@ -68,6 +81,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, + spared: 0, kill_hold: 0, released: 0, epoch: 0, @@ -92,7 +106,7 @@ impl State { & Self::MASK == 0 => { - Err(ENOSPC) + Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) } _ => Ok(()), } @@ -115,18 +129,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,9 +191,52 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; } } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; + } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true + } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { return None; @@ -188,9 +247,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -283,23 +364,33 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, } impl Pool { - pub(crate) fn new() -> Result> { + pub(crate) fn new(max_queues: u32) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, }), GFP_KERNEL, ) } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); + if count >= self.max_queues { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( count, @@ -316,16 +407,22 @@ impl Pool { self: &Arc, owner: u64, request: Option>, + deadline_ns: u64, + check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let started = crate::g17::now_ns(); + let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); + let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); let mut expired = false; let mut ticket = None; let result = (|| loop { + check()?; + let now = crate::g17::now_ns(); + let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -337,20 +434,26 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { - return Err(EIO); + if now >= quarantine_deadline { + // Behind an innocent pair's quarantine (another VM wedged the GPU) only this + // publication fails, not the VM: it recovers once the pair drains. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + return Err(if spared { EBUSY } else { EIO }); + } + let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { + return Err(ERESTARTSYS); } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; continue; } state.refusal(owner)?; + expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -376,18 +479,17 @@ impl Pool { } } } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { + } + // Keep the established hazard wait/fallback rule, but cap every + // sleep with the caller's original deadline and cancellation poll. + let slice = if plan != Plan::Any { + remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) + } else { + remaining_ms + }; + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { return Err(ERESTARTSYS); } })(); @@ -420,8 +522,33 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -446,12 +573,82 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; + state.spared &= !released; state.kill_hold = 0; drop(state); self.released.notify_all(); } } +#[kunit_tests(asahi_g17_admission_accounting)] +mod accounting_tests { + use super::*; + + #[test] + fn every_logical_owner_fits_without_speculative_spares() { + let mut state = State::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot = state.acquire(owner).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); + state.release(slot); + } + assert_eq!(state.active, 0); + assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); + } + + #[test] + fn close_does_not_reuse_an_active_lease() { + let mut state = State::new(); + let slot = state.acquire(1).unwrap(); + state.released |= 1u64 << slot; + assert!(!state.reassign_unused_spare(slot, Some(1), None)); + state.release(slot); + assert!(state.reassign_unused_spare(slot, Some(1), None)); + assert_eq!(state.owners[usize::from(slot)], 0); + assert_eq!(state.refs[usize::from(slot)], 0); + assert_eq!(state.active & (1u64 << slot), 0); + } + + #[test] + fn warming_quarantine_and_exit_hold_exclude_reassignment() { + let mut state = State::new(); + let primary = state.acquire(1).unwrap(); + // A retained legacy sibling still obeys all quarantine/kill guards. + state.owners[2] = 1; + let spare = state.borrow_warm_sibling(1, primary).unwrap(); + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.release(spare); + state.quarantined |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.quarantined = 0; + state.kill_hold |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.kill_hold = 0; + assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); + assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + state.release(primary); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + } + + #[test] + fn logical_queue_count_returns_on_drop() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..MAX_EXECUTION_CONTEXTS { + leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); + assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); + drop(leases.pop()); + let replacement = pool.reserve_queue().unwrap(); + drop(leases); + assert_eq!(pool.queues.load(Ordering::Acquire), 1); + drop(replacement); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } +} + /// Counted once per userspace logical queue, independently of installed graphs. pub(crate) struct QueueSlot { pool: Arc, @@ -468,17 +665,40 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } - pub(crate) fn quarantine(&self) { - self.pool.state.lock().quarantined |= 1u64 << self.slot; + /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine + /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. + pub(crate) fn quarantine(&self, spared: bool) { + let bit = 1u64 << self.slot; + let mut state = self.pool.state.lock(); + if !spared { + state.spared &= !bit; + } else if state.quarantined & bit == 0 { + state.spared |= bit; + } + state.quarantined |= bit; + drop(state); self.pool.released.notify_all(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - self.pool.state.lock().quarantined &= !(1u64 << self.slot); + let mut state = self.pool.state.lock(); + state.quarantined &= !(1u64 << self.slot); + state.spared &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { @@ -539,3 +759,169 @@ impl Pool { self.released.notify_all(); } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; + let state = self.state.lock(); + let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { + bits | if *owner != 0 { 1u64 << slot } else { 0 } + }) & State::MASK; + let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; + writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", + used.count_ones(), State::MASK.count_ones(), retained.count_ones(), + (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), + (state.kill_hold & State::MASK).count_ones()) + } +} + +#[kunit_tests(asahi_g17_fault_admission)] +mod fault_capacity_tests { + use super::*; + + #[test] + fn a_guilty_failure_makes_a_spared_quarantine_guilty() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + lease.quarantine(true); + assert_ne!(pool.state.lock().spared & bit, 0); + lease.quarantine(false); + assert_eq!(pool.state.lock().spared & bit, 0); + // Once guilty, a later spared failure does not make it spared again. + lease.quarantine(true); + assert_eq!(pool.state.lock().spared & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + pool.release_owner(1); + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.quarantined | state.spared, 0); + } + + #[test] + fn closed_faulting_clients_return_leases_after_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + for owner in 1..=512 { + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + lease.quarantine(false); + pool.release_owner(owner); + let bit = 1u64 << lease.slot(); + assert_ne!(pool.state.lock().active & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + // The runtime, not an error fence, supplies this exact drain witness. + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.active, 0); + assert_eq!(state.quarantined, 0); + assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } + } + + #[test] + fn published_exit_kill_holds_survive_close_and_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); + // A failed (unpublished) control may roll back its hold. Published kills + // keep the existing processor-stop rule; this test must not weaken it. + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().kill_hold, 0); + } +} + +impl Pool { + pub(crate) fn owner_active(&self, owner: u64) -> bool { + let state = self.state.lock(); + state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) + } +} + +#[kunit_tests(asahi_g17_shared_render_depth)] +mod shared_depth_tests { + use super::*; + #[test] + fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { + let mut state = State::new(); + let first = state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } + assert_eq!(state.acquire(1), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + state.release(first); + assert_eq!(state.acquire(1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_admission_deadline)] +mod deadline_tests { + use super::*; + + #[test] + fn all_pairs_busy_no_progress_expires_without_release() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + // No pair has room, including the requesting owner's retained pair. + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + assert!(matches!(result, Err(error) if error == EBUSY)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn busy_pair_observes_cancellation_without_a_release_event() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }); + assert!(matches!(result, Err(error) if error == ECANCELED)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn cancellation_precedes_available_capacity() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + Err(error) if error == ECANCELED)); + assert_eq!(pool.state.lock().active, 0); + } + + #[test] + fn unrelated_wake_storm_never_renews_absolute_deadline() { + let deadline = 2_000_000_000; + // Each wake is allowed to report the full jiffy budget remaining. Only + // monotonic time, not that report or a wake count, bounds admission. + for now in (0..deadline).step_by(100_000) { + assert!(wait_slice_ms(deadline, now).unwrap() <= 10); + } + assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); + assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb760..713809907dd4b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,7 +9,12 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{ + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, + }, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -113,17 +118,26 @@ struct Graph { records: KernelObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new( + alloc: &Allocator<'_>, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; @@ -141,7 +155,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; @@ -160,6 +174,8 @@ impl Graph { records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, + operand_state, }) } @@ -221,7 +237,7 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) @@ -258,7 +274,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); + return Err(EAGAIN); } } Ok(()) @@ -276,7 +292,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(POINTERS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -389,11 +407,21 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous { - binding: Option, - context: Arc, +struct Previous> { + binding: Option, + context: C, publication: Option<(u64, u64)>, } +impl Previous { + /// The exact drain witness covers the replacement publication even when + /// its public fence failed. An unpublished replacement still needs rollback. + fn retire_witnessed(previous: &mut Option) { + if previous.as_ref().is_some_and(|p| p.publication.is_some()) { + drop(previous.take()); + } + } +} + /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -453,12 +481,21 @@ pub(crate) enum PoolPhase { Vacant, } +/// Builder ownership of one physical release transaction. The preparation +/// lease pins this registry until abort/commit; owner close may remove lookups. +#[derive(Clone, Copy)] +pub(crate) struct Growth { + qid: u8, + phase: PoolPhase, +} + pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -471,6 +508,8 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, + /// This binding has never crossed the successor visibility boundary. + unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -509,9 +548,11 @@ impl Queue { backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -520,6 +561,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -527,6 +569,7 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, + unpublished_binding: true, kick, pool, graph, @@ -566,6 +609,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -574,18 +632,16 @@ impl Queue { pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - pub(crate) fn room(&self) -> bool { - !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT - } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: Arc::as_ptr(active.packet.completion.status()) as usize, + vm: active.packet.context.vm().id(), published: active.published, finished: active.observed.is_some(), + failed: self.quarantined, }) } @@ -626,6 +682,25 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + + pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { + self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + } + + /// A closed owner can still retain its last commands or their completion + /// stamps. Admission may wait for retirement, without treating that close + /// as permission to reuse any firmware-owned backing. + pub(crate) fn release_settling(&self) -> bool { + // release_owner keeps the old owner while its last commands retire. + // Waiting only once owner is None misses precisely that interval. + release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -780,9 +855,7 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = front - .observed_at - .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); + let record_aged = retirement::record_aged(front.observed_at, now_ns()); let timestamps = retirement::poll( &ticket, front.observed, @@ -829,6 +902,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { @@ -929,6 +1003,10 @@ impl Queue { self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; + // The stamp and fully consumed ring proved the replacement retired. + // Keeping its predecessor only on the fault path prevents recycling + // forever: neither another retirement nor an exit kill can release it. + Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -937,15 +1015,9 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } if self.released { self.owner = None; + self.released_at = now_ns(); } } Ok(()) @@ -1027,6 +1099,102 @@ impl Queue { } } +/// Validate the complete unpublished item budget before changing descriptors, +/// work counts or queue producers. Leave one item empty across ring wrap. +fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { + if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); + return Err(EAGAIN); + } + Ok(()) +} + +/// Retained commands are capacity, not a failed VM. Nothing has been written +/// when this check runs, so the outer publisher can wait for retirement under +/// its original admission deadline, just as it does for a full firmware ring. +fn retained_capacity( + quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, +) -> Result { + if quarantined || released { + return Err(EBUSY); + } + if active >= IN_FLIGHT || (replays != 0 && !replay_front) { + return Err(EAGAIN); + } + Ok(()) +} + +/// Logical close promises a future healthy handoff, not immediate reuse. +/// Exact retirement still owns the transition. +fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { + released && !quarantined && (active != 0 || retiring) +} + +#[kunit_tests(asahi_g17_compute_admission)] +mod admission_tests { + use super::*; + + #[test] + fn retained_ring_pressure_waits_without_discarding_work() { + let mut active = Fifo::new(); + for sequence in 0..IN_FLIGHT { + active.push(sequence).unwrap(); + } + assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!(active.front(), Some(&0)); + assert_eq!(active.len, IN_FLIGHT); + assert_eq!(active.pop(), Some(0)); + assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!(active.front(), Some(&1)); + } + + #[test] + fn replay_front_and_terminal_ownership_remain_distinct() { + assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); + assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); + assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + for (quarantined, released) in [(true, false), (false, true), (true, true)] { + assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + } + } + + #[test] + fn item_refusal_is_retryable_and_preserves_the_empty_slot() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + } + + #[test] + fn bad_item_cursors_are_not_backpressure() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); + assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + } + + #[test] + fn closed_owner_waits_for_its_last_command() { + assert!(release_pending(true, false, 1, false)); + assert!(release_pending(true, false, 0, true)); + assert!(!release_pending(true, false, 0, false)); + } + + #[test] + fn live_or_quarantined_owners_do_not_promise_capacity() { + assert!(!release_pending(false, false, 1, true)); + assert!(!release_pending(true, true, 1, true)); + } +} + impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1107,6 +1275,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1134,6 +1303,13 @@ impl Queue { } self.retirement_ready = true; } + // Rebinding an already detached queue has no Previous. If its + // first command is rejected or cancelled before publication, the + // empty active FIFO is still the original idle witness. No future + // completion can turn retirement_ready back on for this owner. + if self.previous.is_none() { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1259,6 +1435,7 @@ impl Queue { ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); + self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1283,7 +1460,7 @@ impl Queue { let pages = self.pool.take_pages(); let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; + self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1298,12 +1475,11 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } + if !self.can_rearm() { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1314,11 +1490,7 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { - if blocks < self.growth_blocks { return Err(EAGAIN); } - self.growth_blocks = blocks; - return Ok(false); - } + if self.growing { return Err(EAGAIN); } if self.quarantined || self.released || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) @@ -1327,7 +1499,8 @@ impl Queue { let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - } else { self.pool_phase = PoolPhase::Vacant; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) @@ -1344,16 +1517,35 @@ impl Queue { return Err(EAGAIN); } } + if alias.is_none() { return Err(EINVAL); } + let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - let old_alias = binding.pool_alias.take(); - let old_pages = self.pool.take_pages(); - self.pool.rearm(pages.take().ok_or(EINVAL)?)?; - init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; - binding.pool_alias = alias.take(); + // No ownership moves before every fallible descriptor/header operation. + // The old release is consumed; no queue can use this row until commit. + init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + self.pool.replace_pages(pages)?; + let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + } + + pub(crate) fn growth_token(&self) -> Option { + self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + } + + /// Transfer an abandoned builder's release back to the event worker. Do not + /// revoke the published release or free backing here: vacancy still requires + /// its exact control-consumer witness. A different transaction is untouched. + pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { + if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { + return Ok(None); + } + self.growing = false; + self.growth_blocks = 0; + self.release_owner().map(Some) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -1436,14 +1628,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(POINTERS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(POINTERS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1535,7 +1730,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1618,6 +1813,7 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); + item_capacity(self.graph.cursors()?, entry_signal)?; let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1640,6 +1836,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -1726,16 +1923,6 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); - } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1743,6 +1930,7 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1779,6 +1967,7 @@ impl Queue { Err(error) })?; visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1898,11 +2087,8 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } + retained_capacity(self.quarantined, self.released, self.active.len, + self.replays.len, replay_front)?; if !packet.context.is_current() || !self .binding @@ -1969,7 +2155,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -1982,3 +2168,153 @@ impl Queue { self.terminal.take() } } + +impl Queue { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", + self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), + u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), + u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; + match self.graph.cursors() { + Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + _ => writeln!(f, "items_unavailable=1"), + }?; + writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), + u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + if let Some(binding) = self.binding.as_ref() { + binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + } + if let Some(previous) = self.previous.as_ref() { + previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + } + Ok(()) + } +} + +#[kunit_tests(asahi_g17_fault_ownership)] +mod fault_ownership_tests { + use super::*; + use core::sync::atomic::AtomicUsize; + + struct Lease(Arc); + impl Lease { + fn new(used: &Arc) -> Self { + used.fetch_add(1, Ordering::Relaxed); + Self(used.clone()) + } + } + impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + + // This tests the production predecessor ownership operation without firmware + // storage. Runtime tests must separately establish its drain/kill witness. + fn faulting_clients(publication: Option<(u64, u64)>) { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + for _ in 0..512 { + let mut previous = Some(Previous { + binding: Some(Lease::new(&used)), + context: Lease::new(&used), + publication, + }); + assert_eq!(used.load(Ordering::Relaxed), 2); + Previous::retire_witnessed(&mut previous); + assert_eq!(previous.is_none(), publication.is_some()); + if publication.is_none() { + assert_eq!(used.load(Ordering::Relaxed), 2); + // An unpublished replacement returns its original owner on close. + drop(previous.take()); + } + assert_eq!(used.load(Ordering::Relaxed), 0); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn faulted_published_replacements_return_to_baseline() { + faulting_clients(Some((1, 7))); + } + + #[test] + fn unpublished_replacements_survive_until_rollback() { + faulting_clients(None); + } + + #[test] + fn same_vm_replacement_without_binding_returns_its_context() { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + let mut previous = Some(Previous:: { + binding: None, + context: Lease::new(&used), + publication: Some((9, 3)), + }); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } +} + +impl Queue { + /// Offer an open client's idle physical channel to another ready job. + /// Logical queue/context ownership is unchanged. The existing binding is + /// retained through the successor's exact replacement witness. + pub(crate) fn share_idle(&mut self) -> Result { + if !self.installed || self.owner.is_none() || !self.retirement_ready + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.previous.is_some() || self.pool_phase != PoolPhase::Populated + { + return Ok(false); + } + self.release_owner()?; + Ok(self.reusable()) + } +} + +impl Queue { + pub(crate) fn can_rearm(&self) -> bool { + self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + } +} + +impl Queue { + /// A failed first attempt has no future completion to undo its binding. + /// Return the replaced aliases to the caller for off-lock destruction. The + /// caller validates the current preparation epoch and cancels only a true + /// result's never-published QID; installed rings and identities stay intact. + pub(crate) fn rollback_unpublished( + &mut self, owner: u64, context: &Arc, + ) -> Result<(bool, Option)> { + if self.owner != Some(owner) || !self.unpublished_binding + || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.retire_pending || self.failure_status_pending + || self.pool_phase != PoolPhase::Populated + || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + { + return Ok((false, None)); + } + if !self.installed { return Ok((true, None)); } + let mut retired = None; + if let Some(previous) = self.previous.as_ref() { + self.graph.set_owner(&previous.context)?; + } + if let Some(mut previous) = self.previous.take() { + if let Some(binding) = previous.binding.take() { + retired = self.binding.replace(binding); + } else { + self.binding.as_mut().ok_or(EIO)?.context = previous.context; + } + } + self.unpublished_binding = false; + self.retirement_ready = true; + self.released = true; + self.owner = None; + self.released_at = now_ns(); + Ok((false, retired)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 187fd27f8e78e..d7386e335e167 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -220,6 +220,9 @@ impl Queue { ))?; drop(self.active.pop()); } + // The acknowledged kill and command settlement replace the ordinary + // drain witness, including a failed first publication after rebinding. + super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421..e938ec566dbac 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -66,21 +66,37 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. The firmware stores the stamp after the cache -/// maintenance that follows the record; it occasionally never stores it for a command, most -/// visibly while a render of the same VM is running. -pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; +/// before it retires on the record alone. +/// +/// The firmware stores the stamp after the cache maintenance that follows the record, so the +/// stamp, not the record, proves that the command's writes are complete and that its mappings +/// may be released. While other commands of the same submit ioctl are still running, the stamp +/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms +/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as +/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail +/// its VM. +pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; + +// A command that waits out the fallback must still retire well inside the scheduler timeout. +const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); + +/// Whether a completion record latched at `observed_at` (host time, ns) has waited out +/// [`STAMP_FALLBACK_NS`] by `now`. +pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { + observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) +} static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -/// Counts a command that retired on its record without its completion stamp; logs the first and -/// every 64th. +/// Counts a command that retired on its record because its completion stamp never arrived; +/// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, + STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -90,7 +106,12 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// must be consumed. +#[expect( + clippy::too_many_arguments, + reason = "Retirement validates the ticket against independent firmware observations." +)] pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -208,7 +229,8 @@ impl Batch { if descriptor == 0 || record.read(7)? == 0 { return Ok(()); } - let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -240,3 +262,143 @@ impl Batch { Ok(request.observed.take()) } } + +#[kunit_tests(asahi_g17_compute_stamp_wait)] +mod tests { + use super::*; + + const KICK: u64 = 0x12_3456_7801; + + fn ticket() -> Ticket { + Ticket { + owner: 1, + submission: 1, + context_id: 1, + qid: 28, + descriptor: 0x1000, + work_node: 0x2000, + kick: KICK, + ordinal: 1, + item_producer: 4, + item_count: 64, + epoch: 1, + } + } + + fn consumed() -> Cursors { + Cursors { + producer: 4, + consumer: 4, + count: 64, + } + } + + fn own(kick: u64) -> u32 { + (kick as u32).wrapping_shl(8) + } + + #[test] + fn stamp_covers_its_kick_and_earlier_ones() { + assert!(stamp_covers(own(KICK), KICK)); + assert!(stamp_covers(own(KICK + 1), KICK)); + assert!(!stamp_covers(own(KICK - 1), KICK)); + assert!(!stamp_covers(own(KICK) | 1, KICK)); + assert!(stamp_covers(0, 0xff_ffff)); + } + + #[test] + fn a_late_stamp_is_waited_for() { + // Under games on J700 the stamp lands up to about 4 ms after the record while the + // rest of the command's ioctl runs: the command keeps its mappings until then. + let record = Some(Observation { timestamps: [10, 20] }); + let at = 1_000_000; + assert!(!record_aged(Some(at), at + 4_000_000)); + assert!(!record_aged(Some(at), at + 10_000_000)); + let waiting = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + 10_000_000), + ); + assert_eq!(waiting, Ok(None)); + let stamped = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(stamped, Ok(Some([10, 20]))); + } + + #[test] + fn a_later_kick_stamp_retires_it() { + let record = Some(Observation { timestamps: [10, 20] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK + 3), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(result, Ok(Some([10, 20]))); + } + + #[test] + fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { + let record = Some(Observation { timestamps: [0x1111, 0x2222] }); + let at = 1_000_000; + assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); + assert!(!record_aged(None, u64::MAX)); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + STAMP_FALLBACK_NS), + ); + // The record's own timestamps are published, never synthesized ones. + assert_eq!(result, Ok(Some([0x1111, 0x2222]))); + } + + #[test] + fn an_aged_record_still_waits_for_consumed_items() { + let record = Some(Observation { timestamps: [10, 20] }); + let busy = Cursors { + producer: 4, + consumer: 2, + count: 64, + }; + let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); + assert_eq!(result, Ok(None)); + } + + #[test] + fn a_zero_duration_record_fails() { + let record = Some(Observation { timestamps: [5, 5] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::Allowed, + 0, + None, + true, + ); + assert_eq!(result, Err(ENODATA)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8..6a9b15789339f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -56,10 +56,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +77,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,6 +92,7 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], @@ -94,6 +102,8 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } +type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); + impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -107,6 +117,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -119,6 +130,7 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), @@ -128,7 +140,8 @@ impl Queue { } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Admission is checked even when this queue already owns a scheduler. + // Validate every request, including an existing scheduler. Physical + // admission happens after its DRM dependencies, in backend publication. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -156,6 +169,10 @@ impl Queue { }); } + #[expect( + clippy::too_many_arguments, + reason = "Enqueue keeps validated command, ordering, and destination owners explicit." + )] fn enqueue( &mut self, id: u64, @@ -270,6 +287,13 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { + // An input that already signalled without error orders nothing and has no status + // left to consume (like a completed frontier producer above). Keeping it would + // send the render through the scheduler thread instead of inline publication: + // most compositor and client renders carry such already-signalled buffer fences. + if job::fence_status(fence) > 0 { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } @@ -307,7 +331,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { + ) -> Result { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -320,6 +344,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { @@ -384,7 +409,10 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes, objects)?; + let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + } let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e40..fb6c77fbdf04f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -87,6 +87,8 @@ pub(crate) trait Host { fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; + /// The next publication number, shared with compute (timeout ordering). + fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -120,6 +122,8 @@ struct Prepared { mcache: Option, } struct Active { + /// Publication number, shared with compute (crate::g17::timeout). + published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -152,8 +156,11 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -168,14 +175,28 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// A retired, never-executed group awaiting republication on this pair. - replay: Option>, + /// Retired, never-executed groups awaiting republication on this pair, in + /// their original order. Nothing else is prepared on the pair before them. + replay: Tracker>, + /// When the oldest parked group was parked or the last one republished. + replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "The packet constructor spells out the paired render firmware inputs." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -186,7 +207,9 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -211,10 +234,11 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -226,9 +250,10 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -239,8 +264,11 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, + deflake, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -251,8 +279,11 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -280,20 +311,25 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.is_some() + self.active.len() != 0 || self.replay.len() != 0 } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_stalled(&self, now: u64) -> bool { + self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.clone() + self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay = None; + self.replay.pop(); + // Progress: the expiry restarts for the next parked group, if any. + self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -315,6 +351,7 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() + || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { @@ -334,14 +371,17 @@ impl Pair { let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -371,6 +411,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -402,11 +443,18 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -503,9 +551,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -517,6 +573,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -619,6 +682,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -628,6 +692,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -649,6 +714,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -658,6 +724,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -694,8 +761,10 @@ impl Pair { let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; + let published = host.next_publication()?; self.active .push(Active { + published, qos_completed: false, flist_completed: false, outer_published: false, @@ -730,9 +799,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -747,7 +813,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -781,6 +852,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -924,6 +1000,22 @@ impl Pair { .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) }) } + /// The oldest published group still running, for cross-VM timeout attribution + /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains + /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. + pub(crate) fn timeout_head(&self) -> Option { + self.active + .iter() + .filter(|active| active.outer_published) + .find(|active| active.drained_result.is_none()) + .map(|active| crate::g17::timeout::Head { + vm: active.packet.context.vm().id(), + published: active.published, + finished: false, + failed: self.quarantined, + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -934,7 +1026,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -955,7 +1047,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -1084,161 +1176,218 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - loop { - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { return Err(EIO); } - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { - return Err(EIO); - } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); } - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(true) } - /// The front group was retired and unlinked by firmware without executing - /// (job list empty, stamps cover it, no recovery, fragment end never - /// written). Nothing of this group reached its attachments, so the same - /// command is published again instead of failing the owner's VM. The - /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, - /// and only for a lone group whose completion still accepts a replay. + /// Every retained group was retired and unlinked by firmware without + /// executing (job list empty, stamps cover it, no recovery, fragment end + /// never written): after another VM's recovery the firmware drops an + /// innocent pair's queued groups this way (seen on 25G83 and 26A428). + /// Nothing of them reached their attachments, so the same commands are + /// published again, in order, instead of failing their owners. Only the + /// whole set is replayed; a replayed prefix would run after a live + /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS + /// apart (worker polling); the render timeout, long after publication, + /// takes it once as its retirement does. A single sample can still + /// straddle a firmware update of the fragment end record; the cost is + /// that a group which did execute runs a second time (blending twice). fn replay_abandoned( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, ) -> Result { use retirement::Memory; - if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } - let front = self.active.front().ok_or(EIO)?; - if !front.outer_published || front.failed { - return Ok(false); + use crate::g17::replay::{self, Decision, Group}; + let qids = self.qids(); + let memory = &self.memory; + let mut failure = None; + let groups = self.active.iter().map(|active| { + let innocent = + active.outer_published && !active.failed && !active.packet.completion.guilty(); + let blamed = active.packet.completion.blamed(); + let exhausted = + active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; + // decide() stops at the first group that is not abandoned, an error included. + let abandoned = innocent + && !blamed + && (|| -> Result { + let mut observer = Observer { + memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof; a later poll or the timeout looks again. + Ok(matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + )) + })() + .unwrap_or_else(|error: Error| { + failure = Some(error); + false + }); + Group { + innocent, + blamed, + abandoned, + exhausted, + } + }); + let decision = replay::decide(groups); + if let Some(error) = failure { + return Err(error); } - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(front.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - let abandoned = - retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if !abandoned { - front.abandoned_since = None; - return Ok(false); + match decision { + Decision::Refuse => return Ok(false), + Decision::Wait => { + front.abandoned_since = None; + return Ok(false); + } + Decision::Replay => {} } - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } } - if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_info!( - "render slot {} retired without execution (pass {:?}); replaying\n", + // decide() lets only the front be exhausted. An exhausted front fails, and its + // successors, which consume its output, fail with it rather than run without it. + let mut failing = self.active.front().is_some_and(|front| { + front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS + }); + if failing { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", + self.slot, + self.active.len() + ); + } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replaying\n", self.slot, - snapshot.pass + self.active.len() ); } let qos = self.qos_owners(); let buffer = self.buffer_id(); let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.flist_completed = true; - front.qos_completed = true; - } - let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - if !packet.completion.begin_replay() { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); + // Every group parked below shares this expiry (an early return keeps it). + self.replay_since = Some(now); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + failing = failing || !packet.completion.begin_replay(); + if failing { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + continue; } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - return Ok(true); + packet.reset_publication_for_replay(); + self.replay.push(packet).map_err(|_| EIO)?; } - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.replay = Some(packet); Ok(true) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). ECANCELED leaves the VM usable; any other error - /// is the ordinary device-loss outcome the replay tried to avoid. + /// publication refusal). Its successors cannot run without it and fail with + /// it. ECANCELED leaves the VM usable; any other error is the ordinary + /// device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(packet) = self.replay.take() else { - return Ok(()); - }; - defer(Deferred::Retired(packet.completion.clone(), Err(error))) + self.replay_since = None; + while let Some(packet) = self.replay.pop() { + defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; + } + Ok(()) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1357,22 +1506,51 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - if !self.quarantined && self.active.len() == 1 { - let active = self.active.front().ok_or(EIO)?; - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if active.outer_published - && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? - { + // Groups the firmware dropped unexecuted are published again, with their successors. + if self.replay_abandoned(host, defer, false)? { + return Ok(true); + } + // Fallback when a live or exhausted group prevents an ordered replay: a spared front group + // the firmware retired without executing during another VM's recovery is retired on its + // own even with successors queued, so the pair is not quarantined into a sticky EIO. It + // fails with ECANCELED, which spared work never reports as device loss; consecutive + // abandoned spared successors are retired the same way. + let front_spared = self.active.front().is_some_and(|a| { + Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() + }); + if !self.quarantined && (self.active.len() == 1 || front_spared) { + let mut retired = false; + loop { + let Some(active) = self.active.front() else { + break; + }; + let spared = active.packet.completion.spared(); + if retired && !spared { + break; + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof, as in replay_abandoned(): the group stays retained + // (quarantined with the pair if nothing was retired) instead of the error + // stopping the runtime for every client. + if !active.outer_published + || !matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + break; + } let qos = self.qos_owners(); let buffer = self.buffer_id(); + let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1384,19 +1562,30 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !active.packet.completion.spared() { + if !active.failed && !spared { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } + pr_warn!( + "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", + active.ticket.ordinal, + left + ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(EIO), + Err(if spared { ECANCELED } else { EIO }), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + retired = true; + if !spared { + break; + } + } + if retired { return Ok(true); } } @@ -1424,7 +1613,8 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - if let Some(packet) = self.replay.take() { + self.replay_since = None; + while let Some(packet) = self.replay.pop() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -1578,6 +1768,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -1621,6 +1823,10 @@ impl Pair { self.owner = None; } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Binding keeps the render owners and first packet explicit." + )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -1654,7 +1860,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -1670,6 +1876,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -1736,6 +1943,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d13..80ee0b871cc82 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -94,6 +97,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +106,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers)?; Ok(()) })?; } @@ -166,6 +172,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +195,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +217,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -310,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -365,6 +386,10 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Render memory construction keeps each allocation and firmware input explicit." + )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -376,10 +401,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -505,6 +531,9 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { @@ -598,6 +627,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e07..209ce0b8176ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,12 +25,19 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -48,11 +55,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -76,12 +92,15 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -96,11 +115,22 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Preparation forwards the render binding inputs without changing their ownership." + )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -125,7 +155,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -141,6 +171,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -190,6 +221,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, @@ -202,7 +234,9 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -213,8 +247,11 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 0fc9c1d9a15dc..9f4cb05b6bede 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,21 +73,28 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, @@ -116,7 +124,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = (&self.config as *const QueueConfig).cast::(); + let source = core::ptr::from_ref(&self.config).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index d0a2d566bdc8d..24aeed6bcd0ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -203,8 +203,12 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - let observation = observe(ticket, snapshot)?; - complete_poll(ticket, memory, snapshot, observation) + match observe(ticket, snapshot) { + Ok(observation) => complete_poll(ticket, memory, snapshot, observation), + // The borrowed snapshot can straddle a firmware cursor update like any other sample: + // take the ordinary poll's fresh snapshot and reread instead of failing the client. + Err(_) => poll(ticket, memory), + } } fn complete_poll( diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec6..da90488bfedec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0)?; } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ba368341d84d2..89f8d33bdb80c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,11 +9,12 @@ mod attribution; mod slot; +mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; @@ -99,6 +100,8 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, + timeouts: safety::Storm, + restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -112,8 +115,12 @@ impl State { Self { accepted: 0, pending: None, + timeouts: safety::Storm::new(), + restarts: safety::Storm::new(), } } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -220,9 +227,17 @@ pub(super) fn service( } host.recovery_mut().pending = None; - let cancelled = classify(host, entries, request.blamed, &mut sources); + let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { - host.settle_blamed_render(qid, EIO); + pr_warn!( + "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", + sources.reason, + qid, + if blame { "failing its oldest render" } else { "attribution cleared it; spared" } + ); + if blame { + host.settle_blamed_render(qid, EIO); + } } if cancelled != 0 { host.settle_cancelled_renders(cancelled); @@ -235,7 +250,10 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { +/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed +/// render that had not started while others had (or with no culprit at all) is innocent, and its +/// work is spared like that of every other innocent VM instead of failing with EIO. +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); @@ -245,8 +263,9 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; + let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return 0; + return (0, blame); } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -282,7 +301,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return 0; + return (0, blame); } let spared = queues.classify_work(&|status| { guilty[..count] @@ -295,5 +314,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }) .unwrap_or(Class::Spared) }); - cancelled & !(guilty_qids & !collateral) & spared + (cancelled & !(guilty_qids & !collateral) & spared, blame) } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs new file mode 100644 index 0000000000000..c30e45e22579a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fail closed on repeated watchdog/restart traffic, even across client loss. + +const WINDOW_NS: u64 = 30_000_000_000; +const LIMIT: u8 = 8; + +pub(super) struct Storm { + start: Option, + count: u8, + stopped: bool, +} +impl Storm { + pub(super) const fn new() -> Self { + Self { start: None, count: 0, stopped: false } + } + pub(super) fn note(&mut self, now: u64) -> bool { + if self.stopped { return true; } + if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { + self.start = Some(now); + self.count = 0; + } + self.count += 1; + self.stopped = self.count >= LIMIT; + self.stopped + } +} + +#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] +mod tests { + use super::*; + use crate::g17::{preparation::Gate, recovery::State}; + use kernel::prelude::*; + + #[test] + fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { + let gate = Gate::new().unwrap(); + let lease = gate.try_enter().unwrap(); + let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); + gate.close(); + for n in 0..64 { + let stopped = { + let mut state = state.lock(); + state.request(0, -1).unwrap(); + assert!(state.pending()); + let stopped = state.timeout_limit(n * 2_000_000_000); + if stopped { gate.fail(); } // production state -> gate order + stopped + }; + assert_eq!(stopped, n >= u64::from(LIMIT - 1)); + } + // Recovery/stop must release state before joining the retained builder. + assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), + Err(error) if error == ETIMEDOUT)); + assert!(!gate.reopen()); // failure never gives back an owned reservation + drop(lease); + gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); + assert!(!gate.reopen()); + } + #[test] + fn successful_handshakes_do_not_reset_a_restart_storm() { + let mut state = State::new(); + for n in 0..64 { + assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); + } + assert!(state.restart_limit(u64::MAX)); + } + #[test] + fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } + for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } + assert!(storm.note(64 * WINDOW_NS)); + for _ in 0..1024 { assert!(storm.note(u64::MAX)); } + } + #[test] + fn replay_dependency_checks_never_wait_for_event_service() { + use crate::g17::dependency::replay_status; + assert_eq!(replay_status(0), Err(EBUSY)); + assert_eq!(replay_status(1), Ok(())); + assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 307534708e922..bd9d4c96be14f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as _, + bindings::MEMREMAP_WB as usize, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs new file mode 100644 index 0000000000000..94daa5ec0dfeb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Whether a pair's retained render groups, retired by firmware without executing, are +//! published again. Only the whole set is replayed, so no replayed group runs after a live +//! successor, and work a recovery blamed is never run again. + +/// One retained group, front first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + /// Published, not failed and not classified guilty by a recovery. + pub(crate) innocent: bool, + /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): + /// it may have caused the fault. + pub(crate) blamed: bool, + /// Carries the strict abandonment witness. + pub(crate) abandoned: bool, + /// Already republished the maximum number of times. + pub(crate) exhausted: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Replay, + /// A group may still execute: look again later. + Wait, + /// Never replay this set; the render timeout settles it. + Refuse, +} + +/// Groups are read in order and the first one that decides stops the walk, so a caller can +/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its +/// successors, which consume its output, with it. An exhausted successor would break the order. +pub(crate) fn decide(groups: impl Iterator) -> Decision { + let mut front_exhausted = None; + let mut successor_exhausted = false; + for group in groups { + if !group.innocent || group.blamed { + return Decision::Refuse; + } + if !group.abandoned { + return Decision::Wait; + } + match front_exhausted { + None => front_exhausted = Some(group.exhausted), + Some(_) => successor_exhausted |= group.exhausted, + } + } + match front_exhausted { + Some(true) => Decision::Replay, + Some(false) if !successor_exhausted => Decision::Replay, + _ => Decision::Refuse, + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] +mod tests { + use super::*; + fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { + Group { innocent, blamed: false, abandoned, exhausted } + } + const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; + const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + #[test] + fn every_group_abandoned_and_innocent_is_replayed() { + assert_eq!(decide([OK].into_iter()), Decision::Replay); + assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); + } + #[test] + fn nothing_retained_is_never_replayed() { + assert_eq!(decide([].into_iter()), Decision::Refuse); + } + #[test] + fn guilty_work_is_never_replayed() { + assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); + } + #[test] + fn blamed_work_is_never_replayed_even_when_spared() { + // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. + assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); + assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); + // Also once the replay limit is reached, where an innocent front alone is replayed. + let exhausted = Group { exhausted: true, ..BLAMED }; + assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); + } + #[test] + fn a_live_group_waits_and_stops_the_walk() { + assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); + // Nothing after the live group is looked at, guilty or not. + assert_eq!( + decide([group(true, false, false), group(false, true, false)].into_iter()), + Decision::Wait + ); + let mut seen = 0; + let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); + assert_eq!(decide(groups), Decision::Wait); + assert_eq!(seen, 1); + } + #[test] + fn an_exhausted_successor_refuses_an_exhausted_front_replays() { + assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); + assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); + assert_eq!( + decide([group(true, true, true), group(true, true, true)].into_iter()), + Decision::Replay + ); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9..6f4a5355d6eb3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,6 +25,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -70,20 +71,20 @@ impl Registry { /// registry on the probe stack. No additional allocation is required. pub(super) fn new( clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, - render: render::RenderState::new(clusters, descriptor_flags)?, + admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -121,7 +122,8 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -129,7 +131,14 @@ impl Registry { Rebind::Fresh }); } + // Physical channels are a device budget, not a permanent entitlement + // of each open logical queue. Only exact idle retirement can yield one. + for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -137,6 +146,13 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -167,6 +183,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -174,6 +191,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -267,6 +288,7 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -275,6 +297,17 @@ impl Registry { Ok(freed) } + /// Builders and released queues waiting for their exact retirement or pool + /// witness temporarily occupy identities. A failed admission must let those + /// transactions finish rather than fail a healthy successor immediately. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -292,11 +325,13 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or(ENOSPC)?; + .ok_or_else(|| self.compute_admission_error( + crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -373,6 +408,32 @@ pub(super) enum Rebind { Fresh, } + +#[kunit_tests(asahi_g17_uninstalled_cleanup)] +mod uninstalled_cleanup_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + #[test] + fn empty_reservation_prevents_reuse_until_explicit_cancellation() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let first = registry.reserve_compute(1).unwrap(); + assert!(registry.reserved(first).is_ok()); + assert!(registry.reserve_compute(1).is_err()); + let second = registry.reserve_compute(2).unwrap(); + assert_ne!(first.id.qid(), second.id.qid()); + registry.cancel_compute(first).unwrap(); + assert!(registry.reserved(first).is_err()); + let next = registry.reserve_compute(3).unwrap(); + assert_eq!(first.id.qid(), next.id.qid()); + assert!(first.id != next.id); + } +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -380,6 +441,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + closed: core::sync::atomic::AtomicBool, } impl Backend { @@ -389,24 +451,54 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -429,6 +521,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -438,13 +531,15 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, admission: &Admission<'_>) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -463,7 +558,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } }; let alloc = super::object::Allocator { @@ -474,7 +571,9 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -507,10 +606,12 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, + admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -522,14 +623,18 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -551,8 +656,23 @@ impl Backend { result } - fn grow_compute(&self, required: usize) -> Result { + /// The caller keeps its preparation lease through this two-phase rollback. + /// An empty registry entry prevents reusing the QID/free-list identity while + /// its old aliases still occupy canonical or client addresses. Recovery joins + /// this lease before removing the session, even if its epoch was invalidated. + fn finish_uninstalled_compute( + &self, retired: Option<(ComputeReservation, KBox)>, + ) -> Result { + let Some((reservation, queue)) = retired else { return Ok(()); }; + drop(queue); + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.queues.cancel_compute(reservation) + } + + fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { use kernel::time::{delay::fsleep, Delta}; + admission.check()?; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -560,45 +680,79 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); - loop { + let mut growth = None; + let result = (|| loop { let result = (|| { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } - if queue.begin_growth(&firmware.init, required)? { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + if growth.is_none() { + let notify = queue.begin_growth(&firmware.init, required)?; + // Capture physical identity before any following fallible + // operation. Close can remove the owner-keyed lookup. + growth = queue.growth_token(); + if notify { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + Err(error) if error == EAGAIN && lease.is_current() => { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } Err(error) => return Err(error), } + })(); + if result.is_err() { + if let Some(token) = growth { + let mut retired = None; + let mut state = self.shared.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + for index in 0..firmware.queues.compute.len() { + let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; + let Some(queue) = entry.queue.as_deref_mut() else { continue; }; + if queue.abort_growth(token)? == Some(true) { + // Keep the empty entry and QID reserved until all + // graph aliases have been destroyed outside the lock. + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + break; + } + } + } + drop(state); + // Tentative replacement mappings also belong to this builder. + // Discard them before a fresh QID can acquire a successor. + drop(alias.take()); + drop(pages.take()); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + } } + result } - fn build_compute(&self, required: usize) -> Result { + fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; + self.admission_open()?; loop { - match self.shared_compute() { + match self.shared_compute(admission) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -607,10 +761,12 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -619,12 +775,42 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context, required)? { + self.admission_open()?; + let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { + Err(error) if error == ENOSPC => { + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; + match rebound { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // The event worker owns release observation and + // drops vacated aliases/backing off this mutex. + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } @@ -638,13 +824,32 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - let ring = compute::DescriptorRing::new(&alloc)?; + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + Ok((pages, compute::DescriptorRing::new(&alloc)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => { + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // A competing ready owner may have taken/rearmed it while + // backing was built off-lock. Leave rejected backing local + // so it is dropped after the device guard, not in rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm()); + if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -660,10 +865,12 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } + self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -671,8 +878,15 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + self.compute_scheduler(&alloc, &lease, admission)?; + let queue = compute::Queue::new( + &alloc, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + self.shared.operand_state, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -682,12 +896,14 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - firmware.queues.install_compute(reservation, &mut built) + admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) } }; // The old aliases must disappear before a waiting constructor can @@ -703,9 +919,11 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result != Err(EAGAIN) { - return result; + if result == Err(ENOSPC) { + self.wait_compute_capacity(admission)?; + continue; } + if result != Err(EAGAIN) { return result; } } } } @@ -780,6 +998,9 @@ impl DeferredBatch { drop(release); } } + pub(super) fn vacated_room(&self) -> bool { + self.vacated.len() < self.vacated.capacity() + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -847,7 +1068,7 @@ struct ComputeHost<'a> { } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_compute_publication() + self.qids.next_publication() } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -918,6 +1139,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, @@ -1053,12 +1283,14 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; for _ in 0..SPARED_WAIT_MS { + admission.check()?; let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1070,7 +1302,10 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + match self.build_compute(required, admission) { + Err(error) if error == EAGAIN => continue, + result => return result, + } } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. @@ -1078,6 +1313,7 @@ impl Backend { } // No new command has been accepted. Keep the spared queue and its // retained work intact until a later admission observes its witness. + admission.check()?; Err(EBUSY) } @@ -1089,23 +1325,48 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; - let mut outer_wait = None; + let mut state_delay = 20i64; + let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let check = || { + if packet.cancel_requested() { return Err(ECANCELED); } + self.admission_open()?; + if packet.completion.status().get() != 0 { return Err(EIO); } + Ok(()) + }; + let admission = Admission::new(deadline, &check); loop { - if packet.completion.status().get() != 0 { - return Err(EIO); + admission.check()?; + // All DRM dependencies were satisfied before this scheduler callback. + // Recheck on every retry: pressure may yield an idle binding between + // admission and publication without changing this logical queue. + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(&admission)?; + // Work states free as this context's earlier commands retire. + // Wait for one under the packet's admission deadline, as render + // publication does, instead of failing the packet. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + admission.check()?; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - if packet.cancel_requested() { - return Err(ECANCELED); - } + admission.check()?; if self .shared .crashed @@ -1126,7 +1387,13 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if firmware.queues.compute(self.owner).is_none() { + drop(state); + drop(lease); + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), @@ -1149,11 +1416,6 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. drop(state); @@ -1199,6 +1461,40 @@ impl super::Firmware { } impl super::Shared { + /// Charge only real watchdog expirations, after the per-job allowance. A + /// changing client or recovery generation cannot mint a fresh storm budget. + pub(super) fn watchdog_timeout(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { return true; } + let stop = { + let mut state = self.state.lock(); + state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) + }; + if stop { self.lose_device_quietly(); } + stop + } + + /// Workload-triggered stop: halt producers first, then join only for a + /// bounded interval. A stalled borrower leaves stopped firmware and all its + /// mappings owned in state. Only a later joined stop may detach/free them. + pub(super) fn stop_runtime(&self) { + let Some(_stopping) = self.stop.try_lock() else { return; }; + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } + firmware.primary.stop(); + } + } + if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { + dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); + return; + } + let firmware = self.state.lock().take(); + if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } + } + /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -1211,7 +1507,9 @@ impl super::Shared { let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() @@ -1259,12 +1557,23 @@ impl super::Firmware { } } self.queues.observe_pool_releases(&self.init, deferred)?; + if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_stalled(now)) { + return Err(ETIMEDOUT); + } if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -1276,3 +1585,163 @@ impl super::Firmware { } } + +impl Registry { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.qids.write_capacity(f)?; + self.admission.write_capacity(f)?; + self.buffers.write_capacity(f)?; + writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; + writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; + self.render.write_capacity(f)?; + writeln!(f, "compute_slots used={} total={} reusable={} building={}", + self.compute.iter().flatten().count(), self.compute.len(), + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), + self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; + writeln!(f, "compute_channels used={} total={}", + self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.write_capacity(f)?; + } + Ok(()) + } +} + +impl Backend { + pub(super) fn wait_capacity(&self, started: u64) -> Result { + let check = || self.admission_open(); + let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); + self.wait_compute_capacity(&admission) + } + + fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + admission.check()?; + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + // A short reclaim transaction obeys the same session/epoch gate as + // publication. Never keep this lease while sleeping for capacity. + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { + return Ok(()); + } + for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } + // Consumption, not publication or an elapsed timeout, frees backing. + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + if !firmware.render_control_backpressured() { + let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; + if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } + } + Ok(()) + })(); + deferred.finish(); + self.shared.queue_events(); + match result { + Err(error) if error == EAGAIN => {} + result => result?, + } + admission.check()?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } +} + +#[kunit_tests(asahi_g17_shared_capacity)] +mod shared_capacity_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + + #[test] + fn all_logical_render_owners_fit_beside_shared_compute() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + // Physical compute channels are device-owned and bounded independently + // of how many logical clients have queued unsignalled work. + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); + // Every accepted logical render owner now has its physical pair; no + // published identity was cancelled or retyped to achieve the fit. + } + + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init( + Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), + GFP_KERNEL, + ) + .unwrap(); + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + + #[test] + fn four_clients_have_capacity_before_any_owner_closes() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); + for client in 0..4u64 { + for queue in 0..9u64 { + let owner = 1 + client * 9 + queue; + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + } + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert_eq!(leases.len(), 36); + } +} + +impl Backend { + /// Logical close is independent of whether an off-lock builder has already + /// installed a physical graph. No constructor can resurrect a closed owner. + fn admission_open(&self) -> Result { + if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } + if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !self.context.is_current() { return Err(EFAULT); } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f..7d690470cd7b5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -18,8 +18,39 @@ impl Backend { } fn stop_after_error(&self) { + // Scheduler/fence callbacks never synchronously join builders or RTKit + // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - self.shared.stop_queues(); + } + + fn rollback_unpublished_compute(&self) -> Result { + // No outer preparation lease survives compute_packet's return. Hold a + // fresh epoch through alias destruction so recovery can join this work. + loop { + let lease = self.shared.preparations.enter(false)?; + let mut retired = None; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { drop(state); drop(lease); continue; } + let mut binding = None; + if let Some(index) = firmware.queues.compute.iter().position(|entry| { + entry.as_ref().and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.owner() == Some(self.owner)) + }) { + let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; + let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? + .rollback_unpublished(self.owner, &self.context)?; + binding = replaced; + if cancel { + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + } + } + drop(state); + drop(binding); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + return Ok(()); + } } /// Complete all failure fanout before signalling this packet's unpublished @@ -36,6 +67,11 @@ impl Backend { } deferred.finish(); if !packet.is_published() { + if packet.engine() == Engine::Compute { + if self.rollback_unpublished_compute().is_err() { + self.stop_after_error(); + } + } packet.completion.fail_unpublished(error); } Err(error) @@ -46,6 +82,9 @@ impl Backend { } fn release_queue(&self) -> Result { + // A builder may own only an off-lock reservation when close arrives. + // Record intent before scanning queues; every later install/rearm checks it. + self.closed.store(true, Ordering::Release); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -75,7 +114,11 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - Backend::ensure_compute(self, scs_pages) + // A reverse timeline chain must be submitted before the host signal. + // Validate here, but do not consume physical capacity behind unsignalled + // DRM dependencies. compute_packet admits it when the job can run. + crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + .map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -147,7 +190,17 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); + if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -177,10 +230,47 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + use crate::g17::timeout::{self, Decision}; + // As for compute, a final retirement pass precedes the attribution, and all completion + // callbacks run after releasing the device mutex. Without a collector the attribution + // goes ahead without that pass, as it did before it existed. + let mut deferred = DeferredBatch::worker().ok(); + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(false); + }; + if let Some(deferred) = deferred.as_mut() { + firmware.scan_renders(deferred)?; + } + // Borrowed-graph and queued-retirement deferrals share the renewal bound. + if timeout::transaction_grace(renewals) + && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) + { + return Ok(true); + } + let verdict = firmware.queues.render_timeout_decision(packet, renewals); + let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); + // As compute does: work still blocked behind another VM's older unfinished work + // after the whole grace is innocent, so its failure must stay non-sticky. The pair's + // quarantine extends the class to the pair's other groups. + if decision == Decision::Spare { + packet.completion.classify(crate::g17::recovery::Class::Spared); + } + if let Some(verdict) = verdict { + if let Some(blocker) = verdict.reported_blocker(renewals) { + dev_warn!(self.shared.dev.as_ref(), + "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, verdict.own.vm, verdict.own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); + } + } + Ok(decision == Decision::Defer) + })(); + if let Some(deferred) = deferred.as_mut() { + deferred.finish(); + } + matches!(result, Ok(true)) } fn try_early(&self, packet: &Arc) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34ae..c407d9576e9c4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -27,19 +27,22 @@ impl Backend { let Some(own) = queue.timeout_head() else { return Ok(Decision::Own); }; - let others = firmware.queues.compute.iter().flatten() + let computes = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let decision = timeout::classify(own, renewals, others); + let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); + let verdict = timeout::attribute(own, renewals, others); + let decision = verdict.decision; if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + if let Some(blocker) = verdict.reported_blocker(renewals) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", - self.owner, own.published, renewals, decision); + "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, own.vm, own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); } Ok(decision) })(); @@ -50,7 +53,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0d4a29e779b4a..d71f410c76fb4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -83,7 +83,7 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; + packet.check_replay_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a33..4fd6e56741986 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,14 +24,17 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); @@ -263,6 +266,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -300,7 +304,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); + self.stop_runtime(); return; } let mut deferred = { @@ -324,7 +328,7 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); } } @@ -345,7 +349,7 @@ impl crate::g17::Shared { }; deferred.finish(); if recover { - self.preparations.wait_drained(); + self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -419,7 +423,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } @@ -444,7 +448,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf2045526..59db76b9f109a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -24,6 +24,7 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -46,7 +47,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -55,6 +62,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -65,7 +78,11 @@ pub(super) struct RenderState { } impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + pub(super) fn new( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + ) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -76,6 +93,7 @@ impl RenderState { Ok(Self { config: Config { clusters, + deflake, descriptor_flags, }, entries, @@ -124,6 +142,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -157,6 +183,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -176,6 +203,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -198,7 +226,114 @@ impl RenderState { } impl Registry { - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } @@ -228,7 +363,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -262,12 +399,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -400,6 +533,11 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -419,6 +557,9 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } + fn next_publication(&mut self) -> Result { + self.firmware.queues.qids.next_publication() + } fn publish_outer_pair( &mut self, priority: u8, @@ -553,6 +694,44 @@ impl crate::g17::Firmware { Ok(()) } + /// A final retirement pass before a render watchdog attributes its timeout, as compute runs + /// scan_compute() first: a head that finished but whose completion the event worker has not + /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred + /// timeout or error, stay with the worker, which observes and settles them in its order. + pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()) + && entry.preparing.is_none() + && !entry.growing + && !entry.pending_timeout + && entry.pending_error.is_none() + }) + { + continue; + } + self.retire_render_slot(slot, None, deferred)?; + self.queues.render.collect_retired(slot)?; + self.settle_render_terminal(slot, deferred)?; + } + Ok(()) + } + pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -763,9 +942,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } @@ -816,4 +1005,103 @@ impl Registry { entry.pending_timeout = true; true } + + /// The oldest published group of every render pair, for cross-VM timeout attribution + /// (crate::g17::timeout). + pub(in crate::g17) fn render_timeout_heads( + &self, + ) -> impl Iterator + '_ { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .filter_map(|pair| pair.timeout_head()) + } + + /// A render watchdog blocked behind another VM's older unfinished work gets the same + /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU + /// stalls on one client, the others' renders then outlive the culprit's teardown instead + /// of all timing out together. + pub(in crate::g17) fn render_timeout_decision( + &self, + packet: &Arc, + renewals: u8, + ) -> Option { + let own = self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.contains_packet(packet)) + .and_then(|pair| pair.timeout_head()); + let computes = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.timeout_head()); + // Without a head of its own the timeout is its own. + own.map(|own| { + crate::g17::timeout::attribute( + own, + renewals, + Iterator::chain(self.render_timeout_heads(), computes), + ) + }) + } +} + +impl RenderState { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + self.pools.write_capacity(f)?; + for (slot, entry) in self.entries.iter().enumerate() { + if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { + context.write_capacity(f, "render", slot)?; + } + } + Ok(()) + } +} + +impl Registry { + /// Give idle private backing to another ready client under capacity pressure. + /// Keep graph/QID/context ownership; the next use rearms the same pool after + /// its consumed release, exactly as eviction-driven reclaim does. + pub(super) fn reclaim_idle_render_pools( + &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { break; } + let Some(pool) = entry.pool.as_ref() else { continue; }; + // None means this pool never acquired a firmware registration; + // otherwise require its exact final release to have been consumed. + if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) + || self.admission.owner_active(entry.reservation.owner) { continue; } + // A sibling or replacement may be building off-lock before it has + // installed its pool reference. Its logical owner also excludes us. + let mut idle = true; + for other in render.entries.iter().flatten().filter(|other| { + other.reservation.owner == entry.reservation.owner + || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) + }) { + if other.constructing || other.growing || other.preparing.is_some() + || other.pending_error.is_some() || other.pending_timeout { + idle = false; + break; + } + if let Some(pair) = other.pair.as_ref() { + if !pair.idle_for_context(pair.context())? { idle = false; break; } + } + } + if idle { + if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } + } + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55c..e3cb174721aaa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + request.vm_slot == -1 || request.vm_slot as u32 == context.id() }) }) }) @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,7 +271,10 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c8001..4ede20d316c64 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -152,7 +154,7 @@ impl crate::g17::Shared { drop(state); drop(epoch); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); if let Some(mut pair) = pair { pair.stopped(&mut |event| { event.finish(); @@ -163,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index f7f48e853f225..4713eb5b6b152 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -239,3 +239,11 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } + +impl Pools { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "render_pool_leases used={} total={} retained={}", + self.leases.iter().flatten().filter(|lease| !lease.released()).count(), + POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7a..e2034c3176595 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,13 +29,14 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) @@ -44,6 +46,10 @@ impl BuildResources { } } +#[expect( + clippy::large_enum_variant, + reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." +)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -72,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -133,6 +145,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +183,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -238,10 +259,11 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; Pair::new( - &alloc, + alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -252,7 +274,9 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, + resources.config.deflake, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { @@ -277,7 +301,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -316,9 +340,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -360,7 +384,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -397,6 +421,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -429,9 +456,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -463,10 +490,14 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } firmware.queues.teardown.displace( pair.context(), @@ -475,7 +506,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -499,6 +530,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -556,8 +591,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -568,10 +603,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -583,8 +620,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -660,6 +698,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index ce9771778acd4..499e3858fcca1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packet; this pass -//! prepares and publishes the same command again under the device mutex. +//! recovery, fragment end never written) and parks the packets in order; this +//! pass prepares and publishes the same commands again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,79 +27,91 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - let Some((packet, owner, qids, closed, busy)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - )) - }) - else { - continue; - }; - if busy { - continue; + // Publish the parked groups front first; a pause keeps the rest parked. + while self.replay_render_front(slot, deferred)? {} + } + Ok(()) + } + + /// Publishes the oldest parked group of `slot`; returns whether it was published. + fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { + let Some((packet, owner, qids, closed, busy, stalled)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + pair.replay_stalled(crate::g17::now_ns()), + )) + }) + else { + return Ok(false); + }; + if stalled { + return Err(ETIMEDOUT); + } + if busy { + return Ok(false); + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); } - if closed || packet.cancel_requested() || !packet.context.is_current() { + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; + return Ok(false); + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + Ok(true) } - if packet.completion.status().get() != 0 { + Err(error) if error == EBUSY || error == EAGAIN => Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, - }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - continue; - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - } - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - } + Ok(false) } } - Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b93216..24c724cb32470 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -33,7 +37,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as _, + kernel::time::msecs_to_jiffies(2000) as isize, ) }; if completed > 0 { @@ -121,6 +125,14 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { + // Unclassified work counts as innocent, and a blamed group must never be replayed. + // Work an earlier recovery spared is collateral here (classify() blames before its + // collateral check) and stays spared, so its failure stays non-sticky; the blame + // itself is recorded apart from the class, and refuses any replay of it. + packet.completion.note_blamed(); + if !packet.completion.spared() { + packet.completion.classify(crate::g17::recovery::Class::Guilty); + } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c..b77fc13b7ed2f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,29 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -23,6 +45,128 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + sleep: bool, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + let started = super::super::now_ns(); + let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + loop { + self.admission_open()?; + if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } + match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + // Atomics only: the callback runs under the pool mutex. + if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + Ok(()) + }) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + + /// A pair is borrowed, still settling, or waiting for the release of the + /// scheduler it last ran for to be consumed and the pair parked. The event + /// worker observes both, so kick it on every poll; time out only after the + /// same bound as a capacity wait. + fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + + /// A closing owner may still hold its pair until exact retirement. Wait + /// outside both the device mutex and preparation gate so the event worker + /// and recovery can return capacity. No packet is firmware-visible here. + fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + self.wait_capacity(started) + } + fn render_with_lease( &self, packet: &Arc, @@ -30,6 +174,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -37,22 +182,17 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; + let capacity_started = super::super::now_ns(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -65,7 +205,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -76,6 +220,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { + if !sleep { + return Err(FALLBACK); + } if state_polls == 1000 { return Err(ETIMEDOUT); } @@ -88,19 +235,23 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, sleep) { + Err(error) if error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); @@ -109,6 +260,9 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { + if !sleep { + return Err(FALLBACK); + } if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { return Err(ETIMEDOUT); } @@ -116,11 +270,9 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -163,6 +315,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -296,6 +458,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0..eb04f80361955 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01..f8020e8962f84 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b..53518a76e91a4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index e76dab07f56db..d951bd164d6e9 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,11 +6,62 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// A borrowed render graph or queued off-lock retirement is not evidence of +/// progress. Share the finite deferral bound with cross-VM attribution. +pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } + +/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. +/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four +/// extra polls give it ten seconds, still one tenth of the legacy scheduler's +/// fallback. This is an execution allowance, not evidence of GPU progress, and +/// is consumed once per scheduler job, independently of cross-VM/replay grace. +pub(crate) const POLL_MS: u32 = 2000; +const EXECUTION_RENEWALS: u8 = 4; + +/// A background control/replay poll has no scheduler fence to expire it. +/// Its timestamp belongs to the exact transaction and is never renewed by work. +pub(crate) fn background_stalled(since: Option, now: u64) -> bool { + since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) +} + +/// Input comes only from fully validated commands. Render metadata covers the +/// VDM half (vertex/tessellation/geometry) and the fragment half independently. +/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. +pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { + compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build +} + +/// Firmware dependencies can hold an untagged helper behind a long command on +/// the other engine. Classify the complete validated ioctl before enqueueing +/// anything; no state or allowance leaks into a later submission. +pub(crate) fn select_submission(commands: impl Iterator) -> bool { + commands.into_iter().any(|enabled| enabled) +} + +/// Only the explicit AS-build bit is newly accepted; reserved and unrelated +/// flags remain invalid on G17P. Return None before any command is published. +pub(crate) fn as_build(flags: u32) -> Option { + const AS_BUILD: u32 = 1 << 2; + if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } +} + +pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= EXECUTION_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - pub(crate) vm: usize, + /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. + pub(crate) vm: u64, pub(crate) published: u64, pub(crate) finished: bool, + /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains + /// it. No later watchdog of its own will settle it. + pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -20,23 +71,156 @@ pub(crate) enum Decision { Spare, } -pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - if own.finished || !others.into_iter().any(|other| { - other.vm != own.vm && !other.finished && other.published < own.published - }) { - Decision::Own - } else if renewals < MAX_RENEWALS { - Decision::Defer - } else { - Decision::Spare +/// Another VM's older unfinished work that can block `own`: live work before work whose queue +/// already failed, the oldest of each first. +pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { + others + .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) + .min_by_key(|other| (other.failed, other.published)) +} + +/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that +/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind +/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only +/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame +/// published before the culprit, which timed out first, would spare the culprit, and a pair that +/// never drains would spare every later timeout. +pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { + match blocker { + _ if own.finished => Decision::Own, + None => Decision::Own, + Some(_) if renewals < MAX_RENEWALS => Decision::Defer, + Some(blocker) if blocker.failed => Decision::Own, + Some(_) => Decision::Spare, + } +} + +/// A watchdog's decision with the heads it rests on, for its log line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Verdict { + pub(crate) decision: Decision, + pub(crate) own: Head, + pub(crate) blocker: Option, +} + +impl Verdict { + /// The blocker a watchdog line names: at the first deferral and once the grace has run out. + pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { + self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) + } +} + +pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { + let blocker = blocker(own, others); + Verdict { + decision: decide(own, renewals, blocker), + own, + blocker, } } -#[cfg(test)] +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; - fn head(vm: usize, published: u64, finished: bool) -> Head { - Head { vm, published, finished } + #[test] + fn selects_compute_both_render_halves_and_builds() { + assert!(!select(false, [false; 2], false)); + assert!(select(true, [false; 2], false)); + assert!(select(false, [true, false], false)); + assert!(select(false, [false, true], false)); + assert!(select(false, [true; 2], false)); + assert!(select(false, [false; 2], true)); + } + #[test] + fn helpers_share_only_the_current_submission_allowance() { + assert!(select_submission([false, true, false].into_iter())); + assert!(select_submission([true, false].into_iter())); + assert!(select_submission([false, true].into_iter())); + assert!(!select_submission([false, false].into_iter())); + assert!(!select_submission([].into_iter())); + let mut helper = 0; + for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } + assert!(!renew_execution(true, true, &mut helper)); + assert!(!renew_execution(true, false, &mut 0)); + } + #[test] + fn build_hint_rejects_reserved_and_unknown_flags() { + assert_eq!(as_build(0), Some(false)); + assert_eq!(as_build(1 << 2), Some(true)); + for bit in 0..32 { + if bit == 2 { continue; } + assert_eq!(as_build(1 << bit), None); + assert_eq!(as_build((1 << bit) | (1 << 2)), None); + } + assert_eq!(as_build(u32::MAX), None); + } + #[test] + fn borrowed_render_and_queued_retirement_cannot_defer_forever() { + for count in 0..=u8::MAX { + assert_eq!(transaction_grace(count), count < MAX_RENEWALS); + } + } + #[test] + fn background_control_and_replay_polls_have_an_absolute_deadline() { + let since = 42; + assert!(!background_stalled(None, u64::MAX)); + assert!(!background_stalled(Some(since), since - 1)); + assert!(!background_stalled(Some(since), since + 9_999_999_999)); + for _ in 0..1024 { + assert!(background_stalled(Some(since), since + 10_000_000_000)); + } + assert!(background_stalled(Some(since), u64::MAX)); + } + fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + attribute(own, renewals, others).decision + } + fn head(vm: u64, published: u64, finished: bool) -> Head { + Head { vm, published, finished, failed: false } + } + fn failed(vm: u64, published: u64) -> Head { + Head { vm, published, finished: false, failed: true } + } + #[test] + fn ray_query_execution_is_bounded_at_ten_seconds() { + let mut renewals = 0; + for poll in 1..=4 { + assert!(renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, poll); + } + for _ in 0..256 { + assert!(!renew_execution(true, true, &mut renewals)); + } + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); + } + #[test] + fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { + for (published, enabled) in [(false, false), (false, true), (true, false)] { + let mut renewals = 0; + assert!(!renew_execution(published, enabled, &mut renewals)); + assert_eq!(renewals, 0); + } + } + #[test] + fn ray_query_and_cross_vm_credits_are_independent_and_finite() { + let mut ray = 0; + let own = head(5, 101, false); + for _ in 0..4 { + assert!(renew_execution(true, true, &mut ray)); + } + for victim in 0..MAX_RENEWALS { + assert!(!renew_execution(true, true, &mut ray)); + assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); + } + #[test] + fn exhausted_or_invalid_credit_never_wraps() { + for value in 4..=u8::MAX { + let mut renewals = value; + assert!(!renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, value); + } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -52,6 +236,53 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] + fn a_blocker_that_already_failed_defers_then_never_spares() { + // An innocent frame published before the culprit timed out first and failed; its + // quarantined pair keeps its head until the firmware drains it. + let culprit = head(7, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); + } + assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); + assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); + // A pair that never drains does not spare any later timeout either. + let drained_never = [failed(5, 100), failed(5, 102)]; + for own in [head(7, 105, false), head(9, 200, false)] { + assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); + } + } + #[test] + fn a_live_blocker_still_spares_beside_a_failed_one() { + let own = head(9, 103, false); + let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; + assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); + assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); + // Failed or not, a blocker must be older, unfinished and of another VM. + let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; + assert_eq!(blocker(own, late.into_iter()), None); + assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); + // Among failed blockers the oldest is reported. + let both_failed = [failed(8, 102), failed(5, 100)]; + assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); + } + #[test] + fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { + let own = head(9, 103, false); + let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); + let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; + assert_eq!(verdict, expected); + assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); + assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); + let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); + assert_eq!(last.decision, Decision::Own); + assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); + let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); + assert_eq!(spared.decision, Decision::Spare); + assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); + // Nothing blocks: nothing to report. + assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); + } + #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef..b2a6cf6e8bfe3 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -90,10 +90,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +108,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 3bcf8a9692dcc..72238159d0475 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,8 +21,10 @@ use kernel::{ }, // }, error::Result, + new_mutex, prelude::*, sync::aref::ARef, + sync::Mutex, uapi, // }; @@ -40,6 +42,7 @@ use crate::{ AsahiDrmDriver, // }, file, + memacct, mmu, }; @@ -60,6 +63,11 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, + /// Memory accounting and process attribution of the held backing. + charge: memacct::Charge, + /// Serializes overlapping timestamp writes through every binding of this BO. + #[pin] + pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -74,6 +82,9 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. +/// +/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object +/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -99,6 +110,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } + gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -106,7 +118,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result<&'a kernel::scatterlist::SGTable> { +) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -115,9 +127,7 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(gem) - { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { return Err(EINVAL); } Ok(()) @@ -211,6 +221,7 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -229,20 +240,23 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -fn checked_object_size(size: usize) -> Result { - if size == 0 { return Err(EINVAL); } +pub(crate) fn checked_object_size(size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } - fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { + let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - checked_object_size(size)?, + size, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, + pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -250,6 +264,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -259,12 +274,17 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. +/// +/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the +/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, + pages_no_reclaim: bool, + commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -276,6 +296,7 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, + pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -283,6 +304,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + charge: memacct::Charge::user_current(commit), }, )?; @@ -308,16 +330,22 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), + charge: args.charge, + timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. + /// + /// Their memory is not counted: an import's backing belongs to, and is charged by, its + /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, + charge: memacct::Charge::none(), }) } @@ -372,6 +400,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether CPU mappings use the cacheable WRITEBACK policy. + pub(crate) fn is_writeback(&self) -> bool { + self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 + } + pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } @@ -439,3 +472,22 @@ extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } #[cfg(not(CONFIG_DRM_ASAHI_G15))] pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b1..be65af433a633 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -268,6 +267,11 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; @@ -285,6 +289,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + *crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -303,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { if self.queue_limits().is_some() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + if self.soft_faults() && self.supports_soft_faults() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags @@ -312,32 +320,39 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. + fn vm_destroyed(&self) {} /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 /// T8122 runtime admits any client. @@ -350,6 +365,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -359,12 +381,12 @@ pub(crate) trait Gpu: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -752,6 +774,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -1427,15 +1450,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0b..2650131a4a293 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 0000000000000..7a4d6f288dc3d --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), + // Measured on the 20-core M4 Pro; other core counts are refused at probe. + core_layout: CoreLayout { + cores: Some((2, 20)), + operand_state: fw::compute::OperandState::TWO_BY_TEN, + deflake: fw::render::Deflake::TWO_CLUSTERS, + }, +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f2..75925713bf10b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,10 +4,11 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -23,10 +24,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,13 +36,30 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + // The layouts the A18 Pro has always run with. + core_layout: crate::g17::CoreLayout { + cores: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -67,6 +85,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } @@ -273,7 +293,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -384,6 +404,9 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; + /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the + /// policy word of the user classes is kept (no macOS trace of class 1 yet). + pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -393,7 +416,9 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + // One physical pair per logical owner leaves room for every execution + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809..bcb9fd32c7d25 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -254,7 +254,8 @@ impl gpu::Gpu for Backend { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } - fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + crate::timestamp::Buffer::firmware( + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range)?) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ca16e18f10fbf..a60e38bf9addd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -18,12 +18,13 @@ impl Destination { let guard = objects.lock(); let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; match &*object { - file::Object::TimestampBuffer(mapping) => { + file::Object::TimestampBuffer(buffer) => { + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { return Err(EINVAL); } - Ok(Some(Self { mapping: mapping.clone(), offset })) + Ok(Some(Self { mapping, offset })) } } } diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 7a5587ef6ec78..5f0ab2b4aea4f 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,39 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +/// +/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per +/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second +/// UAT with another policy would change it for both. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,12 +108,16 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -167,6 +201,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -185,7 +223,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = ((asid as u64) << 48) + let val = (u64::from(asid) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs new file mode 100644 index 0000000000000..9da09968171e4 --- /dev/null +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU memory attribution, accounting and admission limits. +//! +//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the +//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of +//! the task that faulted it in, but the OOM killer scores a task by the resident set +//! of its mm, and GPU-only pages are mapped into no process. A game holding +//! gigabytes of buffers therefore scores like a tiny process, and so does a +//! compositor; which one an OOM kill hits is left to the other terms of the score. +//! +//! This module closes that gap: +//! +//! - Every user object remembers the mm of the process that created it (an `mmgrab` +//! reference, which keeps only the `mm_struct` itself). While the GPU holds the +//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. +//! They are shmem pages, so this is where `oom_badness()`, the OOM report's +//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages +//! are uncharged when idle reclaim or purge releases the backing and when the +//! object is freed; the mm reference is dropped after the last uncharge, so the +//! counter is back to its own value before `__mmdrop()` checks it. +//! - Each client (DRM file) and the device keep totals of created and held bytes, +//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the +//! `gpu_memory` sysfs file next to `agx_stats`. +//! - A new user object is admitted only while its client stays within a per-client +//! limit and the device within a device-wide limit, both fractions of RAM. +//! A refused allocation fails with `ENOMEM`, which userspace reports as +//! out-of-device-memory, instead of pushing the machine into a global OOM. +//! +//! Imported dma-bufs are charged to their exporter and are not counted here. +//! Kernel objects (firmware structures, per-queue buffers and the global heaps) +//! count towards the device total and its limit, but are attributed to no process. + +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + mm::{ + Mm, + RssCounter, // + }, + page::PAGE_SIZE, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::module_parameters; + +/// Device-wide GPU memory totals, in bytes. +/// +/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). +#[repr(C)] +pub(crate) struct Ledger { + /// Size of every live user object (created, not imported). + pub(crate) user_committed: AtomicU64, + /// Size of the live user objects whose backing the GPU currently holds. + pub(crate) user_resident: AtomicU64, + /// Size of the live kernel objects whose backing the GPU holds. + pub(crate) kernel_resident: AtomicU64, + /// User object creations refused by a limit. + pub(crate) refused: AtomicU64, + /// Device-wide limit on user committed plus kernel resident bytes (0: none). + pub(crate) limit: AtomicU64, + /// Per-client limit on committed bytes (0: none). + pub(crate) client_limit: AtomicU64, +} + +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(Ledger, refused) == 24); + assert!(core::mem::offset_of!(Ledger, client_limit) == 40); + assert!(core::mem::size_of::() == 48); +}; + +/// The ledger of the (single) AGX GPU. +pub(crate) static LEDGER: Ledger = Ledger::new(); + +/// The limits that follow from the amount of RAM and two percentages. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct Limits { + /// Device-wide limit in bytes (0: none). + pub(crate) device: u64, + /// Per-client limit in bytes (0: none). + pub(crate) client: u64, +} + +fn pct_of(total: u64, pct: u32) -> u64 { + match pct { + 0 => 0, + // Above 100% cannot be held anyway; treat it as "no limit". + p if p >= 100 => 0, + p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, + } +} + +/// Computes the limits for `total_ram` bytes of RAM. +/// +/// A percentage of 0 or 100 and above disables that limit. A per-client limit +/// is never larger than the device limit. +pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { + let device = pct_of(total_ram, device_pct); + let mut client = pct_of(total_ram, client_pct); + if device != 0 && (client == 0 || client > device) { + client = device; + } + Limits { device, client } +} + +/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). +/// +/// Returns whether the bytes were added. Concurrent callers cannot overshoot. +fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { + let mut cur = counter.load(Ordering::Relaxed); + loop { + let Some(new) = cur.checked_add(bytes) else { + return false; + }; + if limit != 0 && new.saturating_add(extra) > limit { + return false; + } + match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => return true, + Err(actual) => cur = actual, + } + } +} + +fn sub(counter: &AtomicU64, bytes: u64) { + let prev = counter.fetch_sub(bytes, Ordering::AcqRel); + if prev < bytes { + pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + counter.store(0, Ordering::Relaxed); + } +} + +impl Ledger { + pub(crate) const fn new() -> Self { + Self { + user_committed: AtomicU64::new(0), + user_resident: AtomicU64::new(0), + kernel_resident: AtomicU64::new(0), + refused: AtomicU64::new(0), + limit: AtomicU64::new(0), + client_limit: AtomicU64::new(0), + } + } + + /// Sets the limits from the module parameters and the amount of RAM. + pub(crate) fn configure(&self) { + // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. + let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; + // SAFETY: `si_meminfo()` only fills in the structure it is given. + unsafe { kernel::bindings::si_meminfo(&mut info) }; + let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); + let limits = limits_for( + total, + *module_parameters::mem_limit_pct.value(), + *module_parameters::client_mem_limit_pct.value(), + ); + self.set_limits(limits); + pr_info!( + "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + limits.device >> 20, + limits.client >> 20 + ); + } + + /// Admission capacities exported in the optional GET_PARAMS tail. + pub(crate) fn limits(&self) -> Limits { + Limits { + device: self.limit.load(Ordering::Relaxed), + client: self.client_limit.load(Ordering::Relaxed), + } + } + + pub(crate) fn set_limits(&self, limits: Limits) { + self.limit.store(limits.device, Ordering::Relaxed); + self.client_limit.store(limits.client, Ordering::Relaxed); + } + + /// Admits a new user object of `bytes` for `client`. + /// + /// Reserves the bytes against both limits; the returned [`Commit`] gives + /// them back when it is dropped. + pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { + let client_limit = self.client_limit.load(Ordering::Relaxed); + if !try_add_within(&client.committed, bytes, client_limit, 0) { + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + // Kernel objects share the device limit but are never refused. + let kernel = self.kernel_resident.load(Ordering::Relaxed); + if !try_add_within( + &self.user_committed, + bytes, + self.limit.load(Ordering::Relaxed), + kernel, + ) { + sub(&client.committed, bytes); + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + Ok(Commit { + ledger: self, + client: client.clone(), + bytes, + }) + } +} + +/// Per-client (DRM file) totals, in bytes. +pub(crate) struct Client { + /// Size of the live user objects this client created. + committed: AtomicU64, + /// Size of those whose backing the GPU currently holds. + resident: AtomicU64, +} + +impl Client { + pub(crate) fn new() -> Result> { + Ok(Arc::new( + Self { + committed: AtomicU64::new(0), + resident: AtomicU64::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn committed(&self) -> u64 { + self.committed.load(Ordering::Relaxed) + } + + pub(crate) fn resident(&self) -> u64 { + self.resident.load(Ordering::Relaxed) + } +} + +/// Bytes of a user object reserved against the limits. +pub(crate) struct Commit { + ledger: &'static Ledger, + client: Arc, + bytes: u64, +} + +impl Drop for Commit { + fn drop(&mut self) { + sub(&self.client.committed, self.bytes); + sub(&self.ledger.user_committed, self.bytes); + } +} + +enum Kind { + /// An object the DRM core created (a dma-buf import): charged to its exporter. + None, + /// A driver-owned object. + Kernel(&'static Ledger), + /// A user object, attributed to the process that created it. + User { + commit: Commit, + mm: Option>, + }, +} + +/// The accounting state of one GEM object. +pub(crate) struct Charge { + kind: Kind, + bytes: u64, + /// Whether the bytes are currently counted as held by the GPU. + held: AtomicBool, +} + +// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. +unsafe impl Send for Charge {} +// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. +unsafe impl Sync for Charge {} + +impl Charge { + /// An object that is not counted (a dma-buf import). + pub(crate) fn none() -> Self { + Self { + kind: Kind::None, + bytes: 0, + held: AtomicBool::new(false), + } + } + + /// A kernel object of `bytes`. + pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { + Self { + kind: Kind::Kernel(ledger), + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object admitted by `commit`, attributed to `mm` if there is one. + pub(crate) fn user(commit: Commit, mm: Option>) -> Self { + let bytes = commit.bytes; + let mm = if *module_parameters::oom_attribution.value() != 0 { + mm + } else { + None + }; + Self { + kind: Kind::User { commit, mm }, + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object attributed to the current process. + pub(crate) fn user_current(commit: Commit) -> Self { + let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + Self::user(commit, mm) + } + + fn pages(&self) -> isize { + isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) + } + + fn apply(&self, held: bool) { + match &self.kind { + Kind::None => {} + Kind::Kernel(ledger) => { + if held { + ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&ledger.kernel_resident, self.bytes); + } + } + Kind::User { commit, mm } => { + if held { + commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); + commit + .ledger + .user_resident + .fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&commit.client.resident, self.bytes); + sub(&commit.ledger.user_resident, self.bytes); + } + if let Some(mm) = mm { + let pages = self.pages(); + mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); + } + } + } + } + + /// Records that the GPU now holds (or no longer holds) the object's backing. + /// + /// Idempotent: only a change of state is counted. + pub(crate) fn set_held(&self, held: bool) { + if self.held.swap(held, Ordering::AcqRel) != held { + self.apply(held); + } + } +} + +impl Drop for Charge { + fn drop(&mut self) { + // Uncharge before `kind` (and with it the mm reference) is dropped. + self.set_held(false); + } +} + +#[kunit_tests(asahi_memacct)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + const MIB: u64 = 1 << 20; + + #[test] + fn default_limits_follow_ram() { + // The module parameter defaults. + let l = limits_for(8 * GIB, 67, 50); + assert_eq!(l.device, 8 * GIB * 67 / 100); + assert_eq!(l.client, 4 * GIB); + } + + #[test] + fn disabled_and_clamped_limits() { + assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); + assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); + // A client limit above the device limit is clamped to it. + let l = limits_for(8 * GIB, 25, 50); + assert_eq!(l.client, l.device); + // No client limit with a device limit means the device limit. + let l = limits_for(8 * GIB, 50, 0); + assert_eq!(l.client, 4 * GIB); + // No device limit leaves the client limit alone. + let l = limits_for(8 * GIB, 0, 25); + assert_eq!(l, Limits { device: 0, client: 2 * GIB }); + } + + #[test] + fn percentages_do_not_overflow() { + let l = limits_for(u64::MAX, 99, 98); + assert!(l.device > l.client); + assert!(l.device <= u64::MAX / 100 * 99 + 99); + } + + #[test] + fn try_add_respects_limit_and_extra() { + let c = AtomicU64::new(0); + assert!(try_add_within(&c, 6, 10, 0)); + assert!(!try_add_within(&c, 5, 10, 0)); + assert!(try_add_within(&c, 4, 10, 0)); + assert_eq!(c.load(Ordering::Relaxed), 10); + let d = AtomicU64::new(0); + assert!(!try_add_within(&d, 6, 10, 5)); + assert!(try_add_within(&d, 5, 10, 5)); + let e = AtomicU64::new(u64::MAX - 1); + assert!(!try_add_within(&e, 2, 0, 0)); + assert!(try_add_within(&e, 1, 0, 0)); + } + + static ADMIT: Ledger = Ledger::new(); + + #[test] + fn admission_reserves_and_releases() -> Result { + ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let c = Client::new()?; + let a1 = ADMIT.admit(&a, 40 * MIB)?; + // The client limit refuses a's second 40 MiB; nothing stays reserved. + assert!(ADMIT.admit(&a, 40 * MIB).is_err()); + assert_eq!(a.committed(), 40 * MIB); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); + let b1 = ADMIT.admit(&b, 50 * MIB)?; + // The device limit refuses c's 20 MiB and rolls back c's reservation. + assert!(ADMIT.admit(&c, 20 * MIB).is_err()); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); + assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); + drop(a1); + assert_eq!(a.committed(), 0); + let c1 = ADMIT.admit(&c, 20 * MIB)?; + drop((b1, c1)); + assert_eq!(b.committed(), 0); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + static KERNEL: Ledger = Ledger::new(); + + #[test] + fn kernel_objects_count_against_the_device_limit() -> Result { + KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); + let k = Charge::kernel(&KERNEL, 48 * MIB); + k.set_held(true); + let a = Client::new()?; + assert!(KERNEL.admit(&a, 32 * MIB).is_err()); + let c = KERNEL.admit(&a, 16 * MIB)?; + drop(k); + assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); + drop(c); + Ok(()) + } + + static HELD: Ledger = Ledger::new(); + + #[test] + fn held_transitions_are_idempotent() -> Result { + HELD.set_limits(Limits { device: 0, client: 0 }); + let a = Client::new()?; + let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); + assert!(!charge.held.load(Ordering::Relaxed)); + charge.set_held(true); + charge.set_held(true); + assert_eq!(a.resident(), 8 * MIB); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); + charge.set_held(false); + charge.set_held(false); + assert_eq!(a.resident(), 0); + charge.set_held(true); + // Dropping a held object uncharges it and returns its commit. + drop(charge); + assert_eq!(a.resident(), 0); + assert_eq!(a.committed(), 0); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); + assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + #[test] + fn imports_are_not_counted() { + let charge = Charge::none(); + charge.set_held(true); + assert!(charge.held.load(Ordering::Relaxed)); + drop(charge); + } + + static MM: Ledger = Ledger::new(); + + // `mm_alloc()` is only exported to modules in the KUnit namespace. + #[cfg(not(MODULE))] + #[test] + fn held_pages_are_charged_to_the_creating_mm() -> Result { + MM.set_limits(Limits { device: 0, client: 0 }); + // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. + let raw = unsafe { kernel::bindings::mm_alloc() }; + if raw.is_null() { + return Err(ENOMEM); + } + // SAFETY: `raw` is a live mm until the `mmput()` below. + let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); + let a = Client::new()?; + let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); + assert_eq!(mm.rss_counter(RssCounter::Anon), 0); + charge.set_held(false); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + // Freeing a held object uncharges it before dropping its mm reference. + drop(charge); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + drop(mm); + // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then + // checks that every RSS counter is back to zero. + unsafe { kernel::bindings::mmput(raw) }; + Ok(()) + } +} + +#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] +mod exported_limits_tests { + use super::*; + + #[test] + fn exported_capacities_match_admission() { + let ledger = Ledger::new(); + assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); + let limits = limits_for(8 << 30, 67, 50); + ledger.set_limits(limits); + assert_eq!(ledger.limits(), limits); + assert!(limits.client < limits.device); + assert_eq!(limits.client, 4 << 30); + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bf4a6823024d..cda3ac8d206bd 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -115,14 +114,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +156,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -363,6 +421,8 @@ struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } @@ -375,7 +435,20 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -494,22 +567,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -556,6 +623,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -641,7 +709,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -684,7 +753,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -719,14 +793,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -736,6 +803,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -790,15 +870,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -826,10 +910,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -837,7 +946,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -895,8 +1011,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -974,7 +1089,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -982,8 +1097,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1024,6 +1140,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1064,10 +1186,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1117,22 +1236,51 @@ pub(crate) struct TimestampMapping { } impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: gem.owned_vmap::<0>()?, + offset, + }) + } + pub(crate) fn write(&self, value: u64) { + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.mapping.owner().timestamp_lock.lock(); + // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + let pointer = unsafe { base.add(self.offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(self.offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.mapping.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1156,12 +1304,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)? + .page_table + .translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; @@ -1174,10 +1324,7 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + TimestampMapping::new(gem, start) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1292,10 +1439,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1385,6 +1532,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1427,8 +1587,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1437,6 +1616,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1453,6 +1634,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1461,14 +1647,44 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1482,11 +1698,15 @@ impl UatShared { } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1503,20 +1723,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1569,9 +1907,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1654,16 +1992,36 @@ impl HandoffFlush { fn prefetch_sgt_segments( dev: &device::Device, gem: &gem::Object, -) -> Result> { +) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { segments, _lease: sgt.into_lease() }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1686,12 +2044,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1717,6 +2075,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1771,7 +2130,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1784,13 +2146,16 @@ impl Vm { /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1899,9 +2264,7 @@ impl Vm { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } @@ -1962,6 +2325,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1976,17 +2370,16 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let size = object_range.range(); let mut vm_exec = unsafe { @@ -1994,14 +2387,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2022,7 +2407,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2040,9 +2425,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2060,17 +2452,16 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(&gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2082,14 +2473,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2124,9 +2507,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2144,17 +2534,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2172,14 +2561,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2222,6 +2603,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2265,22 +2662,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2485,8 +2895,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2499,9 +2919,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2532,6 +2953,18 @@ pub(crate) struct AdoptedRegions { } impl Uat { + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. fn map_physical(range: &Range, min_size: usize) -> Result { @@ -2581,10 +3014,24 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, + ) -> Result { + Self::new_adopted_with_user_upper(dev, geometry, regions, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, ) -> Result { let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; @@ -2596,8 +3043,12 @@ impl Uat { let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2606,10 +3057,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2621,8 +3076,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2725,7 +3186,10 @@ impl Uat { } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2754,7 +3218,9 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); @@ -2773,16 +3239,24 @@ impl Uat { let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2791,8 +3265,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2811,6 +3289,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for index in UAT_USER_CTX_START..UAT_NUM_CTX { + if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2862,6 +3369,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2871,14 +3379,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2900,7 +3411,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -2920,7 +3431,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -2936,15 +3447,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2961,24 +3476,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -2988,7 +3512,12 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -2997,12 +3526,16 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3011,10 +3544,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3030,12 +3567,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3043,11 +3595,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3098,11 +3650,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3114,7 +3670,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3123,8 +3679,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3142,3 +3702,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = lock_vm(&self.gpuvm, None, false)?; + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 44a1575ecfd2b..e222c7cd8e6a9 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,6 +17,7 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, + _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -57,7 +58,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state without taking DMA reservations or changing any leaf. + /// Prepares host state before taking the GPUVM exec lock or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -80,24 +81,20 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - crate::gem::held_sg_table(gem, dev)?; + let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - { - let mut segments = vm_bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetch_sgt_segments(dev, gem)?); - } - } + prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, + _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 30010be497973..66d507c2d7694 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,9 +79,13 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 && state.commits == 0 - && !state.blocked && !state.quarantined && !state.draining - && !state.closed && !state.cleanup_failed + state.active == 0 + && state.commits == 0 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed }) } @@ -339,10 +343,12 @@ impl Vm { drop(previous); } - /// Whether userspace has permanently closed this VM's map admission. - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed + /// admission for good). G15 releases its per-VM state after that. pub(crate) fn is_closed(&self) -> bool { - self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -411,7 +417,10 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); + return Ok(VmJobGuard { + vm: self.clone(), + _residency: residency, + }); } lifetime.changed.wait(&mut state); } @@ -471,7 +480,11 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { + pub(super) fn defer_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + start: usize, + ) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -485,7 +498,8 @@ impl Vm { } if batch .ops - .iter().skip(start) + .iter() + .skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -496,16 +510,21 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = + self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + if !state + .unmaps + .iter() + .any(|old| old.iova == unmap.iova && old.size == unmap.size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -530,16 +549,30 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { return Err(ENOENT); } - if !state.blocked { return Ok(false); } - if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -634,8 +667,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); - let objects = core::mem::replace(&mut state.objects, KVec::new()); + let unmaps = core::mem::take(&mut state.unmaps); + let objects = core::mem::take(&mut state.objects); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -671,7 +704,9 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { status.record(error); } + if let Some(status) = &self.status { + status.record(error); + } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a813426..8611221af492a 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs new file mode 100644 index 0000000000000..7cdad5e1f3831 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/ppl.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire +//! guard lifetime: firmware exclusion alone does not serialize two AP threads. +//! This core has no kernel dependencies so the production guard can be exercised +//! against deterministic firmware and concurrent host transports. + +pub(super) trait Memory { + fn firmware_interested(&self) -> bool; + fn firmware_turn(&self) -> bool; + fn ap_interest(&self, interested: bool); + fn give_firmware_turn(&self); + fn barrier(&self); + fn expired(&self) -> bool; + fn relax(&self); + /// Latch failure before the transport releases host exclusion on timeout. + fn fail(&self); +} + +/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. +#[must_use = "retain the guard until every protected table mutation has completed"] +pub(super) struct Guard { + memory: T, +} + +impl Guard { + pub(super) fn acquire(memory: T) -> Result { + let timeout = || { + memory.ap_interest(false); + memory.barrier(); + memory.fail(); + Err(()) + }; + memory.ap_interest(true); + memory.barrier(); + while memory.firmware_interested() { + if memory.expired() { + return timeout(); + } + if memory.firmware_turn() { + memory.ap_interest(false); + memory.barrier(); + while memory.firmware_turn() { + if memory.expired() { + return timeout(); + } + memory.relax(); + } + memory.ap_interest(true); + memory.barrier(); + } + memory.relax(); + } + memory.barrier(); + Ok(Self { memory }) + } + + pub(super) fn memory_mut(&mut self) -> &mut T { + &mut self.memory + } +} + +impl Drop for Guard { + fn drop(&mut self) { + self.memory.give_firmware_turn(); + self.memory.barrier(); + self.memory.ap_interest(false); + self.memory.barrier(); + // Only after these stores complete may `memory` release the host mutex. + } +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 5370b87ccf1b5..49c2241bc399f 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,6 +251,8 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} +// SAFETY: Shared callbacks use the VM's residency/reservation locks. The +// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -273,7 +275,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -281,23 +283,36 @@ impl VmShrinker { Ok(owner) } + /// Count reclaimable pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker. The core must + /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() as _ + owner.vm.idle_reclaim_count() } + /// Reclaim idle pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker, and `sc` must + /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - let target = unsafe { (*sc).nr_to_scan } as usize; - owner.vm.reclaim_idle_pages(target) as _ + // SAFETY: The shrinker core supplies a live initialized control. + let target = unsafe { (*sc).nr_to_scan }; + owner.vm.reclaim_idle_pages(target) } } @@ -348,7 +363,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -361,11 +376,17 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Purgeable objects count on busy VMs too; idle backing only on idle ones. - if object.idle_reclaim_candidate() - && (object.madv_locked() > 0 || idle) - && object.sole_sg_lease_releasable_locked() - { + // Count retained backing on idle VMs, and purgeable contents even if idle + // reclaim already released the table. Never wait on the BO mutex under resv. + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + let advised = object.madv_locked() > 0; + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + (advised || object.has_sg_table_locked()) + && object.unleased_pages_releasable_locked() + }; + if object.idle_reclaim_candidate() && (advised || idle) && releasable { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -385,7 +406,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -396,7 +417,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -408,9 +429,12 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_some() && !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -467,7 +491,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) || !self .lifetime .as_ref() @@ -482,7 +506,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -494,9 +518,14 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_none() || !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + // A competing lease may have made the previous release fail after we + // dropped our segments. Retry that table, but do not count empty backing. + object.has_sg_table_locked() && object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index b9e2975c898ff..fec7a6b61ad8f 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: gem as *const gem::Object as usize, + object: core::ptr::from_ref(gem) as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = gem as *const gem::Object as usize; + let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 38a2268137eff..a66a82d92a583 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -409,7 +409,7 @@ impl> GpuObject { }; let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index b88353a2f7344..c5a47c24ec928 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -53,19 +54,20 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). +/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local +/// CTR_EL0 read can select too large a stride if the task migrates during the loop. #[inline(always)] fn dcache_line_size() -> usize { - let ctr: u64; - // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. - unsafe { - core::arch::asm!( - "mrs {ctr}, ctr_el0", - ctr = out(reg) ctr, - options(nomem, nostack, preserves_flags) - ) - }; - 4 << ((ctr >> 16) & 0xf) + use kernel::bindings; + let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) + | (bindings::SYS_CTR_EL0_Op1 << 16) + | (bindings::SYS_CTR_EL0_CRn << 12) + | (bindings::SYS_CTR_EL0_CRm << 8) + | (bindings::SYS_CTR_EL0_Op2 << 5); + // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine + // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. + let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; + 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -95,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -116,20 +117,114 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). +/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -137,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -253,6 +351,8 @@ impl Prot { Self { memattr, ..*self } } + /// The protection bits of a leaf with this protection (no address, table type or non-global + /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() @@ -278,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -296,6 +412,12 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Ownership could not be relinquished to firmware; physical tables remain until reboot. + retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -305,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -317,45 +438,49 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + retained: false, + reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb osh`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// `dsb sy`. Call it before anything is mapped and before any context names the root. pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -366,6 +491,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -381,7 +507,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -392,15 +520,23 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + retained: false, + reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } + /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim + /// this tree merely because a userspace handle closed or a fence reported an error. + pub(crate) fn retain(&mut self) { + self.retained = true; + } + pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -470,6 +606,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -513,55 +650,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); } - })?; + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -597,13 +737,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -611,7 +748,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -636,16 +778,9 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { + fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } + tables.contains(&phys) } else { true }; @@ -653,32 +788,44 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; + let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; tables.swap_remove(index); } + if self.cleans_poc() { + // Complete the parent descriptor clear before its detached child can be reused. + // Callers have already removed this tree from live contexts before freeing tables. + poc_barrier(); + } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -769,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -788,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -814,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -861,6 +1019,9 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { + if self.retained { + return; + } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index a68352828cfbc..20396f2e77225 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.clone(), + mapping: mapping.firmware_mapping()?, offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 0872a50ce5d46..4abdd3e485f72 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -262,7 +262,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { QueueTimeout { dev: self.dev.clone(), id: self.id } @@ -754,7 +755,22 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - cmdbuf.skip(header.size as usize); + let payload_size = header.size as usize; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; + if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; + if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + _ => cmdbuf.skip(payload_size), + } nr_commands += 1; match header.cmd_type as u32 { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index fb22569ea5fdf..0a4ee8e7563ec 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,10 +13,11 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, c_str, + bindings, + c_str, device::Core, - devres::Devres, error::{from_err_ptr, to_result}, + devres::Devres, io::{ mem::IoMem, Io, @@ -222,7 +223,10 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + from_err_ptr(bindings::apple_mbox_get( + pdev.as_ref().as_raw(), + index.try_into()?, + ))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -235,7 +239,10 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { dev: pdev.into(), mbox }) + Ok(Self { + dev: pdev.into(), + mbox, + }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -251,7 +258,11 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + dev_err!( + self.dev.as_ref(), + "Could not stop GPU coprocessor: {:?}\n", + error + ); } } } @@ -560,7 +571,10 @@ impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + let sgx = Arc::new( + sgx_req.iomap_sized::()?.into_devres()?, + GFP_KERNEL, + )?; Ok(G17Resources { dev: pdev.into(), @@ -701,7 +715,6 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } - } /// Split the packed per-cluster core masks into one mask per cluster. @@ -732,11 +745,7 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); + dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); return Err(EIO); } @@ -744,7 +753,10 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision( + dev: &platform::Device, + id_version: u32, +) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -758,3 +770,29 @@ fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevis } }) } + +/// Device-managed SGX mapping retained by the G16 firmware session. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[derive(Clone)] +pub(crate) struct G16Resources { + sgx: Arc>>, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl G16Resources { + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; + Ok(Self { + sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, + }) + } + pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_read32(offset) + } + #[allow(dead_code)] + pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_write32(value, offset) + } +} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 9f0baa4350d93..7d7d785a83133 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -32,9 +32,21 @@ struct asahi_stats_snapshot { u64 jobs; }; +/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ +struct asahi_gpu_memory { + u64 user_committed; + u64 user_resident; + u64 kernel_resident; + u64 refused; + u64 limit; + u64 client_limit; +}; + struct asahi_stats_attribute { struct device_attribute attr; + struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; + const struct asahi_gpu_memory *memory; bool export_enabled; }; @@ -72,20 +84,52 @@ static ssize_t agx_stats_show(struct device *dev, return n; } +/* + * GPU memory held by the driver, in bytes: user objects created (committed) + * and held by the GPU (resident), held kernel objects, the limits that admit + * user objects (0: none) and the number of refused creations. + */ +static ssize_t gpu_memory_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, memory_attr); + const struct asahi_gpu_memory *mem = stats->memory; + ssize_t n = 0; + + n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", + (unsigned long long)READ_ONCE(mem->user_committed)); + n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", + (unsigned long long)READ_ONCE(mem->user_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", + (unsigned long long)READ_ONCE(mem->kernel_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", + (unsigned long long)READ_ONCE(mem->limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", + (unsigned long long)READ_ONCE(mem->client_limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", + (unsigned long long)READ_ONCE(mem->refused)); + + return n; +} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle) + int export_enabled, const void *memory, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || (export_enabled && !snapshot)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); + BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -102,17 +146,70 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->snapshot = snapshot; stats->export_enabled = export_enabled; + sysfs_attr_init(&stats->memory_attr.attr); + stats->memory_attr.attr.name = "gpu_memory"; + stats->memory_attr.attr.mode = 0444; + stats->memory_attr.show = gpu_memory_show; + stats->memory = memory; + ret = device_create_file(dev, &stats->attr); if (ret) { kfree(stats); return ret; } + ret = device_create_file(dev, &stats->memory_attr); + if (ret) { + device_remove_file(dev, &stats->attr); + kfree(stats); + return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); +/* + * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores + * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores + * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs + * about 1 ms, so userspace may lengthen it on AC power. + */ +static u32 *asahi_idle_word; + +static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); +} + +static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + u32 ms; + + if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) + return -EINVAL; + WRITE_ONCE(*asahi_idle_word, ms); + return count; +} + +static DEVICE_ATTR_RW(idle_off_delay_ms); + +int asahi_sysfs_register_idle(struct device *dev, u32 *word) +{ + int ret; + + if (!dev || !word) + return -ENODEV; + asahi_idle_word = word; + ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); + if (ret) + asahi_idle_word = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); + void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -121,7 +218,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) return; /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); + if (asahi_idle_word) { + device_remove_file(dev, &dev_attr_idle_off_delay_ms); + asahi_idle_word = NULL; + } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 5969a1718925f..f1873f9be6caa 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -6,7 +6,8 @@ struct device; int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle); + int export_enabled, const void *memory, void **handle); +int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 079623d1ef210..cee5d52a4e261 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,7 +3,7 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::stats::StatsSnapshot; +use crate::{memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; extern "C" { @@ -11,12 +11,30 @@ extern "C" { dev: *mut c_void, snapshot: *const StatsSnapshot, export_enabled: c_int, + memory: *const Ledger, handle: *mut *mut c_void, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } +extern "C" { + fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; +} + +/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). +pub(crate) fn register_idle( + dev: *mut kernel::bindings::device, + word: *mut u32, +) -> kernel::error::Result { + // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes + // it with READ_ONCE/WRITE_ONCE. + let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; + if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } +} + /// Register only after initialization succeeds, keeping the snapshot Arc alive. +/// +/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, @@ -30,6 +48,7 @@ pub(crate) fn register( dev.cast(), snapshot, c_int::from(export_enabled), + &crate::memacct::LEDGER, &mut handle, ) }; diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs new file mode 100644 index 0000000000000..976ec9f6bf48b --- /dev/null +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared timestamp destinations shared by native command completion paths. + +use crate::mmu; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use crate::{file, gem}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::ops::Range; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::sync::atomic::{fence as barrier, Ordering}; +use kernel::{prelude::*, sync::Arc}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use kernel::{uapi, xarray}; + +/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. +/// CPU-retired timestamps retain their backing without consuming global firmware address space. +pub(crate) enum Buffer { + Firmware(Arc), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Cpu { + // The GEM pin keeps the object's held SG table alive through CPU publication. + gem: gem::KernelMappingPin, + range: Range, + }, +} + +impl Buffer { + pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { + Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + } + + /// Retain a CPU-written byte range without publishing any firmware address. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { + let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { + return Err(EINVAL); + } + let gem = gem::KernelMappingPin::new(&bo.gem); + let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let _ = gem::held_sg_table(&gem, parent.as_ref())?; + Ok(Self::Cpu { gem, range }) + } + + pub(crate) fn size(&self) -> usize { + match self { + Self::Firmware(mapping) => mapping.size(), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { range, .. } => range.end - range.start, + } + } + + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + match self { + Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { gem, range, .. } => { + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) + } + } + } + + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. + pub(crate) fn firmware_mapping(&self) -> Result> { + match self { + Self::Firmware(mapping) => Ok(mapping.clone()), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { + let offset = usize::try_from(offset).map_err(|_| EINVAL)?; + let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; + if aligned && offset % size_of::() != 0 { + return Err(EINVAL); + } + if end > size { + return Err(if aligned { EINVAL } else { ERANGE }); + } + Ok(offset) +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +struct Destination { + // Retain the bound range and backing along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + aligned: bool, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { + _mapping: mapping, + cpu, + })) + } + + fn write(&self, ticks: u64, frequency_hz: u64) { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / frequency_hz) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); + self.cpu.write(ns); + barrier(Ordering::Release); + } +} + +/// Timestamp mappings resolved and retained before any command is enqueued. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct Destinations(Option; 4]>>); + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destinations { + /// G16 retains its eight-byte destination alignment and EINVAL contract. + pub(crate) fn resolve_aligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, true) + } + + fn resolve_inner( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + aligned: bool, + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + /// Publish measured timestamps before signalling completion or releasing dependent work. + pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { + for (destination, ticks) in self.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks, frequency_hz); + } + } + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = self + .0 + .as_deref() + .map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573c..a5f9455b23f5f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -217,3 +217,17 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } + +/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +#[inline(always)] +pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__pinned_init(slot) } +} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e5..9de2e7f0f4f97 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,8 +87,16 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + /* A bulk pin must not run direct reclaim once per folio when + * the driver can instead return an allocation failure. Clearing + * DIRECT_RECLAIM also makes memcg charges fail without reclaim + * or invoking the OOM killer. Preserve placement and kswapd. + */ + if (shmem->pages_no_reclaim) + gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); + mapping_set_gfp_mask(obj->filp->f_mapping, gfp); } return 0; @@ -612,10 +620,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + return !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == 1 && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1075,6 +1083,10 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; + /* Purged contents cannot be restored by rebuilding an empty table. */ + if (shmem->madv < 0) + return ERR_PTR(-EINVAL); + drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa6..30791d328550d 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,6 +37,13 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; + /** + * @pages_no_reclaim: Disable direct reclaim during backing population. + * Set before drm_gem_shmem_init(). Drivers that can fail a large pin + * request cheaply may opt in; background reclaim remains enabled. + */ + bool pages_no_reclaim; + /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -145,7 +152,9 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index d311b92620b86..66396f89cc333 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,6 +212,20 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; + + /** + * @memory_limit: Device-wide limit on user committed plus kernel + * resident bytes. Zero means no driver limit. This is a capacity, + * not current free memory; other clients share it. + */ + __u64 memory_limit; + + /** + * @client_memory_limit: Per-DRM-file limit on user committed bytes. + * Zero means no driver limit. Userspace must zero-initialize the reply + * buffer: older kernels leave these optional tail fields untouched. + */ + __u64 client_memory_limit; }; /** @@ -348,6 +362,12 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, + + /** + * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept + * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. + */ + DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -806,7 +826,8 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Reserved for future extension. + * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails + * with -EACCES. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -1614,6 +1635,17 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + + /** + * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, + * including compute-based transfers. Requires BOUNDED_AS_BUILD. + * This is an execution hint, not proof of progress or permission to + * access additional memory. The kernel validates the command normally + * and caps its execution allowance at ten seconds (four extra 2s polls). + * Other commands in the same submission receive the same allowance so + * ordered helpers cannot time out while waiting for the build. + */ + DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 099e64b3ce5fd..9ab7494251e28 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index b5540997bd20d..d9933c4595682 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,6 +23,18 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } +__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, + long value) +{ + add_mm_counter(mm, member, value); +} + +__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, + int member) +{ + return get_mm_counter_sum(mm, member); +} + __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f245049..e63b62b54a17d 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: None, + show_fdinfo: Some(drm::File::::show_fdinfo_callback), fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8a56f00203f56..88709a7f9b36d 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,6 +12,23 @@ use crate::{ }; use core::marker::PhantomData; +/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. +/// +/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct MemoryStats { + /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). + pub shared: u64, + /// Memory private to the client (part of `drm-total-*`). + pub private: u64, + /// Memory currently resident (`drm-resident-*`). + pub resident: Option, + /// Resident memory userspace marked purgeable (`drm-purgeable-*`). + pub purgeable: Option, + /// Memory in use by active submissions (`drm-active-*`). + pub active: Option, +} + /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -22,6 +39,11 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; + + /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. + fn memory_stats(&self) -> Option { + None + } } /// An open DRM File. @@ -48,6 +70,12 @@ impl File { self.0.get() } + /// Whether this file holds the device's DRM master (the compositor or display server). + pub fn is_current_master(&self) -> bool { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. + unsafe { bindings::drm_is_current_master(self.as_raw()) } + } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -99,6 +127,47 @@ impl File { 0 } + /// The `show_fdinfo` callback of a `struct drm_driver`. + pub(crate) extern "C" fn show_fdinfo_callback( + printer: *mut bindings::drm_printer, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose + // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the + // file, so `postclose_callback()` cannot run concurrently. + let file = unsafe { File::::from_raw(raw_file) }; + let Some(stats) = file.inner().memory_stats() else { + return; + }; + + let mut supported = 0; + let mut raw = bindings::drm_memory_stats { + shared: stats.shared, + private: stats.private, + resident: 0, + purgeable: 0, + active: 0, + }; + if let Some(resident) = stats.resident { + raw.resident = resident; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; + } + if let Some(purgeable) = stats.purgeable { + raw.purgeable = purgeable; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; + } + if let Some(active) = stats.active { + raw.active = active; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; + } + + // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` + // and the region name outlive the call. + unsafe { + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + }; + } + /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483..83eca852139be 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,6 +82,12 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, + /// Fail backing population when it would need direct reclaim. + /// + /// Background reclaim and ordinary allocation placement are preserved. + /// Intended for large, fallible pin requests, not essential kernel backing. + pub pages_no_reclaim: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -94,13 +100,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, + pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -116,12 +122,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// Devres object for unmapping any SGTable on driver-unbind. - sgt_res: ManuallyDrop>>>, + /// One cleanup registration for the C SG-table cache; never holds a table borrow. + sgt_cleanup: ManuallyDrop>>>, #[pin] - /// Lock for protecting initialization of `sgt_res`. + /// Serializes cleanup registration and table acquisition. sgt_lock: Mutex<()>, - /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, #[pin] inner: T, @@ -189,13 +195,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_res` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + // `sgt_cleanup` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -273,67 +279,59 @@ impl Object { self.make_vmap() } - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA - /// pages for this object. + /// Acquire a revocable lease on this object's DMA scatter-gather table. /// - /// This will pin the object in memory. It is expected that `dev` should be a pointer to the - /// same [`device::Device`] which `self` belongs to, otherwise this function will return - /// `Err(EINVAL)`. - pub fn sg_table<'a>( - &'a self, - dev: &'a device::Device, - ) -> Result<&'a scatterlist::SGTable> { + /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. + /// The table borrow cannot outlive the lease. The device must be this object's parent. + /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. + pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // SAFETY: `import_attach` is set before an imported object is published and does not - // change until the object is freed. - if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // The SG table of an imported object is the dma-buf attachment's mapping, which the - // exporter made for this device at import time. It belongs to the attachment, and - // `drm_gem_shmem_release()` hands it back to the exporter through - // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` - // would unmap and free it as if these helpers had built it. - // - // SAFETY: `sgt` of an imported object is set before the object is published and stays - // valid and unmodified until the object is freed, which `'a` cannot outlive. + // Release and purge set the reclaim bit only while they hold this object's reservation. + // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. + while let Err(e) = self.acquire_sgt_access() { + if e != EBUSY { + return Err(e); + } + drop(DmaResvGuard::new(self)); + } + // Every failure after incrementing the count drops this lease. + let lease = SGLease { obj: self.into() }; + + // SAFETY: Import attachment ownership is immutable after publication. + let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // Imports belong to the exporter. Never register them for Rust devres cleanup, + // rebuild their tables, or release their backing. + // SAFETY: The attachment table stays valid until this object is freed. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - // SAFETY: See above. - return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); - } - - // The borrowed reference can outlive any lock, so pin the backing permanently. - self.acquire_sgt_access(true)?; - - let sgt_res = 'out: { - // Fast path: sgt_res is already initialized - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; - } - - // Slow path: Grab the lock and see if we need to initialize sgt_res. + sgt + } else { + // Register cleanup once, but cache the actual table only in C's shmem->sgt. + // Reclaim clears that pointer. No Rust reference to it survives without a lease. let _guard = self.sgt_lock.lock(); - - // If someone initialized it while we were waiting, we can exit early. - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; + if self.sgt_cleanup.as_ref().is_none() { + self.sgt_cleanup.populate(Devres::new( + dev, + Ok::<_, Error>(SGTableCleanup { obj: self.into() }), + )?); } - - // If not, finish initializing and return. `populate()` cannot return false, as - // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. - self.sgt_res - .populate(Devres::new(dev, SGTableMap::new(self))?); - - // SAFETY: We just populated sgt_res above. - unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; + // SAFETY: The live object and lease exclude backing release. The helper locks + // the reservation and either returns the cached table or rebuilds it. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? }; - Ok(sgt_res.access(dev)?) + Ok(SGTable { + // SAFETY: The lease excludes release; the Bound device excludes devres revocation. + table: unsafe { scatterlist::SGTable::from_raw(sgt) }, + lease, + }) } /// Create a new shmem-backed DRM object of the given size. @@ -349,7 +347,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -360,6 +358,10 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + // SAFETY: The object is initialized but not published. The C initializer + // reads this policy when setting up the backing mapping's GFP mask. + unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; + // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -403,7 +405,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -429,23 +431,18 @@ impl Object { } } - fn acquire_sgt_access(&self, permanent: bool) -> Result { + fn acquire_sgt_access(&self) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - let next = if permanent { - state | SGT_PERMANENT - } else { - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } - state + 1 - }; + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } match self.sgt_access.compare_exchange_weak( state, - next, + state + 1, Ordering::AcqRel, Ordering::Acquire, ) { @@ -466,11 +463,34 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } + /// Whether the object still has a cached device SG table. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn has_sg_table_locked(&self) -> bool { + // SAFETY: Reservation ownership serializes the C table pointer. + !unsafe { (*self.as_raw_shmem()).sgt }.is_null() + } + + /// Whether backing without a Rust SG lease can be reclaimed or purged. + /// + /// Also accepts backing whose SG table was already released by idle reclaim. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 0 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that - /// borrow may outlive any lock. + /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on + /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. /// /// # Safety /// @@ -516,7 +536,9 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } + || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + { return Err(EBUSY); } self.sgt_access @@ -706,63 +728,72 @@ where { } -/// A reference to a GEM object that is known to have a mapped [`SGTable`]. -/// -/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables -/// on GEM shmem objects are revoked on driver-unbind. -/// -/// # Invariants +/// An owned backing lease, suitable for retaining copied DMA segments. /// -/// - `self.obj` always points to a valid GEM object. -/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an -/// [`SGTable`]. -/// -/// [`SGTable`]: scatterlist::SGTable -pub struct SGTableMap { - obj: NonNull>, +/// Dropping the last lease permits reclaim after device translations have been invalidated. +/// This does not itself provide access to the table: table access also requires a Bound device. +#[must_use = "retain until all device uses of the backing have ended"] +pub struct SGLease { + obj: ARef>, } -impl Deref for SGTableMap { +impl Drop for SGLease { + fn drop(&mut self) { + let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); + debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); + } +} + +/// A leased SG-table borrow. References obtained from it cannot outlive the lease. +#[must_use = "retain while using the SG table or transfer into an owned lease"] +pub struct SGTable<'a, T: DriverObject> { + table: &'a scatterlist::SGTable, + lease: SGLease, +} + +impl Deref for SGTable<'_, T> { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - // SAFETY: - // - The NonNull is guaranteed to be valid via our type invariants. - // - The sgt field is guaranteed to be initialized and valid via our type invariants. - unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + self.table } } -impl Drop for SGTableMap { - fn drop(&mut self) { - // SAFETY: `obj` is always valid via our type invariants - let obj = unsafe { self.obj.as_ref() }; - let _lock = DmaResvGuard::new(obj); - - // SAFETY: We acquired the lock needed for calling this function above - unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; +impl SGTable<'_, T> { + /// Retain the backing after copying the DMA addresses, without retaining a table borrow. + pub fn into_lease(self) -> SGLease { + self.lease } } -impl SGTableMap { - fn new(obj: &Object) -> impl Init { - // INVARIANT: - // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, - // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. - // SAFETY: - // - `obj` is fully initialized, making this function safe to call. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; +/// Device-unbind cleanup registration for locally allocated SG tables. +/// +/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may +/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. +struct SGTableCleanup { + obj: NonNull>, +} - Ok(Self { obj: obj.into() }) +impl Drop for SGTableCleanup { + fn drop(&mut self) { + // SAFETY: The registration is dropped before its GEM object. + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. + // Reclaim may already have freed the table; the C free helper requires a non-null one. + unsafe { + if !(*obj.as_raw_shmem()).sgt.is_null() { + bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); + } + } } } -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Send for SGTableMap {} -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Sync for SGTableMap {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Send for SGTableCleanup {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Sync for SGTableCleanup {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -839,6 +870,14 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: This new faux device is private to the fixture. Use direct DMA with a + // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -885,19 +924,225 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.read8(offset), expected); + assert_eq!(vmap.try_read8(offset).unwrap(), expected); } Ok(()) } - // TODO: I would love to actually test the success paths of sg_table(), but that would require - // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave - // that for someone else. + #[test] + fn sg_lease_counting() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let second = obj.sg_table(dev.as_ref().as_ref())?; + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(second); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn release_idle_without_mappings() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes these fields. + let empty = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + ) + }; + assert!(empty.0); + assert!(empty.1); + // SAFETY: Still no device mappings or leases; an already idle release succeeds too. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn release_refused_while_leased() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no actual device mappings were installed. + assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; no actual device mappings were installed. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(bind); + // SAFETY: Reservation held; the mapping lease is now the sole owner. + assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; the remaining lease must still exclude release. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; no device mapping or SG lease remains. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn purge_madvised_after_last_lease() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no real device access. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; all device uses and SG leases have ended. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the backing fields and advice. + let (empty_sgt, empty_pages, madv) = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + obj.madv_locked(), + ) + }; + assert!(empty_sgt); + assert!(empty_pages); + assert_eq!(madv, -1); + } + assert!(!obj.madvise(0)); + assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn sg_rebuild_after_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.vmap::<4>()?.as_view().write32(0x12345678, 0); + for _ in 0..3 { + let table = obj.sg_table(dev.as_ref().as_ref())?; + assert!(table.iter().count() > 0); + drop(table); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; all leases have dropped, with no device mappings. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes the table pointer. + assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); + } + assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); + } + // Drop with a rebuilt table exercises the surviving devres cleanup registration too. + drop(obj.sg_table(dev.as_ref().as_ref())?); + Ok(()) + } + + #[test] + fn purge_after_idle_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + unsafe { obj.release_idle_pages_locked()? }; + } + assert!(obj.madvise(1)); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + assert!(unsafe { obj.unleased_pages_releasable_locked() }); + // SAFETY: Reservation held; no device access or SG lease remains. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the advice and table pointer. + let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; + assert_eq!(madv, -1); + assert!(empty); + Ok(()) + } + + #[test] + fn purge_refuses_kernel_vmap() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let vmap = obj.vmap::<4>()?; + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; the vmap must prevent the backing from being freed. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + // SAFETY: Reservation held; the vmap must exclude idle release too. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + } + drop(vmap); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device, vmap or SG users remain. + unsafe { obj.purge_locked()? }; + Ok(()) + } + + #[test] + fn sg_reclaim_excludes_new_leases() -> Result { + let (_dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); + obj.sgt_access.store(SGT_USERS, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); + obj.sgt_access.store(0, Ordering::Release); + Ok(()) + } + + #[test] + fn sg_import_attachment_is_not_owned() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mut table = MaybeUninit::::zeroed(); + let mut attachment = MaybeUninit::::zeroed(); + // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay + // live until the lease drops and are detached before the real object destructor runs. + unsafe { + (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); + (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); + } + let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { + let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held. Capture results before restoring the fixture, so a + // failed assertion cannot leave a mock attachment for the GEM destructor. + unsafe { + let held = obj.release_idle_pages_locked(); + drop(lease); + ( + no_cleanup, + held, + obj.release_idle_pages_locked(), + obj.purge_locked(), + (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), + ) + } + }); + // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. + unsafe { + (*obj.as_raw()).import_attach = ptr::null_mut(); + (*obj.as_raw_shmem()).sgt = ptr::null_mut(); + } + let (no_cleanup, held, unleased, purge, unchanged) = result?; + assert!(no_cleanup); + assert_eq!(held, Err(EBUSY)); + assert_eq!(unleased, Err(EINVAL)); + assert_eq!(purge, Err(EBUSY)); + assert!(unchanged); + Ok(()) + } - // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it - // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device - // bound to it. + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -908,10 +1153,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + // A failed lookup must not bind the cleanup registration to the wrong device. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // TODO: Someday, we should test that creating an sg_table here still succeeds. + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); Ok(()) } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a7..e43d5ae131c85 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,6 +151,59 @@ impl Mm { } } +/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). +/// +/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report +/// show for a process. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RssCounter { + /// Resident file-backed pages (`MM_FILEPAGES`). + File, + /// Resident anonymous pages (`MM_ANONPAGES`). + Anon, + /// Swapped-out anonymous pages (`MM_SWAPENTS`). + Swap, + /// Resident shared-memory pages (`MM_SHMEMPAGES`). + Shmem, +} + +impl RssCounter { + fn as_raw(self) -> core::ffi::c_int { + let member = match self { + Self::File => bindings::MM_FILEPAGES, + Self::Anon => bindings::MM_ANONPAGES, + Self::Swap => bindings::MM_SWAPENTS, + Self::Shmem => bindings::MM_SHMEMPAGES, + }; + member as core::ffi::c_int + } +} + +// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these +// methods are safe even if `mm_users` is zero. +impl Mm { + /// Adds `pages` (which may be negative) to one of the resident-set counters. + /// + /// This is how a driver attributes pages it holds on behalf of a process, but which are not + /// mapped into it, to that process. Every page added must be subtracted again before the + /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad + /// rss-counter state. + #[inline] + pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; + } + + /// Returns the exact current value of one of the resident-set counters. + #[inline] + pub fn rss_counter(&self, counter: RssCounter) -> usize { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } + } +} + // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer. From 3810c8bb8dda82fc049966611df523ba553f748b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:53 -0500 Subject: [PATCH 010/392] Prepare the original context for carried T8122 commit 8a31fe185686 Keep the public dependency patch unchanged. The separate 7.3 adaptation restores Aurora integration after the carried series. Signed-off-by: Satya Benson --- arch/arm64/configs/asahi.config | 35 +++++++-------------------------- drivers/gpu/drm/asahi/Kconfig | 16 +++++++++++++-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 870ff1b6b4dda..98f9a98ab9028 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,16 +8,9 @@ CONFIG_ARCH_APPLE=y # CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set -CONFIG_ARM64_VA_BITS_48=y -CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y -# 16K pages make a page block 32 MiB, and khugepaged then sets min_free_kbytes -# to 352 MiB on an 8 GB Mac. Keep page blocks at 2 MiB as on 4K kernels. -CONFIG_PAGE_BLOCK_MAX_ORDER=7 CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m -CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y -# CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE is not set CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m @@ -27,7 +20,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m @@ -38,7 +31,7 @@ CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=y +CONFIG_SPMI_APPLE=m CONFIG_PINCTRL_APPLE_GPIO=m CONFIG_GPIO_MACSMC=m CONFIG_POWER_RESET_MACSMC=m @@ -48,28 +41,19 @@ CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y CONFIG_APPLE_WATCHDOG=y -# m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). -CONFIG_MTD=y -CONFIG_MTD_PHRAM=m -# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). -CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m -CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m -CONFIG_VIDEO_APPLE_PRORES=m -CONFIG_VIDEO_APPLE_JPEG=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m -CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m -CONFIG_DRM_APPLE_DCP_T8132=y CONFIG_DRM_APPLE_AUDIO=y CONFIG_DRM_ACCEL=y CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m @@ -91,14 +75,14 @@ CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=y +CONFIG_RTC_DRV_MACSMC=m CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m -CONFIG_MFD_MACSMC=y +CONFIG_APPLE_SIO_T6030=y +CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y -CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y @@ -118,20 +102,16 @@ CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m CONFIG_APPLE_PMP_REPORT=y -CONFIG_APPLE_PMP_THERMAL=m -CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m -CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m -CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y @@ -143,4 +123,3 @@ CONFIG_RDMA_SIW=m CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y -CONFIG_DETECT_HUNG_TASK=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 4cfe26f28b891..ec2a588054e22 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -36,11 +36,21 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. +config DRM_ASAHI_UAT_ADOPTION + bool + help + The firmware UAT handoff adoption shared by the G15 backend and the + T8122 runtime (g15/uat.rs, g15/regs.rs). + config DRM_ASAHI_M3_T8122 bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" depends on DRM_ASAHI - # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. - depends on DRM_ASAHI_G15 + # The runtime adopts the firmware's UAT as the G15 backend does, without + # needing that backend: M3 Pro/Max keep their own setting. + # Its PMP votes call apple-pmp directly (m3_device.rs). + depends on APPLE_PMP=y || (APPLE_PMP=m && DRM_ASAHI=m) + select DRM_ASAHI_UAT_ADOPTION + select APPLE_T8122_GPU_PMP_LINK select CRYPTO_LIB_SHA256 default n help @@ -57,6 +67,8 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS + select DRM_ASAHI_UAT_ADOPTION default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max From 0486865371b47b9a6014b49da4f95406ddd0ee7b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:29:01 +0000 Subject: [PATCH 011/392] drm/asahi: prepare supported M3 native GPU defaults Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- arch/arm64/configs/asahi.config | 2 ++ drivers/gpu/drm/asahi/Kconfig | 9 +++---- drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 24 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/t8122_start.rs | 31 ++++++++++++++----------- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 98f9a98ab9028..ffecb85c3beb6 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -45,6 +45,8 @@ CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_M3_T8122=y +CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index ec2a588054e22..45e907b5af86d 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -56,10 +56,11 @@ config DRM_ASAHI_M3_T8122 help Build the M3 runtime that starts the GPU of the M3 MacBook Air (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves - loaded. It starts only with asahi.t8122_start=1 on the kernel - command line, after the boot loader handed the GPU over; without - it the T8122 probe is refused as before. Other GPUs are not - affected. + loaded. On the listed Air boards it starts after the boot loader + handed the GPU over and resource/firmware identity checks pass. + asahi.t8122_start=0 keeps startup disabled for recovery. Unknown + boards and firmware do not gain automatic admission. Other GPUs + are not affected. config DRM_ASAHI_G15 bool "Native G15 firmware backend (M3 Pro/Max)" diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs index b48e561f677ef..ca0f8a6d499cc 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -132,6 +132,17 @@ pub(crate) fn start(raw: &Raw) -> Start { } } +/// The shipped board policy changes only an omitted start parameter. Explicit recovery-off, +/// explicit experimental start, and malformed values retain their original meanings. +/// Hardware resources and firmware identity are admitted separately by the caller. +pub(crate) fn start_for_board(raw: &Raw, admitted_board: bool) -> Start { + if raw.start == UNSET && admitted_board { + Start::On + } else { + start(raw) + } +} + /// The GPU clock-generator IO mapping. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum ClockGen { @@ -232,6 +243,19 @@ mod tests { assert!(with(|r| r.fender = INVALID).values_given()); } + #[test] + fn automatic_start_is_board_scoped_and_explicit_values_win() { + assert_eq!(start_for_board(&Raw::NONE, true), Start::On); + assert_eq!(start_for_board(&Raw::NONE, false), Start::Off); + for admitted_board in [false, true] { + assert_eq!(start_for_board(&with(|r| r.start = 0), admitted_board), Start::Off); + assert_eq!(start_for_board(&with(|_| {}), admitted_board), Start::On); + assert_eq!(start_for_board(&with(|r| r.start = 2), admitted_board), Start::Invalid); + assert_eq!(start_for_board(&with(|r| r.start = INVALID), admitted_board), Start::Invalid); + assert_eq!(start_for_board(&with(|r| r.start = parse_number("bogus")), admitted_board), Start::Invalid); + } + } + #[test] fn defaults_are_the_plan_baseline() { let v = resolve(&with(|_| {})).unwrap(); diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index ca9f21b56dbb6..8f03a973decb8 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The T8122 (M3, G15G) start, `asahi.t8122_start=1`. +//! T8122 (M3, G15G) board-default startup and bounded experiment values. //! -//! The T8122 table (`m3_soc::T8122`) is complete, but the runtime has not passed on an M3 -//! MacBook Air yet, so the SoC is refused unless `asahi.t8122_start=1` arms the start. On a T8122 -//! whose GPU the boot loader handed over (resource admission passed), an armed start uses the -//! table's values, with a performance-state cap. Each value can be overridden for a lab A/B by +//! The T8122 table (`m3_soc::T8122`) is complete. An omitted start parameter starts +//! automatically only on its listed Air boards; explicit `asahi.t8122_start=0` +//! remains the recovery opt-out. Resource and known-firmware admission still run. +//! The same table values can be changed for a controlled experiment by //! rebooting with a parameter: //! //! | Parameter | Default | Others | @@ -19,7 +19,7 @@ //! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 | //! //! The defaults are the T8122 table's values. A parameter given with a value it does not accept -//! refuses the start. Without `asahi.t8122_start=1` nothing here runs, and T8122 is refused. +//! refuses the start. Unlisted boards remain refused by the existing backend admission. //! Nothing here applies to any other SoC. use core::mem::offset_of; @@ -237,16 +237,16 @@ pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot { /// Decide, after resource admission, whether `soc` starts with the experiment values. /// -/// Any SoC but T8122: Ok(None), silently. T8122 without `asahi.t8122_start=1`: Ok(None), so the -/// caller refuses it as before (one extra line when experiment parameters were given). T8122 -/// with `asahi.t8122_start=1`: the experiment, after logging every value, or a refusal for a -/// parameter value it does not accept. +/// Any SoC but T8122: Ok(None). On its listed Air boards, an omitted parameter +/// selects the table values; explicit 0 disables startup, and invalid values fail +/// closed. Existing backend admission still refuses an unlisted board, even +/// with an explicit start request. pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> { if !is_t8122(soc) { return Ok(None); } let raw = crate::m3::m3_params::t8122_params(); - match knobs::start(&raw) { + match knobs::start_for_board(&raw, crate::m3::m3_board::runtime_validated_board(soc)) { Start::On => {} Start::Off => { if raw.values_given() { @@ -283,7 +283,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> let e = Experiment(v); dev_warn!( dev, - "M3 G15G start: armed (asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", + "M3 G15G start: armed (board default or asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", v.initdata_version, v.fender, clkgen_label(v.clock_gen, v.fender), @@ -352,9 +352,12 @@ pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, sta } } -/// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. +/// Log that resource admission refused a T8122 start selected by board default or flag. pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { - if is_t8122(soc) && knobs::start(&crate::m3::m3_params::t8122_params()) == Start::On { + if is_t8122(soc) && knobs::start_for_board( + &crate::m3::m3_params::t8122_params(), + crate::m3::m3_board::runtime_validated_board(soc), + ) == Start::On { dev_err!( dev, "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" From 6b505f46bda1dad14a03f202f006a66b416c1eeb Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 20:36:35 -0400 Subject: [PATCH 012/392] drm/asahi: m3: start the GPU firmware once per boot A second start of the T8122 runtime in one boot (unbind and rebind, a module reload, or a retry after a failed start) hands the firmware freed tables. Runtime::drop and the failed-start arm free the adopted lower root once ASC has stopped, but TTBAT context 0 still names it, and the next start sets RUN before adopt_uat rewrites context 0. stop_asc only clears RUN, so there is no firmware state to restart from anyway. Refuse every start after the first one of the boot instead. The latch is claimed just before the first ASC start and is never released, and claiming it pins the module, so a reload can't forget it. An unbind and rebind now ends in a clean refusal rather than a second firmware boot, and a reboot is the only way to start the GPU again. Pinning the module also means it can't be unloaded right after an unbind while a completion work item is still queued, which was the reason for draining that work in Backend::stop. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 11 +++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 11 +++++++++++ tools/asahi/m3-batching-tests.rs | 13 +++++++++++++ 3 files changed, 35 insertions(+) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 851b4e79f421e..e7a5d6ad3eecb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -87,6 +87,17 @@ pub(crate) const fn retain_runtime(stop_failed: bool, gpu_pending: bool) -> bool stop_failed || gpu_pending } +/// One firmware start per boot. `stop_asc` only clears RUN: it doesn't reset the firmware or +/// give back a reusable RTKit or InitData state, and TTBAT context 0 still names the lower root +/// the first start adopted, which is freed once ASC has stopped. Claimed just before the first +/// ASC start and never released. +pub(crate) struct StartLatch(core::sync::atomic::AtomicBool); +impl StartLatch { + pub(crate) const fn new() -> Self { Self(core::sync::atomic::AtomicBool::new(false)) } + /// True for the first claim only. + pub(crate) fn claim(&self) -> bool { !self.0.swap(true, core::sync::atomic::Ordering::AcqRel) } +} + #[derive(Default)] pub(crate) struct PassCosts { recent: [Option<(u64, u64)>; 8] } impl PassCosts { diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 28bbea3c86889..234678e9aa2cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -974,6 +974,8 @@ impl Inner { // Registration and Mutex initialization otherwise duplicates it on the bounded // kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop. pub(crate) struct Runtime { inner: ManuallyDrop> } +/// Claimed before this boot's first ASC start. +static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); impl Runtime { #[inline(never)] pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { @@ -995,6 +997,15 @@ impl Runtime { if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) { return Err(ENODEV); } + // A second start (unbind and rebind, a module reload, a retry after a failed start) + // would set RUN while TTBAT context 0 still names the freed root of this one. Unloading + // is blocked once the latch is claimed, so a reload can't forget it. + if !FIRMWARE_START.claim() { + dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); + return Err(EBUSY); + } + // SAFETY: never released: the latch must outlive this instance of the module. + unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; // The adopted UAT and its window live outside the closure: once TTBAT context 0 names diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index a6eb52bbf4fe7..f7db83f75862c 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -163,3 +163,16 @@ fn experimental_air_is_active_while_pro_retains_serial_default() { assert!(default_depth(true)<=SLOTS); assert_eq!(MAX_PACKETS,8); } + +#[test] +fn firmware_starts_once_per_boot() { + let latch=StartLatch::new(); + assert!(latch.claim()); + // Unbind and rebind, a reload or a retry: every later start is refused. + for _ in 0..3 {assert!(!latch.claim());} + // Concurrent probes: exactly one claims it. + let latch=std::sync::Arc::new(StartLatch::new()); + let claims:usize=(0..8).map(|_|{let l=latch.clone();std::thread::spawn(move||l.claim())}) + .collect::>().into_iter().map(|t|usize::from(t.join().unwrap())).sum(); + assert_eq!(claims,1); +} From 00311a5359da66cfd5c20c132b511adbf4dcc9bd Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 20:37:29 -0400 Subject: [PATCH 013/392] drm/asahi: m3: don't quarantine VMs for packets the firmware never saw Since a failed job's VM is quarantined before its fence signals, Inner::fail_pending quarantines every pending packet's VM, including packets the firmware never saw: it finishes them without releasing their job guards. A check against the active batches can't tell them apart, because a pending packet can be partly published and fail_active drains the active batches before it calls fail_pending. Give each packet a flag that is set before the first firmware-visible change of any batch that includes it: before the producer state of a batch changes, so partial ring appends that overlapping firmware may consume count as published. fail_pending releases the guard only for a packet whose flag is unset. Everything else keeps quarantining as before, including on ECANCELED: cancellation only withdraws a packet's unpublished remainder, and its published commands may still be running. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 6 ++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 14 ++++++++++++-- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 ++++++++- tools/asahi/m3-batching-tests.rs | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index e7a5d6ad3eecb..0d8cf9056d44f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -83,6 +83,12 @@ pub(crate) const fn packet_retired(pending: bool, active: bool) -> bool { !pending && !active } +/// A failed packet that no batch ever included can drop its VM job guard: the firmware never saw +/// it. Any other keeps the guard, and its VM stays quarantined, until the runtime drops it. +pub(crate) const fn release_unpublished(may_be_published: bool) -> bool { + !may_be_published +} + pub(crate) const fn retain_runtime(stop_failed: bool, gpu_pending: bool) -> bool { stop_failed || gpu_pending } diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 234678e9aa2cd..aeb483cce866d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -169,9 +169,15 @@ impl Inner { } } impl Inner { - /// Fail every packet that was never published. Their fences signal with the error. + /// Fail every packet not yet fully published. Their fences signal with the error. One that no + /// batch ever included releases its VM job guard first, so its VM isn't quarantined for work + /// the firmware never saw. One that may have been published keeps it: a partly published + /// packet, or one whose batch `fail_active` has already drained from `active`. fn fail_pending(&mut self, error: Error) { - for (packet,_) in self.pending.drain_all() {packet.finish(Err(error));} + for (packet,_) in self.pending.drain_all() { + if policy::release_unpublished(packet.may_be_published()) {packet.release_guard();} + packet.finish(Err(error)); + } } /// A published batch failed or never retired: latch the fault, log it, and fail its /// packets. The packets stay retained (with their VM job guards) until teardown, because @@ -374,6 +380,9 @@ impl Inner { for (p,_) in entries { if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} } + // Before any producer state changes: an append the overlapping firmware consumes, or a + // partial publication, makes these packets visible to it. + for (p,_) in entries {p.mark_published();} let found=self.jobs.iter().position(|job| matches!((job,control), (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) | (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..}))); @@ -519,6 +528,7 @@ impl Inner { // Keep the packet before any producer mutation. No other batch is live. self.packets.clear(); self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + packet.mark_published(); let mut render_prepared=false; let mut compute_prepared=false; let prepared=(||->Result { diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ca16e18f10fbf..ffc57c77afcb2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -43,6 +43,9 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, + /// Set before a batch with any of this packet's commands can change firmware-visible state, + /// and never cleared. + published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -78,6 +81,10 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + /// Called before the first firmware-visible change of a batch that includes this packet. + pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Whether the firmware may have seen any of this packet's commands. + pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -232,7 +239,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index f7db83f75862c..2812989e51f9e 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -176,3 +176,17 @@ fn firmware_starts_once_per_boot() { .collect::>().into_iter().map(|t|usize::from(t.join().unwrap())).sum(); assert_eq!(claims,1); } + +#[test] +fn failed_pending_packets_quarantine_only_what_the_firmware_may_have_seen() { + // (in a batch fail_active drained, partly published and still pending, never in a batch) + struct Packet {published:bool,guard:bool} + let mut active=vec![Packet{published:true,guard:true}]; + let mut pending=vec![Packet{published:true,guard:true},Packet{published:false,guard:true}]; + // fail_active drains active before fail_pending: a check against active can't see these. + let drained:Vec<_>=active.drain(..).collect(); + for p in pending.iter_mut() {if release_unpublished(p.published) {p.guard=false;}} + assert!(drained[0].guard); + assert!(pending[0].guard,"a partly published packet keeps its guard"); + assert!(!pending[1].guard,"a never-published packet releases its guard"); +} From c297e5a440103fdac5a658f6b4e650e9e140e71d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:16:02 +0000 Subject: [PATCH 014/392] drm/asahi: align M3 Air startup diagnostics with board defaults Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/m3/m3_soc.rs | 9 ++++----- drivers/gpu/drm/asahi/m3/t8122_start.rs | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index 7be649415cccc..4b81475612b6f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -272,10 +272,9 @@ pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> { /// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20, /// core slots in the first core-mask word only), with the die count of the AGX3 identification /// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration -/// comes from the boot loader. The rest of the runtime configuration is complete, but until the -/// runtime has passed on an M3 MacBook Air the SoC is refused, before the GPU is accessed, unless -/// `asahi.t8122_start=1` arms the start (`t8122_start`), which uses this table's values and adds a -/// performance-state cap. +/// comes from the boot loader. Startup uses the guarded `t8122_start` path, which selects this +/// table's values by default on admitted Air boards and adds a performance-state cap. Explicit +/// `asahi.t8122_start=0` disables startup; firmware, board and resource admission still apply. pub(crate) static T8122: Soc = Soc { name: "T8122", gpu_name: "G15G", @@ -326,7 +325,7 @@ pub(crate) static T8122: Soc = Soc { }, power_from_boot_loader: true, unported: &[ - "a pass on an M3 MacBook Air: until then the GPU starts only with asahi.t8122_start=1", + "M3 Air startup requires the guarded t8122_start path and its performance-state cap", ], // The performance state only. On the single-cluster G15G, reading the engine-busy or the // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s; diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index 8f03a973decb8..737034dd6f8c1 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -252,7 +252,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> if raw.values_given() { dev_info!( dev, - "M3 G15G start: asahi.t8122_* values given without asahi.t8122_start=1; ignored\n" + "M3 G15G start: disabled by asahi.t8122_start=0; other asahi.t8122_* values ignored\n" ); } return Ok(None); From 6cdb981acb5ba801c2891861648005862cf829ec Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 22:37:22 -0400 Subject: [PATCH 015/392] drm/asahi: m3: take the start pin with try_module_get The one-start-per-boot latch pinned the module with __module_get after claiming it. A first probe that isn't the module-init one (deferred, async or a sysfs bind) can run while an ordinary rmmod is under way: once try_stop_module has dropped the reference and marked the module GOING, __module_get revives the count unconditionally, the probe starts ASC, and the module still unloads. The next load then gets a fresh latch and starts the firmware a second time. Take the reference with try_module_get before claiming the latch, and start nothing when the module is already being removed. When another start already holds the latch, drop only the reference this probe just took; the winning claim keeps its reference, so an ordinary unload stays blocked. A forced unload, or a module init that fails after the probe started ASC, still frees the module regardless of references; neither is new here. The order lives in the batch policy (claim_start), so the host test checks it: a pin that fails claims nothing, a start that loses the claim gives back only its own reference, and the winner keeps its reference. Reported-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 25 +++++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 39 +++++++++++++++++---- tools/asahi/m3-batching-tests.rs | 21 +++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 0d8cf9056d44f..27295660640ae 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -104,6 +104,31 @@ impl StartLatch { pub(crate) fn claim(&self) -> bool { !self.0.swap(true, core::sync::atomic::Ordering::AcqRel) } } +/// Why a start was refused before ASC. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum StartRefusal { + /// The module is being removed: nothing is claimed or started. + Removing, + /// This boot's start already happened: the extra module reference is dropped again. + AlreadyStarted, +} + +/// Pins the module, then claims `latch`. The pin comes first, so a probe racing an ordinary +/// rmmod fails closed instead of reviving the module's reference count. A start that loses the +/// claim drops only its own pin; the winner keeps its pin for good, so a reload can't forget the +/// latch. +pub(crate) fn claim_start(latch: &StartLatch, pin: impl FnOnce() -> bool, unpin: impl FnOnce()) + -> Result<(), StartRefusal> { + if !pin() { + return Err(StartRefusal::Removing); + } + if !latch.claim() { + unpin(); + return Err(StartRefusal::AlreadyStarted); + } + Ok(()) +} + #[derive(Default)] pub(crate) struct PassCosts { recent: [Option<(u64, u64)>; 8] } impl PassCosts { diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index aeb483cce866d..1a366d7f4b340 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -986,6 +986,24 @@ impl Inner { pub(crate) struct Runtime { inner: ManuallyDrop> } /// Claimed before this boot's first ASC start. static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); +/// Takes a module reference, unless the module is already being removed. +#[cfg(CONFIG_MODULE_UNLOAD)] +fn pin_module() -> bool { + // SAFETY: THIS_MODULE is this module, whose code is running; try_module_get fails once + // removal has begun instead of reviving the reference count. + unsafe { kernel::bindings::try_module_get(crate::THIS_MODULE.as_ptr()) } +} +/// Without module unloading there is no removal to race. +#[cfg(not(CONFIG_MODULE_UNLOAD))] +fn pin_module() -> bool { true } +/// Drops a reference [`pin_module`] took. +#[cfg(CONFIG_MODULE_UNLOAD)] +fn unpin_module() { + // SAFETY: called only to drop a reference pin_module took. + unsafe { kernel::bindings::module_put(crate::THIS_MODULE.as_ptr()) }; +} +#[cfg(not(CONFIG_MODULE_UNLOAD))] +fn unpin_module() {} impl Runtime { #[inline(never)] pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { @@ -1008,14 +1026,21 @@ impl Runtime { return Err(ENODEV); } // A second start (unbind and rebind, a module reload, a retry after a failed start) - // would set RUN while TTBAT context 0 still names the freed root of this one. Unloading - // is blocked once the latch is claimed, so a reload can't forget it. - if !FIRMWARE_START.claim() { - dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); - return Err(EBUSY); + // would set RUN while TTBAT context 0 still names the freed root of this one. An ordinary + // (not forced) unload is blocked once the latch is claimed, so a reload can't forget it. + // The pin comes first: a later probe can race an ordinary rmmod, and once removal has + // begun nothing starts. + match policy::claim_start(&FIRMWARE_START, pin_module, unpin_module) { + Ok(()) => {} + Err(policy::StartRefusal::Removing) => { + dev_err!(pdev.as_ref(), "M3: the module is being removed; GPU not started\n"); + return Err(ENODEV); + } + Err(policy::StartRefusal::AlreadyStarted) => { + dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); + return Err(EBUSY); + } } - // SAFETY: never released: the latch must outlive this instance of the module. - unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; // The adopted UAT and its window live outside the closure: once TTBAT context 0 names diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index 2812989e51f9e..d9a0a35c04dd0 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -190,3 +190,24 @@ fn failed_pending_packets_quarantine_only_what_the_firmware_may_have_seen() { assert!(pending[0].guard,"a partly published packet keeps its guard"); assert!(!pending[1].guard,"a never-published packet releases its guard"); } + +#[test] +fn the_start_pins_the_module_before_claiming_and_fails_closed() { + use std::cell::Cell; + let latch=StartLatch::new(); + let refs=Cell::new(1i32); // the module's base reference + let pin_ok=||{refs.set(refs.get()+1);true}; + let unpin=||refs.set(refs.get()-1); + // rmmod has begun: no pin, no claim, no start, and the reference count is untouched. + assert_eq!(claim_start(&latch,||false,unpin),Err(StartRefusal::Removing)); + assert_eq!(refs.get(),1); + // The first live start wins and keeps its pin. + assert_eq!(claim_start(&latch,pin_ok,unpin),Ok(())); + assert_eq!(refs.get(),2); + // A later start (rebind, retry) loses the claim and drops only its own pin. + assert_eq!(claim_start(&latch,pin_ok,unpin),Err(StartRefusal::AlreadyStarted)); + assert_eq!(refs.get(),2); + // A later start racing rmmod still claims nothing. + assert_eq!(claim_start(&latch,||false,||panic!("nothing to unpin")),Err(StartRefusal::Removing)); + assert_eq!(refs.get(),2); +} From ba17f5af0ba09970e4d3e33119ad19d93eaaa3f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 08:00:01 -0400 Subject: [PATCH 016/392] drm/asahi: m3: admit clients that ask for the parameters up to the HAL generation The M3 runtime admits a VM only for a file that asked GET_PARAMS for at least AGX3_PARAMS_SIZE bytes, and that was the size of drm_asahi_params_global. The G17 update grows the structure from 600 to 616 bytes with two optional tail fields, memory_limit and client_memory_limit, so Mesa built against the 600-byte structure would be refused VM creation with EOPNOTSUPP until it is rebuilt. The check only needs to know that the client knows the AGX3 fields. Pin the minimum at the end of gpu_hal_generation, 600 bytes, and assert that value at build time. GET_PARAMS still copies only the size the client asked for. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/m3/m3_client.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 8fc7791e939ba..dff1af0783965 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -5,11 +5,11 @@ //! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for //! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 //! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client -//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at -//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks -//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore -//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a -//! VM, so a refused client can still query the parameters but never reaches the hardware. +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields +//! of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks GET_PARAMS +//! for at least the structure up to them; older userspace asks for less. On G15 a file therefore +//! has to ask for those fields before it can create a VM. Every path to the GPU needs a VM, so a +//! refused client can still query the parameters but never reaches the hardware. use core::sync::atomic::{ AtomicU32, @@ -29,15 +29,20 @@ use crate::{ mmu, }; -/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. -const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; +/// GET_PARAMS size an AGX3-aware client asks for: the global parameter structure up to the end +/// of `gpu_hal_generation`. The optional tail fields that follow it are not part of the check, +/// so growing the structure does not refuse userspace built against the shorter one. +const AGX3_PARAMS_SIZE: u64 = + (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + + core::mem::size_of::()) as u64; +const _: () = assert!(AGX3_PARAMS_SIZE == 600); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; static REFUSALS: AtomicU32 = AtomicU32::new(0); -/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the -/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for at least +/// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { { From bdb0b149e156dea2a9dcf7474890e16a596f126c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Bro=C5=84ski?= Date: Fri, 30 Oct 2026 00:22:24 +0100 Subject: [PATCH 017/392] soc: apple: sep: Open the host-state store only for each operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store::open() opens /var/lib/aurora-sep-host-state.bin read-write, and the store keeps the file open until the driver goes away, in practice until power-off. The driver opens it from the kernel, so no task holds it, and killing every process at shutdown does not close it. The file system it lives on can then be neither unmounted nor remounted read-only: every poweroff logs Failed to unmount /persist: Device or resource busy and the file system stays mounted read-write until the machine powers off. This hits any setup where the store is not on the root file system, such as an impermanent root that keeps /var/lib state on a /persist file system through a symlink. Keep the slot table in memory as before, and open the file only to load it, to read a record and for each write, closing it when the operation returns. The write sequence and the on-disk format do not change. The keybag, ref-key and catacomb files are already opened per use, and the xART store is a block device under /dev. Store is now generic over a StoreSource that opens the backing file. An open StoreIo that is Copy is its own source, so the KUnit tests keep using their in-memory files unchanged. Fixes: 24d20c9828b8 ("soc: apple: add Apple SEP driver (Touch ID and key sealing)") Assisted-by: LLM Signed-off-by: Andrzej BroÅ„ski --- drivers/soc/apple/store.rs | 100 ++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index e3abe11c7d5c8..f5ee54b375335 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -131,9 +131,42 @@ impl StoreIo for shim::StoreFile { } } +/// Where a [`Store`] reaches its backing file. Every operation opens it and +/// closes it again before returning. +pub(crate) trait StoreSource { + type Io: StoreIo; + fn open(&self) -> Result; +} + +/// A file that is already open is its own source, as the tests' in-memory +/// files are. +impl StoreSource for T { + type Io = T; + + fn open(&self) -> Result { + Ok(*self) + } +} + +/// The host-state file at [`STORE_PATH`]. +pub(crate) struct HostFile; + +impl StoreSource for HostFile { + type Io = shim::StoreFile; + + fn open(&self) -> Result { + shim::StoreFile::open_host(STORE_PATH).map(|(file, _)| file) + } +} + // Not internally locked; the caller holds a mutex around all access. -pub(crate) struct Store { - file: F, +// +// The store keeps its table in memory and holds no open file between +// operations. The driver opens the file from the kernel, so no task owns it: +// held open for writing, it would pin its file system until power-off, which +// could then be neither unmounted nor remounted read-only. +pub(crate) struct Store { + source: S, slots: [Slot; SLOT_COUNT], generation: u64, } @@ -242,14 +275,21 @@ fn decode(sb: &[u8], slots: &mut [Slot; SLOT_COUNT]) -> Result<(), Damage> { impl Store { pub(crate) fn open() -> Result { let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; - Self::from_file(file, created) + Self::load(HostFile, &file, created) } } -impl Store { - fn from_file(file: F, created: bool) -> Result { +impl Store { + #[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] + fn from_file(source: S, created: bool) -> Result { + let file = source.open()?; + Self::load(source, &file, created) + } + + /// Loads the store from `file`, the open backing file of `source`. + fn load(source: S, file: &S::Io, created: bool) -> Result { let mut store = Store { - file, + source, slots: [Slot::FREE; SLOT_COUNT], generation: 0, }; @@ -257,9 +297,9 @@ impl Store { // New files and interrupted empty or all-zero creations hold no state. // Any other existing file may hold the only copy of the host state, // so one that does not validate is refused and left as it is. - let size = store.file.size()?; + let size = file.size()?; if created && size == 0 { - store.create()?; + store.create(file)?; pr_info!("created a new host-state store\n"); return Ok(store); } @@ -267,19 +307,19 @@ impl Store { let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; let decoded = if size == STORE_SIZE as u64 { - store.file.read_exact(0, &mut sb)?; + file.read_exact(0, &mut sb)?; decode(&sb, &mut store.slots) } else { Err(Damage::Size) }; if let Err(damage) = decoded { if matches!(damage, Damage::Size | Damage::Header) { - if let Some(what) = describe(&store.file, size) { + if let Some(what) = describe(file, size) { pr_warn!( "host-state store file is {} and holds no state; initialising it\n", what ); - store.create()?; + store.create(file)?; return Ok(store); } } @@ -303,26 +343,26 @@ impl Store { EIO } - fn create(&mut self) -> Result<()> { + fn create(&mut self, file: &S::Io) -> Result<()> { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; for b in 0..BLOCK_COUNT { - self.file.write_all((b * BLOCK_SIZE) as u64, &zero)?; + file.write_all((b * BLOCK_SIZE) as u64, &zero)?; } - self.commit()?; + self.commit(file)?; Ok(()) } - fn commit(&mut self) -> Result<()> { - self.write_superblock(Intent::None)?; - self.file.sync() + fn commit(&mut self, file: &S::Io) -> Result<()> { + self.write_superblock(file, Intent::None)?; + file.sync() } - fn write_superblock(&mut self, intent: Intent) -> Result<()> { + fn write_superblock(&mut self, file: &S::Io, intent: Intent) -> Result<()> { let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -348,7 +388,7 @@ impl Store { sb[off + 18..off + 22].copy_from_slice(&slot.len.to_le_bytes()); } - self.file.write_all(0, &sb) + file.write_all(0, &sb) } fn find(&self, key: &Key) -> Option { @@ -374,7 +414,7 @@ impl Store { let mut value = KVec::with_capacity(len, GFP_KERNEL)?; value.resize(len, 0, GFP_KERNEL)?; if len > 0 { - self.file.read_exact(Self::block_offset(idx), &mut value)?; + self.source.open()?.read_exact(Self::block_offset(idx), &mut value)?; } Ok(Some(value)) } @@ -389,19 +429,23 @@ impl Store { let old = self.find(key); let fresh = self.find_free().ok_or(ENOSPC)?; + let file = self.source.open()?; // 1. Record the intent, durably, before touching any data block. - self.write_superblock(Intent::Write { - slot: fresh as u16, - kind: key.kind, - })?; - self.file.sync()?; + self.write_superblock( + &file, + Intent::Write { + slot: fresh as u16, + kind: key.kind, + }, + )?; + file.sync()?; // 2. Fill the new block and make it durable. if !value.is_empty() { - self.file.write_all(Self::block_offset(fresh), value)?; + file.write_all(Self::block_offset(fresh), value)?; } - self.file.sync()?; + file.sync()?; // 3. Switch the slot table, release the old block, clear the intent. self.slots[fresh] = Slot { @@ -414,7 +458,7 @@ impl Store { self.slots[old] = Slot::FREE; } self.generation = self.generation.wrapping_add(1); - self.commit() + self.commit(&file) } } From cf772cf8e32222ebe58940a7b9282b606b346eb5 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 2 Nov 2026 00:48:57 -0500 Subject: [PATCH 018/392] iommu/apple-dart-apif: Park a released device off its domain The driver provided neither a release_domain nor a translation change in release_device(), so a removed device left its SIDs enabled on the domain that the core frees next. When the same device came back, as the N1 Wi-Fi and Bluetooth functions do after a firmware reset removes and rescans them, its new domain was attached over that stale state, and the first DMA through it froze the machine. Provide a blocking release_domain. Its attach takes the released device's SIDs out of the domain it leaves the same way an attach unwind does: disable each SID, undo the domain's mappings on it and clear it from the domain. Retained SIDs keep their current handling. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/iommu/apple-dart-apif.c | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/drivers/iommu/apple-dart-apif.c b/drivers/iommu/apple-dart-apif.c index 7c1519c70e3bd..8b5792c1efd7e 100644 --- a/drivers/iommu/apple-dart-apif.c +++ b/drivers/iommu/apple-dart-apif.c @@ -1548,6 +1548,66 @@ apple_dart_apif_domain_alloc_paging(struct device *dev) return &dom->domain; } +/* + * The core attaches this before release_device(), so a released device no + * longer uses the domain that is about to be freed. Its SIDs leave that + * domain disabled, with the domain's mappings undone, as an attach unwind + * does. Without it the SIDs stayed enabled on the freed domain's tables, and + * the next attach of the same SIDs met that stale state. Retained SIDs keep + * their current handling. + */ +static int apple_dart_apif_release_attach(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + struct apple_dart_apif_domain *prior; + unsigned long flags; + unsigned int sid; + + if (!master || !old || old->owner != &apple_dart_apif_ops || + old->ops != apple_dart_apif_ops.default_domain_ops) + return 0; + prior = to_apif_domain(old); + mutex_lock(&prior->lock); + write_lock_irqsave(&prior->streams_lock, flags); + for (unsigned int i = 0; i < master->num_darts; i++) { + struct apple_dart_apif *dart = master->streams[i].dart; + struct apif_stream_map *before = apif_find_stream(prior, dart); + + if (!before) + continue; + spin_lock(&dart->batch_lock); + for_each_set_bit(sid, master->streams[i].sids, + dart->num_streams) { + if (dart->retained[sid] || !test_bit(sid, before->sids)) + continue; + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ + .op = APIF_DART_DISABLE, + .arg = { sid }, + }; + if (apif_ring(dart)) + dev_err(dart->dev, + "failed to disable SID %u at release\n", + sid); + apif_undo_replay(prior, before, sid, U64_MAX); + clear_bit(sid, before->sids); + } + spin_unlock(&dart->batch_lock); + } + write_unlock_irqrestore(&prior->streams_lock, flags); + mutex_unlock(&prior->lock); + return 0; +} + +static struct iommu_domain apple_dart_apif_release_domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &(const struct iommu_domain_ops){ + .attach_dev = apple_dart_apif_release_attach, + }, +}; + static struct iommu_device *apple_dart_apif_probe_device(struct device *dev) { struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); @@ -1808,6 +1868,7 @@ static const struct iommu_ops apple_dart_apif_ops = { .domain_alloc_paging = apple_dart_apif_domain_alloc_paging, .probe_device = apple_dart_apif_probe_device, .release_device = apple_dart_apif_release_device, + .release_domain = &apple_dart_apif_release_domain, .device_group = apple_dart_apif_device_group, .of_xlate = apple_dart_apif_of_xlate, .owner = THIS_MODULE, From c0e42a1852a6dbd1a31b3321dbe1c0c6827a8d09 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 19:47:02 -0800 Subject: [PATCH 019/392] phy: apple: dptx: Add the T6050 variant The M5 Ultra's lpdptx PHY has the T602x bank layout: core plus dptx, with the PLL command at dptx+0x100c. Its PLL calibration, clocking, rate programming, AUX impedance calibration and PHY power follow the newer blocks. The T6020 sequence replays captured register words, and on this PHY it would program the wrong fields. Give T6050 its own sequence: - activate: core power for PLL, AUX and PHY (1 us apart); source select from the crossbar's ufp endpoint; the board's PLL calibration (DCOCFG split across 0x2240[2:1] and [6:4], DTCVREG, DTCCAL, DCDADJ, DLFCFG, optional RODCOROLEAKCOMP) and fixed settings; PLL setup with the pixel-clock driver selected by pclkdrv-select and single-ended clock out; lanes released; AUX up; impedance calibration, with its result copied into 0x4228. - link reconfiguration, as the DCP brackets it: PHY down, PLL down, lane count, rate, then PLL up and PHY up. A per-rate PLL record gives the divider and analog fields for 1.62 to 8.1 Gbit/s. PLL commands wait 100 us, then check the acknowledgment (0x1014[0]) and the lock (0x1008[3]) separately. - deactivate: the reverse. Every write is a field update. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 [erik: dropped the bring-up debugfs interface] Signed-off-by: Erik Bourget --- drivers/phy/apple/dptx.c | 433 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 433 insertions(+) diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c index 5e58f149ce96a..f191efe4c6286 100644 --- a/drivers/phy/apple/dptx.c +++ b/drivers/phy/apple/dptx.c @@ -16,11 +16,14 @@ #include #include "linux/of.h" #include +#include +#include #include #include #include #include #include +#include #include #define DPTX_MAX_LANES 4 @@ -31,6 +34,23 @@ enum apple_dptx_type { DPTX_PHY_T8112, DPTX_PHY_T6020, + DPTX_PHY_T6050, +}; + +/* The PLL core calibration the board describes, in t6050_pll_core_cfg() order. */ +enum { + T6050_DCOCFG, + T6050_DTCVREG, + T6050_DTCCAL, + T6050_DCDADJ, + T6050_DLFCFG, + T6050_RODCOROLEAKCOMP, + T6050_NR_CAL, +}; + +static const char *const t6050_cal_props[T6050_NR_CAL] = { + "apple,pll-dcocfg", "apple,pll-dtcvreg", "apple,pll-dtccal", + "apple,pll-dcdadj", "apple,pll-dlfcfg", "apple,pll-rodcoroleakcomp", }; struct apple_dptx_phy_hw { @@ -49,6 +69,14 @@ struct apple_dptx_phy { void __iomem *dptx; } regs; + /* T6050 */ + u32 cal[T6050_NR_CAL]; + bool cal_set[T6050_NR_CAL]; + u32 source_select; + bool pclkdrv_select; + bool active; + u32 rate; /* Mbit/s; 0: none programmed */ + struct phy *phy_dp; struct phy_provider *phy_provider; @@ -523,10 +551,398 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) return 0; } +/* + * T6050 (M5 Ultra) lpdptx. The two banks are T602x's ("core" and "dptx", the + * PLL command register at dptx+0x100c), but the PLL calibration, clocking, + * rate programming, AUX calibration and PHY power are T8122/T8132's. Every + * write is a field update; nothing below replays a captured register word. + */ +static void t6050_rmw(void __iomem *base, u32 off, u32 shift, u32 mask, u32 val) +{ + mask32(base + off, mask, (val << shift) & mask); +} + +#define T6050_D(phy, off, shift, mask, val) \ + t6050_rmw((phy)->regs.dptx, off, shift, mask, val) +#define T6050_C(phy, off, shift, mask, val) \ + t6050_rmw((phy)->regs.core, off, shift, mask, val) + +/* core+0x48: PLL (bits 6:4), AUX (10:8) and PHY (2:0) power, 1 us apart. */ +static void t6050_block_power(struct apple_dptx_phy *phy, u32 lsb, bool on) +{ + if (on) { + T6050_C(phy, 0x48, lsb, BIT(lsb), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 1, BIT(lsb + 1), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 2, BIT(lsb + 2), 0); + udelay(1); + } else { + T6050_C(phy, 0x48, lsb + 2, BIT(lsb + 2), 1); + udelay(1); + T6050_C(phy, 0x48, lsb + 1, BIT(lsb + 1), 0); + udelay(1); + T6050_C(phy, 0x48, lsb, BIT(lsb), 0); + udelay(1); + } +} + +static void t6050_device_power(struct apple_dptx_phy *phy, bool on) +{ + T6050_C(phy, 0x10, 0, 0x7, phy->source_select); + t6050_block_power(phy, 4, on); /* PLL */ + t6050_block_power(phy, 8, on); /* AUX */ + t6050_block_power(phy, 0, on); /* PHY */ +} + +/* The board's PLL core calibration (an absent value leaves its field), then fixed settings. */ +static void t6050_pll_core_cfg(struct apple_dptx_phy *phy) +{ + const u32 *v = phy->cal; + const bool *set = phy->cal_set; + + if (set[T6050_DCOCFG]) { + T6050_D(phy, 0x2240, 1, 0x6, (v[T6050_DCOCFG] >> 3) & 3); + T6050_D(phy, 0x2240, 4, 0x70, v[T6050_DCOCFG] & 7); + } + if (set[T6050_DTCVREG]) + T6050_D(phy, 0x2230, 0, 0x1f, v[T6050_DTCVREG]); + if (set[T6050_DTCCAL]) + T6050_D(phy, 0x22e8, 17, 0x60000, v[T6050_DTCCAL]); + if (set[T6050_DCDADJ]) + T6050_D(phy, 0x2240, 22, 0x7c00000, v[T6050_DCDADJ]); + if (set[T6050_DLFCFG]) + T6050_D(phy, 0x22b0, 20, 0x700000, v[T6050_DLFCFG]); + if (set[T6050_RODCOROLEAKCOMP]) + T6050_D(phy, 0x2240, 19, 0x180000, v[T6050_RODCOROLEAKCOMP]); + + T6050_D(phy, 0x22e8, 0, 0x1, 1); + T6050_D(phy, 0x2240, 3, 0x8, 1); + T6050_D(phy, 0x2240, 0, 0x1, 1); + T6050_D(phy, 0x2224, 10, 0xc00, 2); + T6050_D(phy, 0x22b0, 27, 0x08000000, 0); + T6050_D(phy, 0x2338, 23, 0x0f800000, 0xf); +} + +/* The pixel clock driver: dptx+0x2200 bit 3 with pclkdrv-select, else bit 1. */ +static bool t6050_pclk(struct apple_dptx_phy *phy, bool on) +{ + u32 bit = phy->pclkdrv_select ? 3 : 1; + + if (!on) { + if (!(readl(phy->regs.dptx + 0x2200) & BIT(bit))) + return false; + } + T6050_D(phy, 0x2200, bit, BIT(bit), on); + return true; +} + +static void t6050_configure_pll(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x2218, 0, 0x1, 1); + T6050_D(phy, 0x2234, 0, 0x3, 0); + t6050_pclk(phy, true); + T6050_D(phy, 0x2320, 1, 0x6, 1); /* single-ended clock output */ +} + +/* AUX impedance calibration: start, sample completion once, copy the result. */ +static int t6050_impedance_calibration(struct apple_dptx_phy *phy) +{ + u32 done, code; + + T6050_D(phy, 0x4224, 0, 0x1, 0); + usleep_range(1000, 1200); + T6050_D(phy, 0x4224, 0, 0x1, 1); + udelay(1); + T6050_D(phy, 0x4200, 4, 0x3f0, 0x3c); + udelay(1); + T6050_D(phy, 0x4200, 0, 0x1, 1); + udelay(125); + done = readl(phy->regs.dptx + 0x4204) & 1; + udelay(10); + T6050_D(phy, 0x4200, 0, 0x1, 0); + code = (readl(phy->regs.dptx + 0x4210) >> 18) & 0x7f; + if (!done) { + dev_err(phy->dev, "impedance calibration did not complete (0x4204 %#x)\n", + readl(phy->regs.dptx + 0x4204)); + return -ETIMEDOUT; + } + T6050_D(phy, 0x4228, 16, 0x7f0000, code); + T6050_D(phy, 0x4228, 15, 0x8000, 1); + dev_dbg(phy->dev, "impedance calibration code %#x\n", code); + return 0; +} + +static int t6050_activate(struct apple_dptx_phy *phy) +{ + t6050_device_power(phy, true); + phy->active = true; + t6050_pll_core_cfg(phy); /* PLL top, PHY and AUX configuration: nothing on this board */ + t6050_configure_pll(phy); + /* PLL activation */ + T6050_D(phy, 0x1000, 1, 0x2, 0); + T6050_D(phy, 0x4004, 3, 0x8, 1); + /* PHY activation: every lane, highest first */ + udelay(1); + for (int l = DPTX_MAX_LANES - 1; l >= 0; l--) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 9, 0x200, 0); + /* AUX power-up */ + T6050_D(phy, 0xd000, 0, 0x1, 0); + return t6050_impedance_calibration(phy); +} + +/* One PLL command (7 up, 0 down): request, 100 us, acknowledgment, lock state. */ +static int t6050_pll_command(struct apple_dptx_phy *phy, u32 cmd) +{ + bool up = cmd == 7; + u32 ack, sts; + + T6050_D(phy, 0x100c, 0, 0x7, cmd); + T6050_D(phy, 0x100c, 3, 0x8, 1); + udelay(100); + ack = readl(phy->regs.dptx + 0x1014); + T6050_D(phy, 0x100c, 3, 0x8, 0); + sts = readl(phy->regs.dptx + 0x1008); + if (!(ack & 1)) { + dev_err(phy->dev, "PLL power %s: no acknowledgment (0x1014 %#x)\n", + up ? "up" : "down", ack); + return -ETIMEDOUT; + } + if (!!(sts & BIT(3)) != up) { + dev_err(phy->dev, "PLL power %s: lock is %s (0x1008 %#x)\n", + up ? "up" : "down", up ? "missing" : "still set", sts); + return -EIO; + } + return 0; +} + +static int t6050_pll_power_down(struct apple_dptx_phy *phy) +{ + int ret; + + if (!t6050_pclk(phy, false)) + return 0; /* already down */ + ret = t6050_pll_command(phy, 0); + udelay(1); + return ret; +} + +static int t6050_pll_power_up(struct apple_dptx_phy *phy) +{ + int ret = t6050_pll_command(phy, 7); + + if (!ret) + t6050_pclk(phy, true); + return ret; +} + +static void t6050_phy_power_down(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x4004, 0, 0x1, 1); + T6050_D(phy, 0x4000, 6, 0x40, 0); + T6050_D(phy, 0x4004, 3, 0x8, 0); + T6050_D(phy, 0x4000, 25, 0x2000000, 0); + T6050_D(phy, 0x4000, 24, 0x1000000, 0); +} + +static void t6050_phy_power_up(struct apple_dptx_phy *phy) +{ + T6050_D(phy, 0x4004, 3, 0x8, 1); + for (int l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l + 0x10, + 27, 0x38000000, 4); + T6050_D(phy, 0x4030, 0, 0x1, 1); /* TXA LDO low-power mode */ + udelay(1); + T6050_D(phy, 0x4000, 24, 0x1000000, 1); + ndelay(250); + T6050_D(phy, 0x4000, 25, 0x2000000, 1); + udelay(1); + T6050_D(phy, 0x4000, 6, 0x40, 1); + udelay(1); + T6050_D(phy, 0x4004, 0, 0x1, 0); +} + +static int t6050_set_lanes(struct apple_dptx_phy *phy, u32 n) +{ + if (n == 3 || n > DPTX_MAX_LANES) + return -EINVAL; + if (!phy->active) + return 0; + T6050_D(phy, 0x4000, 26, 0x4000000, 1); + udelay(1); + for (u32 l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 8, 0x100, n <= l); + if (n) + T6050_D(phy, 0x4000, 26, 0x4000000, 0); + phy->active_lanes = n; + return 0; +} + +/* + * The PLL record of a link rate (after the T8122 adjustment of its first two + * words): the integer/fraction divider q08 + q0c / q10, the analog fields and + * whether the PLL clock output is bypassed. + */ +struct t6050_pll_cfg { + u32 rate; + u8 q00, q04, q14, q18, q1c; + u16 q08, q0c, q10, q24, q28; + bool bypass; +}; + +static const struct t6050_pll_cfg t6050_pll_cfgs[] = { + { 1620, 1, 0, 1, 5, 0x13, 0x87, 0, 0, 0x80, 0x8804, true }, + { 2700, 1, 0, 1, 5, 0x09, 0x70, 0x1fff, 0x3ffe, 0x80, 0x8804, true }, + { 5400, 1, 1, 1, 5, 0x04, 0x70, 0x1fff, 0x3ffe, 0x100, 0x8984, true }, + { 8100, 1, 1, 1, 6, 0x04, 0xa8, 0x2ffd, 0x3ffc, 0x100, 0x8984, false }, +}; + +static void t6050_shadow_update(struct apple_dptx_phy *phy) +{ + set32(phy->regs.core + 0xa0, 0x8); + set32(phy->regs.core + 0xa0, 0x4); + set32(phy->regs.core + 0xa0, 0x40000); + clear32(phy->regs.core + 0xa0, 0x40000); + set32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x4); + clear32(phy->regs.core + 0xa0, 0x8); +} + +/* Program a link rate's PLL frequency and half-rate fields (the PLL and PHY stay as they are). */ +static int t6050_program_rate(struct apple_dptx_phy *phy, u32 rate) +{ + const struct t6050_pll_cfg *c = NULL; + u32 q10, f, e; + bool half; + + for (int i = 0; i < ARRAY_SIZE(t6050_pll_cfgs); i++) + if (t6050_pll_cfgs[i].rate == rate) + c = &t6050_pll_cfgs[i]; + if (!c) + return -EINVAL; + + q10 = c->q10 ? c->q10 : 1; + f = 4 * c->q08 + (4 * c->q0c) / q10; + e = c->q0c > 1 ? c->q10 : 0; + half = rate >= 2000; + + T6050_D(phy, 0x222c, 1, 0x2, c->bypass); + T6050_D(phy, 0x222c, 0, 0x1, !c->bypass); + T6050_D(phy, 0x2244, 0, 0xffff, c->q28); + T6050_D(phy, 0x2234, 23, 0xff800000, c->q24); + T6050_D(phy, 0x20b0, 0, 0x3ff, f); + T6050_D(phy, 0x20b0, 10, 0x400, 0); + T6050_D(phy, 0x20b4, 14, 0x0fffc000, c->q0c); + T6050_D(phy, 0x20b4, 0, 0x3fff, e); + T6050_D(phy, 0x20b8, 9, 0x3e00, c->q1c); + T6050_D(phy, 0x20b8, 14, 0xc000, c->q14); + T6050_D(phy, 0x20b8, 16, 0xf0000, c->q18); + T6050_D(phy, 0x20b8, 20, 0x300000, c->q00); + T6050_D(phy, 0x20b8, 22, 0x400000, c->q04); + t6050_shadow_update(phy); + + /* half-link-rate: PHY, then PLL */ + T6050_D(phy, 0x4004, 5, 0x20, !half); + T6050_D(phy, 0x4004, 6, 0x40, half); + T6050_D(phy, 0x2000, 29, 0x20000000, !half); + T6050_D(phy, 0x201c, 5, 0x20, !half); + phy->rate = rate; + return 0; +} + +/* + * A link reconfiguration as the DCP brackets it: PHY and PLL down, the new + * lane count and rate, then (for a nonzero rate) PLL and PHY up. + */ +static int t6050_reconfigure(struct apple_dptx_phy *phy, bool set_lanes, + u32 lanes, bool set_rate, u32 rate) +{ + int ret; + + if (!phy->active) + return -ENODEV; + t6050_phy_power_down(phy); + ret = t6050_pll_power_down(phy); + if (ret) + return ret; + if (set_lanes) { + ret = t6050_set_lanes(phy, lanes); + if (ret) + return ret; + } + if (set_rate) { + if (!rate) { + phy->rate = 0; + return 0; + } + ret = t6050_program_rate(phy, rate); + if (ret) + return ret; + } + if (!phy->rate) + return 0; + ret = t6050_pll_power_up(phy); + if (ret) + return ret; + t6050_phy_power_up(phy); + return 0; +} + +static int t6050_deactivate(struct apple_dptx_phy *phy) +{ + int ret; + + if (!phy->active) + return 0; + t6050_set_lanes(phy, 0); + t6050_phy_power_down(phy); + for (int l = 0; l < DPTX_MAX_LANES; l++) + T6050_D(phy, DPTX_LANE0_OFFSET + DPTX_LANE_STRIDE * l, 9, 0x200, 1); + ret = t6050_pll_power_down(phy); + T6050_D(phy, 0xd000, 0, 0x1, 1); /* AUX power-down */ + phy->active = false; + phy->rate = 0; + t6050_device_power(phy, false); + return ret; +} + +static int t6050_probe(struct apple_dptx_phy *phy) +{ + struct device_node *np = phy->dev->of_node; + u32 v; + + for (int i = 0; i < T6050_NR_CAL; i++) { + if (of_property_read_u32(np, t6050_cal_props[i], &v) || v == 0xffffffff) + continue; + phy->cal[i] = v; + phy->cal_set[i] = true; + } + /* The crossbar's ufp endpoint u: 5 and 6 select themselves, 0xf none, any other u + 1. */ + if (of_property_read_u32(np, "apple,ufp-endpoint", &v)) + return dev_err_probe(phy->dev, -EINVAL, "no apple,ufp-endpoint\n"); + phy->source_select = (v == 5 || v == 6) ? v : v == 0xf ? 0 : v + 1; + phy->pclkdrv_select = !of_property_read_u32(np, "apple,pclkdrv-select", &v) && v; + + return 0; +} + static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + if (dptx_phy->hw.type == DPTX_PHY_T6050) { + int ret = -EINVAL; + + mutex_lock(&dptx_phy->lock); + if (mode == PHY_MODE_INVALID) + ret = t6050_deactivate(dptx_phy); + else if (mode == PHY_MODE_DP) + ret = dptx_phy->active ? 0 : t6050_activate(dptx_phy); + mutex_unlock(&dptx_phy->lock); + return ret; + } + switch (mode) { case PHY_MODE_INVALID: return dptx_phy_deactivate(dptx_phy); @@ -574,6 +990,16 @@ static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) enum dptx_phy_link_rate link_rate; int ret = 0; + if (dptx_phy->hw.type == DPTX_PHY_T6050) { + if (!opts->set_lanes && !opts->set_rate) + return 0; + mutex_lock(&dptx_phy->lock); + ret = t6050_reconfigure(dptx_phy, opts->set_lanes, opts->lanes, + opts->set_rate, opts->link_rate); + mutex_unlock(&dptx_phy->lock); + return ret; + } + if (opts->set_lanes) { mutex_lock(&dptx_phy->lock); ret = dptx_phy_set_active_lane_count(dptx_phy, opts->lanes); @@ -656,6 +1082,8 @@ static int dptx_phy_probe(struct platform_device *pdev) if (IS_ERR(dptx_phy->phy_provider)) return PTR_ERR(dptx_phy->phy_provider); + if (dptx_phy->hw.type == DPTX_PHY_T6050) + return t6050_probe(dptx_phy); return 0; } @@ -667,9 +1095,14 @@ static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { .type = DPTX_PHY_T8112, }; +static const struct apple_dptx_phy_hw apple_dptx_hw_t6050 = { + .type = DPTX_PHY_T6050, +}; + static const struct of_device_id dptx_phy_match[] = { { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + { .compatible = "apple,t6050-dptx-phy", .data = &apple_dptx_hw_t6050 }, {}, }; MODULE_DEVICE_TABLE(of, dptx_phy_match); From 5b0f430dae5813da53da05f69338bd5320d83f02 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 19 Oct 2026 07:10:51 -0400 Subject: [PATCH 020/392] soc: apple: rtkit: read crashlogs written after the crash report A co-processor whose crashlog endpoint was never discovered or started, such as a DCP adopted from the bootloader, requests the crashlog buffer only when it crashes and writes the log after receiving it. The header read right after mapping the buffer then fails, and the report was dropped without a word: on T6050 all three DCPs crashed this way within 8 s, display swaps stopped completing and the compositor froze, with nothing in the log but "Message to undiscovered endpoint 0x01". Report such a crash and poll the buffer until the log is written, then dump it. Only do this when APPLE_RTKIT_EP_CRASHLOG is not in rtk->endpoints. On a started crashlog endpoint the first message without a buffer is the normal boot-time buffer request, not a crash; treating it as one reported a crash for every other co-processor at boot (SMC, NVMe, PMP and the DP PHY on T6050). That path stays as it was: map the buffer and return silently. Nothing else changes: the consumer is not told, as before. Signed-off-by: Diljit Singh Signed-off-by: Erik Bourget --- drivers/soc/apple/rtkit-internal.h | 5 +++ drivers/soc/apple/rtkit.c | 69 +++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 0c5acb7fc7090..b555e13cf93e3 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -82,6 +82,11 @@ struct apple_rtkit { bool crashlog_inherited; bool oslog_inherited; + /* A crash reported before the crashlog buffer was mapped: the log is + * read once the co-processor has written it. */ + struct delayed_work crashlog_late_work; + unsigned int crashlog_late_polls; + struct workqueue_struct *wq; }; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d352083aaa26c..04d603129f0fd 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -695,6 +695,51 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, len, DMA_FROM_DEVICE); } +#define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 +#define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 + +static bool apple_rtkit_crashlog_header_valid(struct apple_rtkit *rtk) +{ + struct apple_rtkit_crashlog_header header; + + if (rtk->crashlog_buffer.size < sizeof(header)) + return false; + apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, + sizeof(header)); + return header.fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC && + header.size >= sizeof(header) && + header.size <= rtk->crashlog_buffer.size; +} + +static void apple_rtkit_crashlog_late(struct work_struct *work) +{ + struct apple_rtkit *rtk = container_of(to_delayed_work(work), + struct apple_rtkit, + crashlog_late_work); + u8 *bfr __free(kfree) = NULL; + + if (!apple_rtkit_crashlog_header_valid(rtk)) { + if (++rtk->crashlog_late_polls < APPLE_RTKIT_CRASHLOG_LATE_POLLS) + queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, + msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); + else + dev_err(rtk->dev, + "RTKit: crash reported, but no crashlog was written in %u ms\n", + APPLE_RTKIT_CRASHLOG_LATE_POLLS * + APPLE_RTKIT_CRASHLOG_LATE_POLL_MS); + return; + } + + dev_err(rtk->dev, "RTKit: co-processor crashed; crashlog written after %u ms\n", + (rtk->crashlog_late_polls + 1) * APPLE_RTKIT_CRASHLOG_LATE_POLL_MS); + bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL); + if (!bfr) + return; + apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0, + rtk->crashlog_buffer.size); + apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size); +} + static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); @@ -728,8 +773,27 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) sizeof(header)); if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC || header.size < sizeof(header) || - header.size > rtk->crashlog_buffer.size) + header.size > rtk->crashlog_buffer.size) { + /* + * On a started crashlog endpoint the first message + * without a buffer is the normal boot-time buffer + * request. A co-processor whose crashlog endpoint was + * never discovered or started (a DCP adopted from the + * bootloader) sends it only when it crashes, and writes + * the log after receiving the buffer. Say so and read + * the log once it is there, rather than dropping the + * report. + */ + if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) { + dev_err(rtk->dev, + "RTKit: co-processor reported a crash before its crashlog buffer was mapped\n"); + rtk->crashlog_late_polls = 0; + queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, + msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); + return; + } goto unmapped; + } } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); @@ -1279,6 +1343,7 @@ static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) static void apple_rtkit_stop_rx(struct apple_rtkit *rtk) { apple_rtkit_detach_rx(rtk); + cancel_delayed_work_sync(&rtk->crashlog_late_work); destroy_workqueue(rtk->wq); } @@ -1382,6 +1447,7 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, init_completion(&rtk->coreanalytics_completion); init_completion(&rtk->iop_pwr_ack_completion); init_completion(&rtk->ap_pwr_ack_completion); + INIT_DELAYED_WORK(&rtk->crashlog_late_work, apple_rtkit_crashlog_late); bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); @@ -1511,6 +1577,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) /* make sure we don't handle any messages while reinitializing */ apple_rtkit_detach_rx(rtk); + cancel_delayed_work_sync(&rtk->crashlog_late_work); flush_workqueue(rtk->wq); apple_rtkit_free_buffer(rtk, &rtk->coreanalytics_buffer); From 8ee4d615e076d1294ef6c07fd2c115fdeced5387 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Sun, 1 Nov 2026 22:35:05 -0800 Subject: [PATCH 021/392] soc: apple: rtkit: Report a crash on an undiscovered crashlog endpoint In a session that went through the handshake, the crashlog handler takes the first message on the crashlog endpoint for the buffer request that answers the endpoint's start: it sets the buffer up and returns. That has been the rule since RTKit was added. Two later commits made an inherited session's first message a crash report and kept the rule for every other session: commit e52b589758bd ("soc: apple: rtkit: recognise an initial crash log in adopted sessions") commit 8a2875a6789d ("soc: apple: rtkit: Only parse an initial crash log in inherited sessions") The previous commit reads the log of such a report when it is written after the report, but only in an inherited session, and without telling the client. The rule only holds when the endpoint was in the endpoint map and was started. The external DCP on T6050 (dcpext0 on J775D) does not list the crashlog endpoint in its map, so the endpoint is never started and the co-processor is never asked for a buffer. Its firmware still reports a crash there, naming a buffer of its own, in a session that went through the handshake as well. That report was set up as a buffer and dropped: the instance was never marked crashed and the DCP driver's crash handling never ran. Presents stopped (a swap timed out 36 s later) and every later modeset failed with -EINVAL until a reboot, with nothing in the log but "Message to undiscovered endpoint 0x01". Handle a crashlog message on an undiscovered endpoint as an inherited session's report is handled, in any session: set up the buffer it names and dump a valid log if there is one, then mark the instance crashed and notify the client at once, with or without a readable log. A log that is not written yet is still read once it is there, after the client has been told. A buffer that could not be set up and a buffer that holds no valid log yet are reported separately. Log when the endpoint map lacks the crashlog endpoint. A crashlog message still being handled may queue that late read, so stopping and reinitializing flush the work queue before cancelling it. A discovered endpoint is unchanged: its first message is still the buffer request. This builds on the inherited-session path those commits added, so a backport needs them. Testing: on J775D the map line appears for dcpext0 and for no other co-processor. A crash caused on dcpext0, by an IOMFB call to a method neither of its firmware tables has, was reported on the crashlog endpoint 4.7 ms after the call, with the same buffer as a crash seen in normal use. The instance was marked crashed and the DCP driver's crash handling ran ("DCP has crashed", the connector then reported disconnected). Fixes: 9bd1d9a0d8bb ("soc: apple: Add RTKit IPC library") Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/soc/apple/rtkit.c | 93 ++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 04d603129f0fd..e6c7aa4a28216 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -256,6 +256,9 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; + if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) + dev_info(rtk->dev, "RTKit: crashlog endpoint not in the endpoint map\n"); + for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { switch (ep) { /* the management endpoint is started by default */ @@ -730,6 +733,7 @@ static void apple_rtkit_crashlog_late(struct work_struct *work) return; } + /* The crash was reported and the client told; only dump the log. */ dev_err(rtk->dev, "RTKit: co-processor crashed; crashlog written after %u ms\n", (rtk->crashlog_late_polls + 1) * APPLE_RTKIT_CRASHLOG_LATE_POLL_MS); bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL); @@ -744,7 +748,16 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); u8 *bfr __free(kfree) = NULL; - struct apple_rtkit_crashlog_header header; + /* + * An undiscovered crashlog endpoint was never started, so the + * co-processor was never asked for a buffer and, as in an inherited + * session, a message there can only be a crash report. Firmware that + * leaves this endpoint out of its map still reports crashes on it, + * with a buffer of its own. + */ + bool reports_crash = rtk->crashlog_inherited || + !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints); + bool await_log = false; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -755,45 +768,23 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) - goto unmapped; + goto no_buffer; /* * In a session that went through the handshake, the first - * message on this endpoint is the buffer request. Firmware may - * place that buffer in memory which still holds the log of an - * earlier crash, so only look for one in an inherited session. + * message on a started crashlog endpoint is the buffer request. + * Firmware may place that buffer in memory which still holds + * the log of an earlier crash, so only look for one when the + * message can only be a crash report. */ - if (!rtk->crashlog_inherited) + if (!reports_crash) return; } - if (rtk->crashlog_inherited) { - if (rtk->crashlog_buffer.size < sizeof(header)) - goto unmapped; - - apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, - sizeof(header)); - if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC || - header.size < sizeof(header) || - header.size > rtk->crashlog_buffer.size) { - /* - * On a started crashlog endpoint the first message - * without a buffer is the normal boot-time buffer - * request. A co-processor whose crashlog endpoint was - * never discovered or started (a DCP adopted from the - * bootloader) sends it only when it crashes, and writes - * the log after receiving the buffer. Say so and read - * the log once it is there, rather than dropping the - * report. - */ - if (!test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints)) { - dev_err(rtk->dev, - "RTKit: co-processor reported a crash before its crashlog buffer was mapped\n"); - rtk->crashlog_late_polls = 0; - queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, - msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); - return; - } - goto unmapped; - } + if (reports_crash) { + if (rtk->crashlog_buffer.size < + sizeof(struct apple_rtkit_crashlog_header)) + goto no_buffer; + if (!apple_rtkit_crashlog_header_valid(rtk)) + goto no_log; } dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); @@ -818,14 +809,34 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); return; -unmapped: - /* An inherited session already negotiated its crash buffer. */ - if (!rtk->crashlog_inherited) +no_buffer: + /* Otherwise this was a buffer request that could not be served. */ + if (!reports_crash) return; - dev_err(rtk->dev, "RTKit: adopted co-processor crashed without a readable log\n"); + dev_err(rtk->dev, "RTKit: co-processor crashed without a usable crashlog buffer\n"); + goto crashed; + +no_log: + /* The buffer is mapped, but holds no valid log (yet). */ + dev_err(rtk->dev, "RTKit: co-processor crashed without a readable log\n"); + /* + * A co-processor whose crashlog endpoint was never discovered or + * started (the external DCP on T6050) requests the buffer only when + * it crashes, and writes the log after receiving it. Read the log once + * it is there, rather than dropping it. + */ + await_log = !test_bit(APPLE_RTKIT_EP_CRASHLOG, rtk->endpoints); + +crashed: rtk->crashed = true; if (rtk->ops->crashed) rtk->ops->crashed(rtk->cookie, NULL, 0); + + if (await_log) { + rtk->crashlog_late_polls = 0; + queue_delayed_work(rtk->wq, &rtk->crashlog_late_work, + msecs_to_jiffies(APPLE_RTKIT_CRASHLOG_LATE_POLL_MS)); + } } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) @@ -1343,6 +1354,8 @@ static void apple_rtkit_detach_rx(struct apple_rtkit *rtk) static void apple_rtkit_stop_rx(struct apple_rtkit *rtk) { apple_rtkit_detach_rx(rtk); + /* A crashlog message still being handled may queue the late read. */ + flush_workqueue(rtk->wq); cancel_delayed_work_sync(&rtk->crashlog_late_work); destroy_workqueue(rtk->wq); } @@ -1577,8 +1590,8 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) /* make sure we don't handle any messages while reinitializing */ apple_rtkit_detach_rx(rtk); - cancel_delayed_work_sync(&rtk->crashlog_late_work); flush_workqueue(rtk->wq); + cancel_delayed_work_sync(&rtk->crashlog_late_work); apple_rtkit_free_buffer(rtk, &rtk->coreanalytics_buffer); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); From edbf3214f22ac0ab0dc3049641572d05aec28168 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:09:37 -0800 Subject: [PATCH 022/392] soc: apple: rtkit: Read crashlogs written as a container The external DCP of T6050 (dcpext0 on J775D) writes its crashlog as a container: a 'CLGH' header and a table of records (ID, offset), where the record whose ID is the container's current ID holds this crash's log. That record can be a container itself; on this DCP the buffer holds one per co-processor and, inside, one per task. The log at the end is an ordinary 'CLHE' crashlog with the usual sections. RTKit looked for 'CLHE' at offset 0 only, so such a log was never valid: the crash was reported "without a readable log" although the log was in the buffer. Follow the container to its log, through at most 8 levels and with offsets that only move forward, both when checking that a log is there and when dumping it. A log selected through a container is not complete while bit 8 of its flags is set. Decode the 'Crl4' register frame, at the offsets of Diljit Singh's T8140 DCP decoder. The DCP logs read on J775D so far carry no register frame section, neither 'Crg8' nor 'Crl4', so this decoder has not run on a real log, only in the host test below. Container logs come from firmware that has just crashed, so the section walk no longer trusts their sizes. A section header must fit in the log; a section's size must cover its own 16-byte header and stay inside the log; and a section is decoded only with at least the footer's 16-byte header after it, since the decoders read up to the section's size from 16 bytes in. A zero size used to loop forever, and a size past the log, or a mailbox section under 28 bytes, made the decoders read past the copy. The version and message strings are bounded by the section's size instead of a terminating NUL, and a buffer shorter than a log header is refused. A buffer that starts with 'CLHE' is checked as before, and a well-formed log there is decoded exactly as before. Testing: on J775D, the crash caused on dcpext0 as in the previous commit now prints "Crashlog container: log 'RTAL' at 0x2000 (depth 2)", the log's message naming the method the firmware could not find, and the firmware's mailbox history (128 entries), whose last entry is that call. The log was complete when the crash was reported. A buffer read back after an earlier crash on the 27.2 firmware has the same layout (logs for the root task, RTAL and one untagged task). These runs predate the section bounds, which change nothing for a well-formed log. A host test, tools/asahi/rtkit-crashlog-tests.py, builds this file with ASan and UBSan and feeds it a log with the layout of that crash (two containers, then RTAL's log with its eleven sections and footer) and malformed logs: zero, oversized and footer-less sections, a truncated section header, mailbox sections of 16 to 27 bytes, short, truncated and undecodable 'Crl4' sections, unterminated strings, short buffers and broken containers. Each is decoded or refused without a loop or a read past the log, and removing any one bound makes it fail. The buffer read back after the earlier crash decodes the same with and without the bounds. The container selection, the record layout, the incomplete flag and the 'Crl4' offsets follow that decoder. Here they are in RTKit, so that every client gets them. Co-developed-by: Diljit Singh Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/soc/apple/rtkit-crashlog.c | 225 +++++++++++- drivers/soc/apple/rtkit-internal.h | 4 + drivers/soc/apple/rtkit.c | 20 +- tools/asahi/rtkit-crashlog-tests.py | 512 ++++++++++++++++++++++++++++ tools/asahi/run-host-tests.py | 3 + 5 files changed, 757 insertions(+), 7 deletions(-) create mode 100755 tools/asahi/rtkit-crashlog-tests.py diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 5defa12862b15..2ec4bd9569e46 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -3,6 +3,8 @@ * Apple RTKit IPC library * Copyright (C) The Asahi Linux Contributors */ +#include + #include "rtkit-internal.h" #define FOURCC(a, b, c, d) \ @@ -13,6 +15,51 @@ #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x') #define APPLE_RTKIT_CRASHLOG_TIME FOURCC('C', 't', 'i', 'm') #define APPLE_RTKIT_CRASHLOG_REGS FOURCC('C', 'r', 'g', '8') +#define APPLE_RTKIT_CRASHLOG_REGS_CL4 FOURCC('C', 'r', 'l', '4') + +/* + * Newer firmware writes a container instead of a single crashlog: a + * 'CLGH' header and a table of records, each with an ID and an offset from + * the container. The record whose ID is the container's current ID holds + * the log of this crash, and may itself be a container (on T6050's DCP: + * per-coprocessor, then per-task). The log at the end is an ordinary + * 'CLHE' crashlog with the same sections. + */ +struct apple_rtkit_crashlog_container { + u32 fourcc; + u32 version; + u32 size; + u32 current_id; + u32 count; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_container) == 0x14); + +struct apple_rtkit_crashlog_record { + u64 id; + u64 _unk0; + u32 offset; + u32 _unk1; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_record) == 0x18); + +#define APPLE_RTKIT_CRASHLOG_CONTAINER_VERSION 2 +#define APPLE_RTKIT_CRASHLOG_CONTAINER_MAX_DEPTH 8 +/* Set in a log's flags while the co-processor is still writing it. */ +#define APPLE_RTKIT_CRASHLOG_INCOMPLETE BIT(8) + +/* Register frame of the 'Crl4' section, offsets from the section start. */ +#define APPLE_RTKIT_CRASHLOG_CL4_SECTION_SIZE 0x378 +#define APPLE_RTKIT_CRASHLOG_CL4_FLAGS 0x10 +#define APPLE_RTKIT_CRASHLOG_CL4_SIZE 0x14 +#define APPLE_RTKIT_CRASHLOG_CL4_NOMINAL_SIZE 0xd8 +#define APPLE_RTKIT_CRASHLOG_CL4_REGS 0x18 +#define APPLE_RTKIT_CRASHLOG_CL4_LR 0x108 +#define APPLE_RTKIT_CRASHLOG_CL4_PC 0x110 +#define APPLE_RTKIT_CRASHLOG_CL4_SP 0x118 +#define APPLE_RTKIT_CRASHLOG_CL4_PSR 0x120 +#define APPLE_RTKIT_CRASHLOG_CL4_ESR 0x358 +#define APPLE_RTKIT_CRASHLOG_CL4_FAR 0x360 +#define APPLE_RTKIT_CRASHLOG_CL4_UNDECODABLE BIT(16) /* For COMPILE_TEST on non-ARM64 architectures */ #ifndef PSR_MODE_EL0t @@ -71,8 +118,8 @@ static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr, *newline = tmp; ptr = newline + 1; } else { - dev_warn(rtk->dev, "RTKit: Message (id=%x): %s", idx, - ptr); + dev_warn(rtk->dev, "RTKit: Message (id=%x): %.*s", idx, + (int)(end - ptr), ptr); break; } } @@ -81,7 +128,8 @@ static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr, static void apple_rtkit_crashlog_dump_version(struct apple_rtkit *rtk, u8 *bfr, size_t size) { - dev_warn(rtk->dev, "RTKit: Version: %s", bfr + 16); + /* The string need not be terminated inside the section. */ + dev_warn(rtk->dev, "RTKit: Version: %.*s", (int)(size - 16), bfr + 16); } static void apple_rtkit_crashlog_dump_time(struct apple_rtkit *rtk, u8 *bfr, @@ -100,6 +148,10 @@ static void apple_rtkit_crashlog_dump_mailbox(struct apple_rtkit *rtk, u8 *bfr, size_t n_messages; struct apple_rtkit_crashlog_mbox_entry entry; + if (size < 28) { + dev_warn(rtk->dev, "RTKit: Mailbox section too small: 0x%zx", size); + return; + } memcpy(&type, bfr + 16, 4); memcpy(&index, bfr + 24, 4); n_messages = (size - 28) / sizeof(entry); @@ -173,13 +225,150 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr, dev_warn(rtk->dev, "\n"); } +static void apple_rtkit_crashlog_dump_regs_cl4(struct apple_rtkit *rtk, + u8 *section, size_t size) +{ + u32 flags, regs_size; + int i; + + if (size < APPLE_RTKIT_CRASHLOG_CL4_SECTION_SIZE) { + dev_warn(rtk->dev, "RTKit: Crl4 section too small: 0x%zx", size); + return; + } + flags = get_unaligned_le32(section + APPLE_RTKIT_CRASHLOG_CL4_FLAGS); + regs_size = get_unaligned_le32(section + APPLE_RTKIT_CRASHLOG_CL4_SIZE); + if (regs_size != APPLE_RTKIT_CRASHLOG_CL4_NOMINAL_SIZE || + (flags & APPLE_RTKIT_CRASHLOG_CL4_UNDECODABLE)) { + dev_warn(rtk->dev, "RTKit: Crl4 frame not decodable (flags %#x, size %#x)", + flags, regs_size); + return; + } + + dev_warn(rtk->dev, "RTKit: Exception dump (Crl4):"); + dev_warn(rtk->dev, " PSR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_PSR)); + dev_warn(rtk->dev, " PC = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_PC)); + dev_warn(rtk->dev, " LR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_LR)); + dev_warn(rtk->dev, " ESR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_ESR)); + dev_warn(rtk->dev, " FAR = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_FAR)); + dev_warn(rtk->dev, " SP = 0x%llx", + get_unaligned_le64(section + APPLE_RTKIT_CRASHLOG_CL4_SP)); + for (i = 0; i < 28; i += 4) { + u8 *r = section + APPLE_RTKIT_CRASHLOG_CL4_REGS + i * 8; + + dev_warn(rtk->dev, " x%02d-x%02d = %016llx %016llx %016llx %016llx", + i, i + 3, get_unaligned_le64(r), get_unaligned_le64(r + 8), + get_unaligned_le64(r + 16), get_unaligned_le64(r + 24)); + } +} + +/** + * apple_rtkit_crashlog_locate() - find this crash's log in a crashlog buffer + * @bfr: a copy of the buffer + * @size: its size + * @log_offset: set to the offset of the 'CLHE' log + * @log_id: set to the ID of the record that held it (0 for a bare log) + * @depth: set to the number of containers followed + * + * A bare 'CLHE' log at offset 0 is returned as it is. In a 'CLGH' container, + * the record with the current ID is followed, through nested containers, + * to its log. Offsets only move forward, so a corrupt table cannot loop. + * + * Return: 0 if a complete log was found, -EAGAIN if the selected log is + * still being written, -EINVAL otherwise. + */ +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth) +{ + struct apple_rtkit_crashlog_container container; + struct apple_rtkit_crashlog_header header; + struct apple_rtkit_crashlog_record record; + size_t offset = 0, avail, next; + unsigned int level; + u32 id = 0, i; + + for (level = 0; level <= APPLE_RTKIT_CRASHLOG_CONTAINER_MAX_DEPTH; level++) { + avail = size - offset; + if (avail < sizeof(header)) + return -EINVAL; + memcpy(&header, bfr + offset, sizeof(header)); + + if (header.fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { + if (header.size < sizeof(header) || header.size > avail) + return -EINVAL; + if (level && (header.flags & APPLE_RTKIT_CRASHLOG_INCOMPLETE)) + return -EAGAIN; + *log_offset = offset; + *log_id = id; + *depth = level; + return 0; + } + if (header.fourcc != APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) + return -EINVAL; + + memcpy(&container, bfr + offset, sizeof(container)); + if (((container.version >> 8) & 0xff) != + APPLE_RTKIT_CRASHLOG_CONTAINER_VERSION || + container.size > avail || !container.count || + container.count > (avail - sizeof(container)) / sizeof(record)) + return -EINVAL; + + next = 0; + for (i = 0; i < container.count; i++) { + memcpy(&record, bfr + offset + sizeof(container) + + i * sizeof(record), sizeof(record)); + if (record.id == container.current_id) { + next = record.offset; + break; + } + } + if (next < sizeof(container) || next >= avail) + return -EINVAL; + + id = container.current_id; + offset += next; + } + + return -EINVAL; +} + void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) { size_t offset; u32 section_fourcc, section_size; struct apple_rtkit_crashlog_header header; + if (size < sizeof(header)) { + dev_warn(rtk->dev, "RTKit: Crashlog buffer too small: 0x%zx", + size); + return; + } memcpy(&header, bfr, sizeof(header)); + if (header.fourcc == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) { + unsigned int depth; + u32 id; + int ret; + + ret = apple_rtkit_crashlog_locate(bfr, size, &offset, &id, &depth); + if (ret) { + dev_warn(rtk->dev, + "RTKit: Crashlog container holds no complete log (%d)", + ret); + return; + } + dev_warn(rtk->dev, + "RTKit: Crashlog container: log '%c%c%c%c' at 0x%zx (depth %u)", + (id >> 24) & 0xff, (id >> 16) & 0xff, (id >> 8) & 0xff, + id & 0xff, offset, depth); + bfr += offset; + size -= offset; + memcpy(&header, bfr, sizeof(header)); + } + if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_warn(rtk->dev, "RTKit: Expected crashlog header but got %x", header.fourcc); @@ -196,13 +385,35 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) offset = sizeof(header); while (offset < size) { + if (size - offset < 16) { + dev_warn(rtk->dev, + "RTKit: Crashlog section header at 0x%zx is truncated", + offset); + return; + } memcpy(§ion_fourcc, bfr + offset, 4); memcpy(§ion_size, bfr + offset + 12, 4); - switch (section_fourcc) { - case APPLE_RTKIT_CRASHLOG_HEADER_FOURCC: + if (section_fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { dev_dbg(rtk->dev, "RTKit: End of crashlog reached"); return; + } + /* A zero size would never advance. */ + if (section_size < 16 || section_size > size - offset) { + dev_warn(rtk->dev, + "RTKit: Crashlog section %x at 0x%zx has a bad size 0x%x", + section_fourcc, offset, section_size); + return; + } + /* + * The decoders read up to section_size bytes from 16 bytes into + * the section, so a section is only decoded with at least the + * footer's 16-byte header after it. + */ + if (section_size > size - offset - 16) + break; + + switch (section_fourcc) { case APPLE_RTKIT_CRASHLOG_STR: apple_rtkit_crashlog_dump_str(rtk, bfr + offset + 16, section_size); @@ -223,6 +434,10 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) apple_rtkit_crashlog_dump_regs(rtk, bfr + offset + 16, section_size); break; + case APPLE_RTKIT_CRASHLOG_REGS_CL4: + apple_rtkit_crashlog_dump_regs_cl4(rtk, bfr + offset, + section_size); + break; default: dev_warn(rtk->dev, "RTKit: Unknown crashlog section: %x", diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index b555e13cf93e3..a4976fd1b456c 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -23,6 +23,8 @@ #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 #define APPLE_RTKIT_CRASHLOG_HEADER_FOURCC 0x434c4845 +/* A container of crashlogs (one per task or core): 'CLGH'. */ +#define APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC 0x434c4748 struct apple_rtkit_crashlog_header { u32 fourcc; @@ -91,5 +93,7 @@ struct apple_rtkit { }; void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size); +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth); #endif diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index e6c7aa4a28216..1d56f00c851ec 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -701,17 +701,33 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, #define APPLE_RTKIT_CRASHLOG_LATE_POLL_MS 250 #define APPLE_RTKIT_CRASHLOG_LATE_POLLS 20 +/* + * Whether the crashlog buffer holds a log: an ordinary 'CLHE' log at its + * start, or a 'CLGH' container whose current record leads to a complete one. + */ static bool apple_rtkit_crashlog_header_valid(struct apple_rtkit *rtk) { struct apple_rtkit_crashlog_header header; + size_t size = rtk->crashlog_buffer.size, offset; + unsigned int depth; + u32 id; - if (rtk->crashlog_buffer.size < sizeof(header)) + if (size < sizeof(header)) return false; apple_rtkit_memcpy(rtk, &header, &rtk->crashlog_buffer, 0, sizeof(header)); + if (header.fourcc == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC) { + u8 *bfr __free(kfree) = kmalloc(size, GFP_KERNEL); + + if (!bfr) + return false; + apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0, size); + return !apple_rtkit_crashlog_locate(bfr, size, &offset, &id, + &depth); + } return header.fourcc == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC && header.size >= sizeof(header) && - header.size <= rtk->crashlog_buffer.size; + header.size <= size; } static void apple_rtkit_crashlog_late(struct work_struct *work) diff --git a/tools/asahi/rtkit-crashlog-tests.py b/tools/asahi/rtkit-crashlog-tests.py new file mode 100755 index 0000000000000..382c8d39cd1a3 --- /dev/null +++ b/tools/asahi/rtkit-crashlog-tests.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Feed RTKit's crashlog parser a container log and malformed logs. + +drivers/soc/apple/rtkit-crashlog.c is compiled whole and unchanged, with ASan +and UBSan, against a stand-in for rtkit-internal.h that carries the real +header's crashlog definitions. Every log is allocated at its exact size, so a +read past it is a sanitizer error, and a walk that stops advancing runs into +the line limit. + +The container log has the layout of a T6050 DCP crash (a 'CLGH' container, +a nested one, and at 0x2000 the 'CLHE' log of task RTAL with eleven sections +and a footer), with neutral contents. The malformed logs cover a zero and an +oversized section size, a truncated section header, a short mailbox section, +a short and a truncated 'Crl4' section, unterminated strings, short buffers +and broken containers. --log FILE feeds a raw crashlog buffer read from a +device instead, and prints what the parser makes of it. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/soc/apple/rtkit-crashlog.c' +HEADER = 'drivers/soc/apple/rtkit-internal.h' + +# Each mutation removes one bound; a test case must catch it. +MUTATIONS = { + 'buffer-header': ('if (size < sizeof(header)) {', 'if (0) {'), + 'section-header': ('if (size - offset < 16) {', 'if (0) {'), + 'section-size': ('if (section_size < 16 || section_size > size - offset) {', 'if (0) {'), + 'footer-room': ('if (section_size > size - offset - 16)\n\t\t\tbreak;', ''), + 'mailbox-minimum': ('if (size < 28) {', 'if (0) {'), + 'version-string': ('"RTKit: Version: %.*s", (int)(size - 16), bfr + 16', '"RTKit: Version: %s", bfr + 16'), + 'message-string': ('%.*s", idx,\n\t\t\t\t (int)(end - ptr), ptr', '%s", idx,\n\t\t\t\t ptr'), +} + +STUB = r''' +#include +#include +#include +#include +#include +#include +#include +#include +/* As linux/build_bug.h has it: the message is optional. */ +#undef static_assert +#define __static_assert(expr, msg, ...) _Static_assert(expr, msg) +#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) +typedef uint8_t u8; +typedef uint32_t u32; +typedef unsigned long long u64; +#define __packed __attribute__((packed)) +#define BIT(n) (1UL << (n)) +#define EINVAL 22 +#define EAGAIN 11 +struct device { int id; }; +struct apple_rtkit { struct device *dev; }; +void dev_printk(const char *level, struct device *dev, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); +#define dev_warn(dev, ...) dev_printk("warn", dev, __VA_ARGS__) +#define dev_dbg(dev, ...) dev_printk("dbg", dev, __VA_ARGS__) +@DEFINITIONS@ +int apple_rtkit_crashlog_locate(const u8 *bfr, size_t size, size_t *log_offset, + u32 *log_id, unsigned int *depth); +void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size); +''' + +UNALIGNED = r''' +#include +#include +static inline uint32_t get_unaligned_le32(const void *p) { uint32_t v; memcpy(&v, p, 4); return v; } +static inline unsigned long long get_unaligned_le64(const void *p) +{ unsigned long long v; memcpy(&v, p, 8); return v; } +''' + +MAIN = r''' +#include "rtkit-internal.h" + +#define FOURCC(s) (((u32)(s)[0] << 24) | ((u32)(s)[1] << 16) | ((u32)(s)[2] << 8) | (u32)(s)[3]) +#define MAX_LINES 1000 + +static char lines[MAX_LINES][512]; +static unsigned int n_lines; +static struct device device; +static struct apple_rtkit rtk = { .dev = &device }; + +void dev_printk(const char *level, struct device *dev, const char *fmt, ...) +{ + char *line = lines[n_lines]; + va_list ap; + size_t n; + + assert(dev == &device); + /* A walk that does not advance prints without end. */ + assert(n_lines < MAX_LINES - 1); + n_lines++; + va_start(ap, fmt); + vsnprintf(line, sizeof(lines[0]), fmt, ap); + va_end(ap); + n = strlen(line); + while (n && line[n - 1] == '\n') + line[--n] = 0; + printf(" %s: %s\n", level, line); +} + +static void put32(u8 *b, size_t off, u32 v) { memcpy(b + off, &v, 4); } +static void put64(u8 *b, size_t off, u64 v) { memcpy(b + off, &v, 8); } + +/* A section header: its type, a fill word, flags and its size, header included. */ +static size_t section(u8 *b, size_t off, const char *type, u32 flags, u32 size) +{ + put32(b, off, FOURCC(type)); + put32(b, off + 4, 0xefefefef); + put32(b, off + 8, flags); + put32(b, off + 12, size); + return off + size; +} + +/* A 'CLHE' header (or footer) of a log of `size` bytes at `off`. */ +static void clhe(u8 *b, size_t off, u32 size, u32 flags) +{ + put32(b, off, FOURCC("CLHE")); + put32(b, off + 4, 0x303); + put32(b, off + 8, size); + put32(b, off + 12, flags); +} + +/* A 'CLGH' container at `off` with `count` (id, offset) records. */ +static void clgh(u8 *b, size_t off, u32 size, const char *current, unsigned int count, + const char *const *ids, const u32 *offsets) +{ + put32(b, off, FOURCC("CLGH")); + put32(b, off + 4, 0x202); + put32(b, off + 8, size); + put32(b, off + 12, FOURCC(current)); + put32(b, off + 16, count); + for (unsigned int i = 0; i < count; i++) { + put64(b, off + 0x14 + i * 0x18, FOURCC(ids[i])); + put32(b, off + 0x14 + i * 0x18 + 16, offsets[i]); + } +} + +/* Runs the dump on an exact-size copy of `log` and returns the number of lines. */ +static unsigned int dump(const char *name, const u8 *log, size_t size) +{ + u8 *copy = malloc(size ? size : 1); + + assert(copy); + memcpy(copy, log, size); + n_lines = 0; + printf("%s (%zu bytes)\n", name, size); + apple_rtkit_crashlog_dump(&rtk, copy, size); + free(copy); + return n_lines; +} + +static bool printed(const char *text) +{ + for (unsigned int i = 0; i < n_lines; i++) + if (strstr(lines[i], text)) + return true; + return false; +} + +static bool ended(void) +{ + return n_lines && !strcmp(lines[n_lines - 1], "RTKit: End of crashlog reached"); +} + +#define DCP_SIZE 0x17000 +#define DCP_LOG 0x2000 +#define DCP_LOG_SIZE 0x26e0 +#define MAILBOX_ENTRIES 128 + +/* The layout of a T6050 DCP crashlog: the RTAL task's log, two containers deep. */ +static void dcp_log(u8 *b) +{ + static const char *const top[] = { "CL4U", "CL4K" }, *const tasks[] = { "root", "RTAL" }; + static const u32 top_offsets[] = { 0x44, 0x12000 }, task_offsets[] = { 0x44, 0x1fbc }; + static const u32 pattern[8] = { 0x202, 0x242, 0x302, 0x342, 0x302, 0x342, 0x202, 0x242 }; + size_t off = DCP_LOG + 0x20, body; + + memset(b, 0, DCP_SIZE); + clgh(b, 0, DCP_SIZE, "CL4U", 2, top, top_offsets); + clgh(b, 0x44, 0x11fbc, "RTAL", 2, tasks, task_offsets); + clhe(b, DCP_LOG, DCP_LOG_SIZE, 0x820); + put32(b, DCP_LOG + 0x1c, FOURCC("RTAL")); + + body = off + 16; + off = section(b, off, "Cstr", 0x101, 0x60); + put32(b, body, 1); + strcpy((char *)b + body + 4, "crash message [0x87aa9a0]"); + off = section(b, off, "Cver", 0x300, 0xa0); + body = off + 16; + off = section(b, off, "Ctim", 0x100, 0x20); + put64(b, body, 87481704354ULL); + off = section(b, off, "Ccst", 0x400, 0x70); + off = section(b, off, "Crtk", 0x300, 0x17c0); + body = off + 16; + off = section(b, off, "Cmbx", 0x300, 0xc30); + for (unsigned int i = 0; i < MAILBOX_ENTRIES; i++) { + size_t e = body + 28 + i * 0x18; + + put64(b, e, 0x37); + put64(b, e + 8, pattern[i % 8]); + put32(b, e + 16, 0xda8656a9 + i * 0x1000); + } + off = section(b, off, "Cpct", 0x200, 0x50); + off = section(b, off, "Csta", 0x100, 0x30); + off = section(b, off, "Cced", 0x100, 0x50); + off = section(b, off, "Chex", 0x100, 0x20); + off = section(b, off, "Csta", 0x100, 0x30); + assert(off == DCP_LOG + DCP_LOG_SIZE - 0x20); + clhe(b, off, DCP_LOG_SIZE, 0x820); +} + +static void test_dcp_log(void) +{ + static const char *const expect[] = { + "RTKit: Crashlog container: log 'RTAL' at 0x2000 (depth 2)", + "RTKit: Message (id=1): crash message [0x87aa9a0]", + "RTKit: Version: ", + "RTKit: Crash time: 87481704354", + "RTKit: Unknown crashlog section: 43637374", + "RTKit: Unknown crashlog section: 4372746b", + "RTKit: Mailbox history (type = 0, index = 0)", + "RTKit: Unknown crashlog section: 43706374", + "RTKit: Unknown crashlog section: 43737461", + "RTKit: Unknown crashlog section: 43636564", + "RTKit: Unknown crashlog section: 43686578", + "RTKit: Unknown crashlog section: 43737461", + "RTKit: End of crashlog reached", + }; + static u8 b[DCP_SIZE]; + unsigned int n, i, j = 0, entries = 0; + size_t log_offset; + unsigned int depth; + u32 id; + + dcp_log(b); + assert(!apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth)); + assert(log_offset == DCP_LOG && id == FOURCC("RTAL") && depth == 2); + n = dump("T6050 DCP container log", b, DCP_SIZE); + for (i = 0; i < n; i++) { + if (!strncmp(lines[i], "RTKit: #", 9)) { + entries++; + continue; + } + assert(j < sizeof(expect) / sizeof(expect[0])); + assert(!strcmp(lines[i], expect[j])); + j++; + } + assert(j == sizeof(expect) / sizeof(expect[0])); + assert(entries == MAILBOX_ENTRIES); + assert(printed("RTKit: #000@da8656a9: 0000000000000037 0000000000000202")); + assert(printed("RTKit: #127@da8e46a9: 0000000000000037 0000000000000242")); + + /* Still being written: not a complete log. */ + put32(b, DCP_LOG + 12, 0x820 | 0x100); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EAGAIN); + dump("log still being written", b, DCP_SIZE); + assert(printed("holds no complete log (-11)")); + put32(b, DCP_LOG + 12, 0x820); + + /* A record past the buffer, and a container that names itself. */ + put32(b, 0x44 + 0x14 + 0x18 + 16, DCP_SIZE); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EINVAL); + dump("record past the buffer", b, DCP_SIZE); + assert(printed("holds no complete log (-22)")); + put32(b, 0x44 + 0x14 + 0x18 + 16, 0); + assert(apple_rtkit_crashlog_locate(b, DCP_SIZE, &log_offset, &id, &depth) == -EINVAL); + /* The log's section walk is bounded by the log, not the buffer. */ + dcp_log(b); + put32(b, DCP_LOG + 8, 0x20 + 0x60); + dump("log cut after its first section", b, DCP_SIZE); + assert(printed("End of crashlog reached but no footer present") && !printed("Message")); +} + +/* A bare log: a header, then sections built by the caller, then optionally a footer. */ +static u8 logbuf[0x1000]; + +static size_t bare(void) +{ + memset(logbuf, 0, sizeof(logbuf)); + return 0x20; +} + +static unsigned int finish(const char *name, size_t off, bool footer) +{ + size_t size = off + (footer ? 0x20 : 0); + + if (footer) + clhe(logbuf, off, size, 0); + clhe(logbuf, 0, size, 0); + return dump(name, logbuf, size); +} + +static void test_malformed(void) +{ + size_t off, body; + unsigned int i; + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + section(logbuf, off, "Csta", 0x100, 0); + finish("zero section size", off + 0x30, true); + assert(printed("section 43737461 at 0x40 has a bad size 0x0") && n_lines == 2); + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + section(logbuf, off, "Cmbx", 0x300, 0x10000); + finish("oversized section", off + 0x40, true); + assert(printed("section 436d6278 at 0x40 has a bad size 0x10000")); + + off = bare(); + put32(logbuf, off, FOURCC("Ctim")); + finish("truncated section header", off + 8, false); + assert(printed("section header at 0x20 is truncated") && n_lines == 1); + + off = section(logbuf, bare(), "Ctim", 0x100, 0x20); + finish("section, then the footer", off, true); + assert(printed("Crash time: 0") && ended() && n_lines == 2); + + /* Without a footer the last section may not be decoded: its decoder reads 16 bytes on. */ + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 1); + memset(logbuf + body + 20, 'm', 0x40 - 20); + finish("section reaching the end of a log without a footer", off, false); + assert(printed("End of crashlog reached but no footer present") && n_lines == 1); + + for (u32 size = 16; size < 28; size++) { + off = section(logbuf, bare(), "Cmbx", 0x300, size); + finish("short mailbox section", off, true); + assert(printed("Mailbox section too small") && ended()); + } + off = section(logbuf, bare(), "Cmbx", 0x300, 28 + 2 * 0x18); + finish("two-entry mailbox section", off, true); + assert(printed("RTKit: #001@") && !printed("#002") && ended()); + + body = bare(); + off = section(logbuf, body, "Crl4", 0x100, 0x378); + put32(logbuf, body + 0x14, 0xd8); + put64(logbuf, body + 0x110, 0xfffffe0007001234ULL); + put64(logbuf, body + 0x360, 0xdead0000ULL); + for (i = 0; i < 28; i++) + put64(logbuf, body + 0x18 + i * 8, 0x1000 + i); + finish("whole Crl4 section", off, true); + assert(printed("Exception dump (Crl4):") && printed("PC = 0xfffffe0007001234") && + printed("FAR = 0xdead0000") && + printed("x24-x27 = 0000000000001018 0000000000001019 000000000000101a 000000000000101b") && + ended()); + + off = section(logbuf, bare(), "Crl4", 0x100, 0x378); + finish("truncated Crl4 section", 0x20 + 0x100, true); + assert(printed("section 43726c34 at 0x20 has a bad size 0x378")); + + off = section(logbuf, bare(), "Crl4", 0x100, 0x40); + finish("short Crl4 section", off, true); + assert(printed("Crl4 section too small: 0x40") && ended()); + + body = bare(); + off = section(logbuf, body, "Crl4", 0x100, 0x378); + put32(logbuf, body + 0x10, 1 << 16); + put32(logbuf, body + 0x14, 0xd8); + finish("undecodable Crl4 frame", off, true); + assert(printed("Crl4 frame not decodable") && ended()); + + /* Strings without a terminator, against the end of the log. */ + body = bare(); + off = section(logbuf, body, "Cver", 0x300, 0x40); + memset(logbuf + body + 32, 'v', 0x40 - 32); + put32(logbuf, off, FOURCC("CLHE")); + memset(logbuf + off + 4, 0x55, 12); + clhe(logbuf, 0, off + 16, 0); + dump("unterminated version", logbuf, off + 16); + assert(!strcmp(lines[0], "RTKit: Version: vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvEHLCUUUUUUUUUUUU")); + + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 1); + memset(logbuf + body + 20, 'm', 0x40 - 20); + put32(logbuf, off, FOURCC("CLHE")); + memset(logbuf + off + 4, 0x55, 12); + clhe(logbuf, 0, off + 16, 0); + dump("unterminated message", logbuf, off + 16); + assert(!strcmp(lines[0], "RTKit: Message (id=1): mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmEHLCUUUUUUUUUUUU")); + + body = bare(); + off = section(logbuf, body, "Cstr", 0x101, 0x40); + put32(logbuf, body + 16, 2); + strcpy((char *)logbuf + body + 20, "first\nsecond\n"); + finish("message lines", off, true); + assert(printed("Message (id=2): first") && printed("Message (id=2): second") && ended()); + + /* Buffers too short for a header, and logs larger than their buffer. */ + for (size_t size = 0; size < 0x20; size += 4) { + memset(logbuf, 0, sizeof(logbuf)); + clhe(logbuf, 0, 0x20, 0); + dump("short buffer", logbuf, size); + assert(printed("Crashlog buffer too small")); + } + clhe(logbuf, 0, 0x40, 0); + dump("log larger than its buffer", logbuf, 0x20); + assert(printed("Crashlog size (40) is too large")); + memset(logbuf, 0, sizeof(logbuf)); + put32(logbuf, 0, FOURCC("CLGH")); + dump("container header only", logbuf, 0x20); + assert(printed("holds no complete log (-22)")); +} + +static int test_file(const char *path) +{ + FILE *f = fopen(path, "rb"); + size_t size; + u8 *b; + + assert(f); + fseek(f, 0, SEEK_END); + size = ftell(f); + rewind(f); + b = malloc(size); + assert(b && fread(b, 1, size, f) == size); + fclose(f); + dump(path, b, size); + free(b); + return 0; +} + +int main(int argc, char **argv) +{ + assert(FOURCC("CLHE") == APPLE_RTKIT_CRASHLOG_HEADER_FOURCC); + assert(FOURCC("CLGH") == APPLE_RTKIT_CRASHLOG_CONTAINER_FOURCC); + if (argc > 1) + return test_file(argv[1]); + test_dcp_log(); + test_malformed(); + printf("PASS rtkit crashlog: container log decoded; malformed logs refused without a loop or an over-read\n"); + return 0; +} +''' + + +def definitions(header): + """The crashlog fourccs and header struct of rtkit-internal.h.""" + found = re.findall(r'^#define APPLE_RTKIT_CRASHLOG_\w+_FOURCC .*$', header, re.M) + found += re.findall(r'^struct apple_rtkit_crashlog_header \{.*?^\};\n(?:\nstatic_assert\(.*?\);)?', + header, re.M | re.S) + assert len(found) == 3, found + return '\n'.join(found) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ref', help='Git revision of the parser to exercise') + parser.add_argument('--out', type=Path) + parser.add_argument('--mutation', choices=sorted(MUTATIONS)) + parser.add_argument('--log', type=Path, help='a raw crashlog buffer to dump') + args = parser.parse_args() + + def read(path): + if args.ref: + return subprocess.check_output(['git', 'show', f'{args.ref}:{path}'], cwd=ROOT, text=True) + return (ROOT / path).read_text() + source, header = read(SOURCE), read(HEADER) + if args.mutation: + old, new = MUTATIONS[args.mutation] + # A mutation that no longer matches the parser would test nothing. + assert source.count(old) == 1, 'stale mutation: ' + args.mutation + source = source.replace(old, new) + + def run(out): + out.mkdir(parents=True, exist_ok=True) + (out / 'linux').mkdir(exist_ok=True) + (out / 'rtkit-crashlog.c').write_text(source) + (out / 'rtkit-internal.h').write_text(STUB.replace('@DEFINITIONS@', definitions(header))) + (out / 'linux/unaligned.h').write_text(UNALIGNED) + (out / 'main.c').write_text(MAIN) + exe = out / 'crashlog-tests' + compiler = subprocess.run(['clang', '-std=gnu11', '-O1', '-g', '-Wall', '-Werror', '-Wno-unused-function', + '-fsanitize=address,undefined', '-fno-sanitize-recover=all', + '-I', str(out), str(out / 'rtkit-crashlog.c'), str(out / 'main.c'), + '-o', str(exe)], capture_output=True, text=True) + (out / 'compile.log').write_text(compiler.stdout + compiler.stderr) + if compiler.returncode: + print(compiler.stdout + compiler.stderr) + compiler.check_returncode() + env = dict(os.environ, ASAN_OPTIONS='abort_on_error=1:detect_leaks=0', UBSAN_OPTIONS='print_stacktrace=1') + result = subprocess.run([str(exe)] + ([str(args.log.resolve())] if args.log else []), + capture_output=True, text=True, env=env) + receipt = {'source_sha256': hashlib.sha256(source.encode()).hexdigest(), 'ref': args.ref, + 'mutation': args.mutation, 'log': str(args.log) if args.log else None, + 'exit': result.returncode} + (out / 'results.json').write_text(json.dumps(dict(receipt, stdout=result.stdout, + stderr=result.stderr), indent=2) + '\n') + print(result.stdout + result.stderr) + print(json.dumps(receipt)) + return result.returncode != 0 + + if args.out: + return run(args.out) + with tempfile.TemporaryDirectory(prefix='rtkit-crashlog-') as out: + return run(Path(out)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..b8ea5fdd70b11 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -145,6 +145,9 @@ def mutations(flag, names, base, expect=KILL): + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', 'offset-sign', 'event-bit', 'wrong-base'), out()), 'qspi-nvram-tests.py': [(out('--output'), PASS)], + 'rtkit-crashlog-tests.py': [(out(), PASS)] + + mutations('--mutation', ('buffer-header', 'section-header', 'section-size', 'footer-room', + 'mailbox-minimum', 'version-string', 'message-string'), out()), 'rtkit-oslog-tests.py': [(out(), PASS)] + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), 't6030-clock-tests.py': [(out(), PASS)], From 77d34e19e580bd5ef39586f9d2705d413db3b3c1 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Sun, 25 Oct 2026 21:29:40 -0400 Subject: [PATCH 023/392] soc: apple: sep: preserve backing-store open errors Signed-off-by: Satya Benson --- drivers/soc/apple/shim.rs | 25 +-- drivers/soc/apple/store_shim.c | 16 +- .../tests/test_store_open_errors.py | 203 ++++++++++++++++++ 3 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 tools/aurora-sep/tests/test_store_open_errors.py diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 6d546a5603f37..608a3cba4929d 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -51,11 +51,9 @@ fn result_of(ret: c_long) -> Result { impl StoreFile { pub(crate) fn open(path: &CStr) -> Result { - // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. - let handle = unsafe { sep_store_open(path.as_char_ptr()) }; - if handle.is_null() { - return Err(ENOENT); - } + // SAFETY: `path` is NUL-terminated; the shim returns a live file or + // an error pointer, as required by `from_err_ptr`. + let handle = unsafe { kernel::error::from_err_ptr(sep_store_open(path.as_char_ptr()))? }; Ok(StoreFile { handle }) } @@ -73,11 +71,10 @@ impl StoreFile { } pub(crate) fn open_trunc(path: &CStr) -> Result { - // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. - let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; - if handle.is_null() { - return Err(ENOENT); - } + // SAFETY: `path` is NUL-terminated; the shim returns a live file or + // an error pointer, as required by `from_err_ptr`. + let handle = + unsafe { kernel::error::from_err_ptr(sep_store_open_trunc(path.as_char_ptr()))? }; Ok(StoreFile { handle }) } @@ -91,11 +88,9 @@ impl StoreFile { } pub(crate) fn open_readonly(path: &CStr) -> Result { - // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. - let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; - if handle.is_null() { - return Err(ENOENT); - } + // SAFETY: `path` is NUL-terminated; the shim returns a live file or + // an error pointer, as required by `from_err_ptr`. + let handle = unsafe { kernel::error::from_err_ptr(sep_store_open_ro(path.as_char_ptr()))? }; Ok(StoreFile { handle }) } diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 77319a4702164..c1773fa2b6412 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -50,9 +50,7 @@ static struct file *open_as_kernel(const char *path, int flags, umode_t mode) /* Opens the store, creating if absent. Mode 0600: the file is the driver's own. */ void *sep_store_open(const char *path) { - struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_LARGEFILE, 0600); - - return IS_ERR(f) ? NULL : f; + return open_as_kernel(path, O_RDWR | O_CREAT | O_LARGEFILE, 0600); } /* @@ -78,9 +76,7 @@ void *sep_store_open_host(const char *path, bool *created) */ void *sep_store_open_trunc(const char *path) { - struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); - - return IS_ERR(f) ? NULL : f; + return open_as_kernel(path, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); } /* Admission check only: never create or truncate an existing record. */ @@ -92,14 +88,12 @@ void *sep_store_open_writable_existing(const char *path) } /* - * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so - * the seed cannot be created, truncated or written by mistake. + * Opens an existing file read-only, returning an error pointer on failure. + * O_RDONLY and no O_CREAT: the seed cannot be created, truncated or written. */ void *sep_store_open_ro(const char *path) { - struct file *f = open_as_kernel(path, O_RDONLY | O_LARGEFILE, 0); - - return IS_ERR(f) ? NULL : f; + return open_as_kernel(path, O_RDONLY | O_LARGEFILE, 0); } void *sep_store_open_block(const char *path, int writable) diff --git a/tools/aurora-sep/tests/test_store_open_errors.py b/tools/aurora-sep/tests/test_store_open_errors.py new file mode 100644 index 0000000000000..b55626f14f536 --- /dev/null +++ b/tools/aurora-sep/tests/test_store_open_errors.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: MIT +"""The regular-file C/Rust boundary must not turn an I/O error into absence. + +Compile the actual C wrapper bodies with a fake open_as_kernel(). No kernel, +files, enclave, or stored identity is opened. Rust assertions cover the source +wiring to the same from_err_ptr API already used by open_host(). When rustc is +available, also compile the unchanged Rust module against a tiny kernel API +facade and those C wrappers; this is a userspace boundary fixture, not a kernel +build or a real filesystem test. +""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +C_SOURCE = ROOT / 'drivers/soc/apple/store_shim.c' +RUST_SOURCE = ROOT / 'drivers/soc/apple/shim.rs' +OPENS = ('sep_store_open', 'sep_store_open_trunc', 'sep_store_open_ro') + + +def function(source, signature): + start = source.index(signature) + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +PREFIX = r''' +#include +#include +#include +#include +#ifndef O_LARGEFILE +#define O_LARGEFILE 0 +#endif +struct file { int value; }; +static struct file live_file; +static struct file *next_result; +static struct file *open_as_kernel(const char *path, int flags, unsigned mode) +{ + (void)path; (void)flags; (void)mode; + return next_result; +} +''' + +RUNNER = r''' +int main(void) +{ + void *(*opens[])(const char *) = { + sep_store_open, sep_store_open_trunc, sep_store_open_ro + }; + const int errors[] = { ENOENT, EACCES, EIO, ENOMEM, EROFS, ELOOP }; + for (unsigned i = 0; i < sizeof(opens) / sizeof(opens[0]); ++i) { + next_result = &live_file; + if (opens[i]("unused") != &live_file) return 2; + for (unsigned j = 0; j < sizeof(errors) / sizeof(errors[0]); ++j) { + next_result = (struct file *)(intptr_t)-errors[j]; + if (opens[i]("unused") != next_result) { + fprintf(stderr, "wrapper %u lost errno %d\n", i, errors[j]); + return 1; + } + } + } + puts("21 open results preserved"); + return 0; +} +''' + + +class StoreOpenErrors(unittest.TestCase): + def native(self, negative_control=False): + compiler = shutil.which('cc') + if compiler is None: + self.skipTest('C compiler unavailable') + source = C_SOURCE.read_text() + bodies = '\n'.join(function(source, 'void *' + name + '(') for name in OPENS) + prefix = PREFIX + if negative_control: + # Reintroduce the old error-pointer-to-NULL collapse. The same + # executable must reject it, rather than merely compile it. + prefix += r''' +static void *collapse(struct file *f) +{ + return (uintptr_t)f >= (uintptr_t)-4095 ? NULL : f; +} +''' + bodies, changed = re.subn(r'return (open_as_kernel\([^;]+\));', r'return collapse(\1);', bodies) + self.assertEqual(changed, 3) + with tempfile.TemporaryDirectory(prefix='sep-open-errors-') as tmp: + root = Path(tmp) + (root / 'test.c').write_text(prefix + bodies + RUNNER) + subprocess.run([compiler, '-std=c11', '-Wall', '-Wextra', '-Werror', + str(root / 'test.c'), '-o', str(root / 'test')], check=True) + return subprocess.run([str(root / 'test')], capture_output=True, text=True) + + def test_native_open_results(self): + result = self.native() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('21 open results preserved', result.stdout) + + def test_old_error_collapse_is_rejected(self): + result = self.native(negative_control=True) + self.assertEqual(result.returncode, 1) + self.assertIn('lost errno', result.stderr) + + def test_rust_propagates_each_regular_open_error(self): + source = RUST_SOURCE.read_text() + for method, call in zip(('open', 'open_trunc', 'open_readonly'), OPENS): + with self.subTest(method=method): + body = function(source, 'pub(crate) fn ' + method + '(') + self.assertIn('kernel::error::from_err_ptr(' + call + '(path.as_char_ptr()))?', body) + self.assertNotIn('is_null()', body) + self.assertNotIn('Err(ENOENT)', body) + + def test_block_device_open_contract_unchanged(self): + body = function(C_SOURCE.read_text(), 'void *sep_store_open_block(') + self.assertRegex(body, r'if \(IS_ERR\(f\)\)\s+return NULL;') + + def test_native_rust_c_boundary(self): + compiler, rustc = shutil.which('cc'), shutil.which('rustc') + if compiler is None or rustc is None: + self.skipTest('C and Rust compilers required') + bodies = '\n'.join(function(C_SOURCE.read_text(), 'void *' + name + '(') for name in OPENS) + c = PREFIX + bodies + r''' +static unsigned closes; +void fixture_result(int error) +{ + next_result = error ? (struct file *)(intptr_t)-error : &live_file; +} +void sep_store_close(void *file) +{ + if (file == &live_file) ++closes; +} +unsigned fixture_closes(void) { return closes; } +''' + rust = r''' +#![allow(dead_code)] +extern crate self as kernel; +pub mod error { + #[derive(Debug, PartialEq)] + pub struct Error(pub i32); + impl Error { pub fn from_errno(n: i32) -> Self { Self(n) } } + pub type Result = core::result::Result; + pub const EIO: Error = Error(-5); + pub const ENODEV: Error = Error(-19); + pub fn to_result(n: i32) -> Result { if n < 0 { Err(Error(n)) } else { Ok(()) } } + pub unsafe fn from_err_ptr(p: *mut T) -> Result<*mut T> { + if p as usize >= usize::MAX - 4094 { Err(Error(p as isize as i32)) } else { Ok(p) } + } +} +pub mod prelude { + pub use core::ffi::{c_char,c_int,c_void,c_long,c_uint,c_ulong}; + pub use crate::error::{Error, Result, EIO, ENODEV}; + pub struct CStr(pub &'static [u8]); + impl CStr { pub fn as_char_ptr(&self) -> *const c_char { self.0.as_ptr().cast() } } +} +#[path="shim.rs"] mod shim; +extern "C" { fn fixture_result(error: i32); fn fixture_closes() -> u32; } +fn main() { + let path = prelude::CStr(b"unused\0"); + let opens: [fn(&prelude::CStr) -> error::Result; 3] = [ + shim::StoreFile::open, shim::StoreFile::open_trunc, shim::StoreFile::open_readonly + ]; + for open in opens { + unsafe { fixture_result(0); } + drop(open(&path).expect("successful handle")); + for errno in [2, 13, 5, 12, 30, 40] { + unsafe { fixture_result(errno); } + match open(&path) { + Err(e) if e == error::Error(-errno) => (), + _ => panic!("Rust lost an open error"), + } + } + } + if unsafe { fixture_closes() } != 3 { panic!("only successful handles may close"); } + println!("21 Rust/C results preserved; 3 successful handles closed"); +} +''' + with tempfile.TemporaryDirectory(prefix='sep-open-rust-') as tmp: + root = Path(tmp) + (root / 'test.c').write_text(c) + (root / 'test.rs').write_text(rust) + shutil.copyfile(RUST_SOURCE, root / 'shim.rs') + subprocess.run([compiler, '-std=c11', '-Wall', '-Wextra', '-Werror', '-c', + str(root / 'test.c'), '-o', str(root / 'test.o')], check=True) + subprocess.run([rustc, '--edition=2021', '-C', 'opt-level=1', + '-C', 'link-arg=' + str(root / 'test.o'), + str(root / 'test.rs'), '-o', str(root / 'test')], check=True) + result = subprocess.run([str(root / 'test')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('21 Rust/C results preserved', result.stdout) + + +if __name__ == '__main__': + unittest.main() From 91c26b2035c7f869c0c306c0c273fe144df77f30 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Sun, 25 Oct 2026 21:46:02 -0400 Subject: [PATCH 024/392] soc: apple: sep: refuse unreadable and partial identity records Signed-off-by: Satya Benson --- drivers/soc/apple/bio.rs | 15 +- drivers/soc/apple/keybag.rs | 9 + drivers/soc/apple/refkey.rs | 14 +- .../tests/test_identity_preservation.py | 219 ++++++++++++++++++ 4 files changed, 247 insertions(+), 10 deletions(-) create mode 100644 tools/aurora-sep/tests/test_identity_preservation.py diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 573830b7973d0..1c01e3ca4a19e 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -308,22 +308,19 @@ impl IdentityIndex { let Some(raw) = store.read(&identity_key())? else { return Ok(index); }; - if raw.len() < 4 { - return Ok(index); + if raw.len() < 8 { + return Err(EINVAL); } if u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]) != INDEX_FORMAT { return Err(EINVAL); } - if raw.len() < 8 { - return Ok(index); - } let count = u32::from_le_bytes([raw[4], raw[5], raw[6], raw[7]]) as usize; let stride = UUID_LEN + LABEL_LEN; - for i in 0..count.min(MAX_IDENTITIES) { + if count > MAX_IDENTITIES || raw.len() != 8 + count * stride { + return Err(EINVAL); + } + for i in 0..count { let at = 8 + i * stride; - if at + stride > raw.len() { - break; - } let mut entry = Identity::ZERO; entry.uuid.copy_from_slice(&raw[at..at + UUID_LEN]); entry diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs index 919035b1a09de..22dc6d27929eb 100644 --- a/drivers/soc/apple/keybag.rs +++ b/drivers/soc/apple/keybag.rs @@ -181,6 +181,15 @@ pub(crate) fn read(slot: Slot) -> Result { head[OFF_STATE + 3], ]); if state == STATE_REFUSED { + let expected_crc = crc16_ccitt_false(&checksum_input(state, &[], &[0; UUID_LEN], &[])?); + let crc = u16::from_le_bytes([head[OFF_CRC], head[OFF_CRC + 1]]); + if size != HEADER as u64 + || crc != expected_crc + || head[OFF_LEN..OFF_CRC].iter().any(|byte| *byte != 0) + || head[OFF_CRC + 2..].iter().any(|byte| *byte != 0) + { + return Err(EINVAL); + } return Ok(State::Absent(NoStoredKeyBag { slot })); } let Some(provenance) = UuidProvenance::from_state(state) else { diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs index e505fb8d03bea..ebb7df218ea66 100644 --- a/drivers/soc/apple/refkey.rs +++ b/drivers/soc/apple/refkey.rs @@ -179,7 +179,19 @@ impl SepData { return; } - if let Ok(f) = shim::StoreFile::open_readonly(path) { + let existing = match shim::StoreFile::open_readonly(path) { + Ok(file) => Some(file), + Err(ENOENT) => None, + Err(error) => { + dev_err!( + self.dev, + "sks: cannot read existing machine ref-key: {:?}; refusing replacement\n", + error + ); + return; + } + }; + if let Some(f) = existing { let sz = f.size().unwrap_or(0); if (1..=8192).contains(&sz) { let mut blob: KVec = KVec::new(); diff --git a/tools/aurora-sep/tests/test_identity_preservation.py b/tools/aurora-sep/tests/test_identity_preservation.py new file mode 100644 index 0000000000000..6308c59d40dd4 --- /dev/null +++ b/tools/aurora-sep/tests/test_identity_preservation.py @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: MIT +"""Compile production identity readers against in-memory, device-free seams.""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def body(source, signature): + start = source.index(signature) + at = source.index('{', start) + depth, end = 1, at + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +COMMON = r''' +#![allow(dead_code, unused_variables, unused_imports, unused_macros, non_upper_case_globals)] +use std::cell::{Cell, RefCell}; +use std::ops::{Deref,DerefMut}; +use std::ffi::CStr; +type Result = std::result::Result; +const ENOENT:i32=2; const EIO:i32=5; const EINVAL:i32=22; +const ENOTSYNC:i32=52; const EEXIST:i32=17; const GFP_KERNEL:u32=0; +#[derive(Clone)] struct KVec(Vec); +impl KVec { + fn new()->Self {Self(Vec::new())} + fn with_capacity(n:usize,_:u32)->Result {Ok(Self(Vec::with_capacity(n)))} + fn push(&mut self,v:T,_:u32)->Result<()> {self.0.push(v);Ok(())} +} +impl KVec { + fn resize(&mut self,n:usize,v:T,_:u32)->Result<()> {self.0.resize(n,v);Ok(())} + fn extend_from_slice(&mut self,v:&[T],_:u32)->Result<()> {self.0.extend_from_slice(v);Ok(())} +} +impl Deref for KVec {type Target=Vec; fn deref(&self)->&Vec{&self.0}} +impl DerefMut for KVec {fn deref_mut(&mut self)->&mut Vec{&mut self.0}} +struct Secret(KVec); +impl Deref for Secret {type Target=[u8];fn deref(&self)->&[u8]{&self.0.0}} +macro_rules! dev_err {($($x:tt)*)=>{()};} +macro_rules! dev_warn {($($x:tt)*)=>{()};} +''' + +REFKEY = r''' +thread_local! {static OPEN_ERROR:Cell=const{Cell::new(0)};} +mod sks {pub type KeyBagHandle=u32;} +mod module_parameters { + pub struct Param; pub static refkey_v2:Param=Param; + impl Param {pub fn value(&self)->&u8{&0}} +} +mod shim { + use super::*; + pub struct StoreFile; + impl StoreFile { + pub fn open_readonly(_: &CStr)->Result{OPEN_ERROR.with(|e|if e.get()==0{Ok(Self)}else{Err(e.get())})} + pub fn open(_: &CStr)->Result{panic!("fixture never permits persistence")} + pub fn size(&self)->Result{Ok(1)} + pub fn read_exact(&self,_:u64,_:&mut[u8])->Result<()>{Ok(())} + pub fn write_all(&self,_:u64,_:&[u8])->Result<()>{panic!("write")} + pub fn sync(&self)->Result<()>{panic!("sync")} + } +} +struct Cache; +impl Cache {fn lock(&self)->Option<()>{None}} +struct SepData {machine_refkey:Cache,creates:Cell} +impl SepData { + fn sks_create_refkey(&self,_:u32,_:&[u8])->Option>{self.creates.set(self.creates.get()+1);None} + fn cache_machine_refkey(&self,_:KVec)->bool{true} + METHOD +} +fn main(){ + for error in [0,ENOENT,EIO,13,12,30,40] { + OPEN_ERROR.with(|e|e.set(error)); + let s=SepData{machine_refkey:Cache,creates:Cell::new(0)}; + s.sks_machine_refkey(0,&[]); + assert_eq!(s.creates.get(),usize::from(error==ENOENT),"only absence permits ref-key creation"); + } + println!("refkey cases passed"); +} +''' + +INDEX = r''' +const UUID_LEN:usize=16; const LABEL_LEN:usize=128;const MAX_IDENTITIES:usize=32; +const INDEX_FORMAT:u32=2; +#[derive(Clone)] struct Identity{uuid:[u8;UUID_LEN],label:[u8;LABEL_LEN]} +impl Identity {const ZERO:Self=Self{uuid:[0;UUID_LEN],label:[0;LABEL_LEN]};} +struct Store(Option>); +impl Store {fn read(&mut self,_:&())->Result>>{Ok(self.0.clone().map(KVec))}} +fn identity_key(){} +struct IdentityIndex{entries:KVec} +impl IdentityIndex{ + fn new()->Self{Self{entries:KVec::new()}} + METHOD +} +fn image(count:u32,n:usize)->Vec{ + let mut v=Vec::from(INDEX_FORMAT.to_le_bytes());v.extend(count.to_le_bytes()); + v.resize(8+n*(UUID_LEN+LABEL_LEN),0);v +} +fn main(){ + assert!(IdentityIndex::load(&mut Store(None)).unwrap().entries.is_empty()); + for n in [0,1,MAX_IDENTITIES]{assert_eq!(IdentityIndex::load(&mut Store(Some(image(n as u32,n)))).unwrap().entries.len(),n);} + for len in 0..8 {assert!(IdentityIndex::load(&mut Store(Some(vec![0;len]))).is_err(),"short index admitted");} + let valid=image(1,1); + for len in 8..valid.len(){assert!(IdentityIndex::load(&mut Store(Some(valid[..len].to_vec()))).is_err(),"truncated index admitted");} + for v in [image(2,1),image((MAX_IDENTITIES+1) as u32,MAX_IDENTITIES+1)] { + assert!(IdentityIndex::load(&mut Store(Some(v))).is_err(),"count mismatch admitted"); + } + let mut trailing=image(0,0);trailing.push(0); + assert!(IdentityIndex::load(&mut Store(Some(trailing))).is_err(),"trailing index bytes admitted"); + println!("index cases passed"); +} +''' + +KEYBAG = r''' +const MAGIC:[u8;16]=*b"APPLE-SEP-KBAG01"; +const VERSION:u32=2;const STATE_REFUSED:u32=2;const UUID_LEN:usize=16; +const OFF_VERSION:usize=16;const OFF_STATE:usize=20;const OFF_LEN:usize=24; +const OFF_CRC:usize=28;const OFF_UUID:usize=32;const OFF_SECRET_LEN:usize=48; +const HEADER:usize=52;const MAX_WRAPPED:usize=65536; +#[derive(Clone,Copy)] enum Slot{Identity} +impl Slot{fn path(self)->&'static CStr{c"unused"}} +enum UuidProvenance{Generated,Readback} +impl UuidProvenance{fn from_state(n:u32)->Option{match n{1=>Some(Self::Generated),3=>Some(Self::Readback),_=>None}}} +struct NoStoredKeyBag{slot:Slot} +struct StoredKeyBag{wrapped:KVec,uuid:[u8;16],secret:Secret,provenance:UuidProvenance} +enum State{Present(StoredKeyBag),Absent(NoStoredKeyBag)} +thread_local!{static IMAGE:RefCell>=const{RefCell::new(Vec::new())};} +mod shim{ + use super::*;pub struct StoreFile; + impl StoreFile{ + pub fn open_readonly(_: &CStr)->Result{Ok(Self)} + pub fn size(&self)->Result{Ok(IMAGE.with(|v|v.borrow().len() as u64))} + pub fn read_exact(&self,at:u64,out:&mut[u8])->Result<()>{ + IMAGE.with(|v|{let v=v.borrow();let Some(s)=v.get(at as usize..at as usize+out.len())else{return Err(EIO)};out.copy_from_slice(s);Ok(())}) + } + } +} +@@CRC@@ +CHECKSUM +METHOD +fn refused()->Vec{ + let mut v=vec![0;HEADER];v[..16].copy_from_slice(&MAGIC); + v[OFF_VERSION..OFF_VERSION+4].copy_from_slice(&VERSION.to_le_bytes()); + v[OFF_STATE..OFF_STATE+4].copy_from_slice(&STATE_REFUSED.to_le_bytes()); + let crc=crc16_ccitt_false(&checksum_input(STATE_REFUSED,&[],&[0;16],&[]).unwrap()); + v[OFF_CRC..OFF_CRC+2].copy_from_slice(&crc.to_le_bytes());v +} +fn main(){ + let good=refused();IMAGE.with(|v|*v.borrow_mut()=good.clone()); + assert!(matches!(read(Slot::Identity),Ok(State::Absent(_)))); + for at in OFF_LEN..HEADER { + let mut bad=good.clone();bad[at]^=1;IMAGE.with(|v|*v.borrow_mut()=bad); + assert!(read(Slot::Identity).is_err(),"noncanonical refusal admitted at {}",at); + } + let mut long=good;long.push(0);IMAGE.with(|v|*v.borrow_mut()=long); + assert!(read(Slot::Identity).is_err(),"trailing refusal data admitted"); + let wrapped=[1,2,3];let secret=[4,5];let uuid=[6;16]; + let mut committed=refused();committed[OFF_STATE..OFF_STATE+4].copy_from_slice(&3u32.to_le_bytes()); + committed[OFF_LEN..OFF_LEN+4].copy_from_slice(&3u32.to_le_bytes()); + committed[OFF_SECRET_LEN..OFF_SECRET_LEN+4].copy_from_slice(&2u32.to_le_bytes()); + committed[OFF_UUID..OFF_UUID+16].copy_from_slice(&uuid); + let crc=crc16_ccitt_false(&checksum_input(3,&wrapped,&uuid,&secret).unwrap()); + committed[OFF_CRC..OFF_CRC+2].copy_from_slice(&crc.to_le_bytes()); + committed.extend(wrapped);committed.extend(secret);IMAGE.with(|v|*v.borrow_mut()=committed); + assert!(matches!(read(Slot::Identity),Ok(State::Present(_))),"valid committed bag refused"); + println!("keybag cases passed"); +} +''' + + +@unittest.skipUnless(shutil.which('rustc'), 'rustc required') +class Preservation(unittest.TestCase): + def run_rust(self, text, should_pass=True): + with tempfile.TemporaryDirectory(prefix='sep-identity-readers-') as tmp: + p=Path(tmp);(p/'test.rs').write_text(COMMON+text) + subprocess.run(['rustc','--edition=2021',str(p/'test.rs'),'-o',str(p/'test')],check=True) + result=subprocess.run([str(p/'test')],capture_output=True,text=True) + if should_pass: + self.assertEqual(result.returncode,0,result.stderr) + else: + self.assertNotEqual(result.returncode,0) + self.assertIn('panicked',result.stderr) + + def test_refkey_requires_genuine_absence(self): + text=(ROOT/'refkey.rs').read_text() + method=body(text,' pub(crate) fn sks_machine_refkey(') + self.run_rust(REFKEY.replace('METHOD',method)) + old=re.sub(r' let existing = match.*? if let Some\(f\) = existing \{', + ' if let Ok(f) = shim::StoreFile::open_readonly(path) {',method,flags=re.S) + self.assertNotEqual(old,method) + self.run_rust(REFKEY.replace('METHOD',old),should_pass=False) + + def test_index_requires_complete_record(self): + method=body((ROOT/'bio.rs').read_text(),' pub(crate) fn load(store:') + self.run_rust(INDEX.replace('METHOD',method)) + old=method.replace('if raw.len() < 8 {\n return Err(EINVAL);','if raw.len() < 8 {\n return Ok(index);',1) + self.assertNotEqual(old,method) + self.run_rust(INDEX.replace('METHOD',old),should_pass=False) + + def test_refused_keybag_requires_canonical_writer_record(self): + text=(ROOT/'keybag.rs').read_text() + method=body(text,'pub(crate) fn read(slot:') + crc=body((ROOT/'store.rs').read_text(),'pub(crate) const fn crc16_ccitt_false(') + checksum=body(text,'fn checksum_input(') + make=lambda m:KEYBAG.replace('METHOD',m).replace('CHECKSUM',checksum).replace('@@CRC@@',crc) + self.run_rust(make(method)) + start=method.index(' if state == STATE_REFUSED {') + stop=start+len(body(method[start:],' if state == STATE_REFUSED {')) + old=method[:start]+' if state == STATE_REFUSED { return Ok(State::Absent(NoStoredKeyBag { slot })); }'+method[stop:] + self.run_rust(make(old),should_pass=False) + + +if __name__=='__main__': + unittest.main() From ef4c4274c4232dc69f173edcdb3fdead676284f1 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Sun, 25 Oct 2026 22:22:26 -0400 Subject: [PATCH 025/392] soc: apple: sep: distinguish unreadable catacombs from missing ones catacomb::read() mapped every failure to None, which the enrollment, restore, deletion and owner-probe paths read as "no saved catacomb". Return Result> so that only a missing file reads as absent, add is_saved()/is_absent() for the callers, and make read_stored() propagate read errors so restore reports Failed rather than NoStoredFile. Test both callers that run on these paths: restore_catacomb() must fail without contacting the enclave on an unreadable catacomb or host store, and the owner probe must stop before its enclave check when the owner or master file cannot be read. Drop a source-text assertion made redundant by the native boundary test. Signed-off-by: Satya Benson --- drivers/soc/apple/catacomb.rs | 43 ++++-- drivers/soc/apple/sbio.rs | 71 +++++---- .../tests/test_identity_preservation.py | 145 +++++++++++++++++- .../tests/test_store_open_errors.py | 14 +- 4 files changed, 220 insertions(+), 53 deletions(-) diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs index d71b415fcb321..bae0396a5b9c5 100644 --- a/drivers/soc/apple/catacomb.rs +++ b/drivers/soc/apple/catacomb.rs @@ -64,28 +64,45 @@ pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { file.sync() } -pub(crate) fn read(kind: u8) -> Option> { - let path = path_for(kind)?; - let file = shim::StoreFile::open_readonly(path).ok()?; - let size = file.size().ok()?; +/// `Ok(None)` only when the file is missing. A file that exists but cannot be +/// read or fails validation is an error: "cannot tell" must never read as +/// "nothing saved", or a caller may select a fresh context over a saved one. +pub(crate) fn read(kind: u8) -> Result>> { + let path = path_for(kind).ok_or(EINVAL)?; + let file = match shim::StoreFile::open_readonly(path) { + Ok(file) => file, + Err(e) if e == ENOENT => return Ok(None), + Err(e) => return Err(e), + }; + let size = file.size()?; if size < HEADER as u64 { - return None; + return Err(EINVAL); } let mut head = [0u8; HEADER]; - file.read_exact(0, &mut head).ok()?; + file.read_exact(0, &mut head)?; if head[..MAGIC.len()] != MAGIC { - return None; + return Err(EINVAL); } let len = u32::from_le_bytes([head[8], head[9], head[10], head[11]]) as usize; let crc = u16::from_le_bytes([head[12], head[13]]); if len == 0 || len > MAX_CATACOMB || size != (HEADER + len) as u64 { - return None; + return Err(EINVAL); } - let mut blob: KVec = KVec::with_capacity(len, GFP_KERNEL).ok()?; - blob.resize(len, 0, GFP_KERNEL).ok()?; - file.read_exact(HEADER as u64, &mut blob).ok()?; + let mut blob: KVec = KVec::with_capacity(len, GFP_KERNEL)?; + blob.resize(len, 0, GFP_KERNEL)?; + file.read_exact(HEADER as u64, &mut blob)?; if crc16_ccitt_false(&blob) != crc { - return None; + return Err(EINVAL); } - Some(blob) + Ok(Some(blob)) +} + +/// True only for a present, valid file. +pub(crate) fn is_saved(kind: u8) -> bool { + matches!(read(kind), Ok(Some(_))) +} + +/// True only when the file is provably missing. +pub(crate) fn is_absent(kind: u8) -> bool { + matches!(read(kind), Ok(None)) } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 98a048443da18..91158725d5084 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -552,7 +552,7 @@ impl SepData { dev_err!(self.dev, "enrol: opt-in fresh-context owner export failed\n"); return false; } - if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + if !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) { dev_err!(self.dev, "enrol: opt-in fresh context has no owner file\n"); return false; } @@ -638,9 +638,9 @@ impl SepData { return; } if self.templates_restored.load(Relaxed) - || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() - || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_MASTER).is_none() - || crate::catacomb::read(PRIVATE_TYPE_CATACOMB_USER).is_none() + || !crate::catacomb::is_absent(PRIVATE_TYPE_CATACOMB_OWNER) + || !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_MASTER) + || !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_USER) { dev_warn!(self.dev, "owner-probe: preconditions not met; no context selected\n"); return; @@ -776,8 +776,10 @@ impl SepData { crate::sbio::ComponentAction::AlreadyActive => RestoreOutcome::AlreadyActive, crate::sbio::ComponentAction::Unsupported => RestoreOutcome::Failed, crate::sbio::ComponentAction::Load => { - let Some(blob) = self.read_stored(kind, what) else { - return RestoreOutcome::NoStoredFile; + let blob = match self.read_stored(kind, what) { + Ok(Some(blob)) => blob, + Ok(None) => return RestoreOutcome::NoStoredFile, + Err(_) => return RestoreOutcome::Failed, }; let at = crate::sbio::SBIO_SAVED_USER_ID_AT; @@ -894,8 +896,10 @@ impl SepData { } fn restore_lockout(&self) -> RestoreOutcome { - let Some(blob) = self.read_stored(PRIVATE_TYPE_LOCKOUT, c"bio lockout record") else { - return RestoreOutcome::NoStoredFile; + let blob = match self.read_stored(PRIVATE_TYPE_LOCKOUT, c"bio lockout record") { + Ok(Some(blob)) => blob, + Ok(None) => return RestoreOutcome::NoStoredFile, + Err(_) => return RestoreOutcome::Failed, }; let Some(request) = crate::sbio::SbioLoadLockout::new(&blob) else { return RestoreOutcome::Failed; @@ -915,17 +919,26 @@ impl SepData { } } - fn read_stored(&self, kind: u8, _what: &CStr) -> Option> { + /// `Ok(None)` means provably not stored; any read failure is an error so + /// it cannot be mistaken for a missing file. + fn read_stored(&self, kind: u8, what: &CStr) -> Result>> { if crate::catacomb::is_kind(kind) { - if let Some(blob) = crate::catacomb::read(kind) { - return Some(blob); + match crate::catacomb::read(kind) { + Ok(Some(blob)) => return Ok(Some(blob)), + Ok(None) => {} + Err(e) => { + dev_err!(self.dev, "restore: cannot read {}: {:?}\n", what, e); + return Err(e); + } } } match self.with_host_store(|store| store.read(&store::Key::root(kind))) { - Some(Ok(Some(blob))) => Some(blob), - Some(Ok(None)) => None, - Some(Err(_)) => None, - None => None, + Some(Ok(found)) => Ok(found), + Some(Err(e)) => { + dev_err!(self.dev, "restore: cannot read {}: {:?}\n", what, e); + Err(e) + } + None => Ok(None), } } @@ -979,7 +992,7 @@ impl SepData { } if (who.value() == user.value() || who.value() == crate::sbio::CatacombUser::MASTER.value()) - && crate::catacomb::read(kind).is_none() + && !crate::catacomb::is_saved(kind) { dev_err!(self.dev, "enrol: {} has no durable host file\n", what); return false; @@ -993,7 +1006,7 @@ impl SepData { } fn save_all_components_user_first(&self, user: crate::sbio::UserId) -> bool { - if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + if !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) { dev_err!(self.dev, "enrol: opt-in completion has no owner Catacomb file\n"); return false; } @@ -1032,7 +1045,7 @@ impl SepData { if !self.save_catacomb(who, kind, what) { return false; } - if crate::catacomb::read(kind).is_none() { + if !crate::catacomb::is_saved(kind) { dev_err!(self.dev, "enrol: opt-in {} has no durable host file\n", what); return false; } @@ -1088,7 +1101,7 @@ impl SepData { dev_err!(self.dev, "verify: {} persistence failed\n", what); return false; } - if crate::catacomb::read(kind).is_none() { + if !crate::catacomb::is_saved(kind) { dev_err!(self.dev, "verify: {} has no durable host file\n", what); return false; } @@ -3571,7 +3584,7 @@ impl SepData { let kinds = [PRIVATE_TYPE_CATACOMB_OWNER, PRIVATE_TYPE_CATACOMB_USER, PRIVATE_TYPE_CATACOMB_MASTER]; let readable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) - || crate::catacomb::read(kind).is_some()); + || crate::catacomb::is_saved(kind)); let writable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) || crate::catacomb::writable(kind)); let index = self.bio_index.lock(); @@ -3634,13 +3647,17 @@ impl SepData { if !ids.is_empty() { // An existing identity must already have a durable owner file; // selecting SYSTEM here could disturb its live context. - return if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + return if crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) { bio::ok() } else { Err(ENODATA) }; } - if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + if crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) { return bio::ok(); } + // An unreadable owner record is not permission to select a new context. + if !crate::catacomb::is_absent(PRIVATE_TYPE_CATACOMB_OWNER) { + return Err(EIO); + } if !self.keybag_designated.load(Relaxed) || !self.enrol_material.lock().as_ref() .is_some_and(|material| material.snapshot_identity.is_some()) { @@ -3667,7 +3684,7 @@ impl SepData { if !prepared { return Err(EIO); } break; } - if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() + if !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { return Err(EROFS); } @@ -4069,7 +4086,7 @@ impl SepData { // owner Catacomb provides the recovery path even when the bag snapshot // has no binding; export that owner first if this install lacks a file. for kind in [PRIVATE_TYPE_CATACOMB_USER, PRIVATE_TYPE_CATACOMB_MASTER] { - if crate::catacomb::read(kind).is_none() || !crate::catacomb::writable(kind) { + if !crate::catacomb::is_saved(kind) || !crate::catacomb::writable(kind) { dev_err!( self.dev, "bio: not deleting: user/master Catacomb is unavailable or unwritable\n" @@ -4077,7 +4094,7 @@ impl SepData { return Err(EIO); } } - if orphan && crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + if orphan && crate::catacomb::is_absent(PRIVATE_TYPE_CATACOMB_OWNER) { let Some(reply) = self.read_component_states() else { return Err(EIO); }; let state = crate::sbio::ComponentStates::new(&reply) .state_for(crate::sbio::CatacombUser::OWNER.value()); @@ -4089,7 +4106,7 @@ impl SepData { return Err(EIO); } } - if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() + if !crate::catacomb::is_saved(PRIVATE_TYPE_CATACOMB_OWNER) || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { return Err(EIO); @@ -4176,7 +4193,7 @@ impl SepData { ); return Err(EIO); } - if crate::catacomb::read(kind).is_none() { + if !crate::catacomb::is_saved(kind) { dev_err!(self.dev, "bio: the {} has no durable host file\n", what); return Err(EIO); } diff --git a/tools/aurora-sep/tests/test_identity_preservation.py b/tools/aurora-sep/tests/test_identity_preservation.py index 6308c59d40dd4..3f15f10c7d191 100644 --- a/tools/aurora-sep/tests/test_identity_preservation.py +++ b/tools/aurora-sep/tests/test_identity_preservation.py @@ -51,7 +51,7 @@ def body(source, signature): mod sks {pub type KeyBagHandle=u32;} mod module_parameters { pub struct Param; pub static refkey_v2:Param=Param; - impl Param {pub fn value(&self)->&u8{&0}} + impl Param {pub fn value(&self)->u8{0}} } mod shim { use super::*; @@ -172,6 +172,122 @@ def body(source, signature): } ''' +CATACOMB = r''' +const MAGIC:[u8;8]=*b"SEPCTMB1";const HEADER:usize=16;const MAX_CATACOMB:usize=1<<20; +const PRIVATE_TYPE_CATACOMB_MASTER:u8=1;const PRIVATE_TYPE_CATACOMB_OWNER:u8=2;const PRIVATE_TYPE_CATACOMB_USER:u8=3; +thread_local!{static OPEN:Cell=const{Cell::new(0)};static IMAGE:RefCell>=const{RefCell::new(Vec::new())};} +mod shim{ + use super::*;pub struct StoreFile; + impl StoreFile{ + pub fn open_readonly(_: &CStr)->Result{OPEN.with(|e|if e.get()==0{Ok(Self)}else{Err(e.get())})} + pub fn size(&self)->Result{Ok(IMAGE.with(|v|v.borrow().len() as u64))} + pub fn read_exact(&self,at:u64,out:&mut[u8])->Result<()>{ + IMAGE.with(|v|{let v=v.borrow();let Some(s)=v.get(at as usize..at as usize+out.len())else{return Err(EIO)};out.copy_from_slice(s);Ok(())}) + } + } +} +@@CRC@@ +PATH +METHOD +fn image(blob:&[u8])->Vec{ + let mut v=Vec::from(MAGIC);v.extend((blob.len() as u32).to_le_bytes()); + v.extend(crc16_ccitt_false(blob).to_le_bytes());v.extend([0,0]);v.extend(blob);v +} +fn main(){ + let k=PRIVATE_TYPE_CATACOMB_MASTER; + OPEN.with(|e|e.set(ENOENT));assert!(matches!(read(k),Ok(None)),"missing file must be absence"); + for err in [EIO,13,12,30,40]{OPEN.with(|e|e.set(err));assert!(read(k).is_err(),"open error read as absence");} + OPEN.with(|e|e.set(0)); + let good=image(&[1,2,3]);IMAGE.with(|v|*v.borrow_mut()=good.clone()); + assert_eq!(read(k).unwrap().unwrap().0,vec![1,2,3]); + for len in 0..good.len(){IMAGE.with(|v|*v.borrow_mut()=good[..len].to_vec());assert!(read(k).is_err(),"truncated file read as absence or data");} + let mut bad=good.clone();bad[HEADER]^=1;IMAGE.with(|v|*v.borrow_mut()=bad);assert!(read(k).is_err(),"bad checksum admitted"); + let mut bad=good;bad[0]^=1;IMAGE.with(|v|*v.borrow_mut()=bad);assert!(read(k).is_err(),"bad magic admitted"); + println!("catacomb cases passed"); +} +''' + + +CALLERS = r''' +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +macro_rules! dev_info {($($x:tt)*)=>{()};} +const PRIVATE_TYPE_CATACOMB_MASTER:u8=0xF2;const PRIVATE_TYPE_CATACOMB_OWNER:u8=0xF3;const PRIVATE_TYPE_CATACOMB_USER:u8=0xF4; +const SBIO_PROBE_USER_ID:i32=1; +#[derive(Debug,PartialEq)] enum RestoreOutcome{Restored,AlreadyActive,NoStoredFile,Failed,EmptyTolerated} +enum LoadAnswer{StatusZero,ColdTransition,AlreadyActive} +// 0 = valid file, ENOENT = missing, anything else = read error. +thread_local!{ + static FILE:RefCell<[i32;3]>=const{RefCell::new([0;3])}; + static HOST:Cell=const{Cell::new(-1)}; + static ASKED_ENCLAVE:Cell=const{Cell::new(false)}; + static TRANSFERRED:Cell=const{Cell::new(false)}; +} +mod catacomb{ + use super::*; + pub fn is_kind(_:u8)->bool{true} + pub fn read(kind:u8)->Result>>{ + match FILE.with(|f|f.borrow()[(kind-PRIVATE_TYPE_CATACOMB_MASTER) as usize]){ + 0=>Ok(Some(KVec(vec![1,0,0,0]))),ENOENT=>Ok(None),e=>Err(e)} + } + @@IS_SAVED@@ + @@IS_ABSENT@@ +} +mod store{pub struct Key;impl Key{pub fn root(_:u8)->Self{Key}}} +struct Store; +impl Store{fn read(&self,_:&store::Key)->Result>>{match HOST.with(|h|h.get()){0=>Ok(Some(KVec(vec![1,0,0,0]))),ENOENT=>Ok(None),e=>Err(e)}}} +mod module_parameters{pub struct Param;pub static probe_owner_export:Param=Param;impl Param{pub fn value(&self)->u8{1}}} +mod sbio{ + pub enum ComponentAction{AlreadyActive,Unsupported,Load} + pub fn component_action(_:u32)->ComponentAction{ComponentAction::Load} + pub const SBIO_SAVED_USER_ID_AT:usize=0;pub const SBIO_STATUS_COLD_TRANSITION:u32=0x8002;pub const SBIO_STATUS_ALREADY_ACTIVE:u32=0x101; + pub const COMPONENT_STATE_ACTIVE:u32=1; + pub struct SbioLoadCatacomb; + impl SbioLoadCatacomb{pub fn new(_:&[u8])->Option{Some(Self)}pub fn opcode(&self)->u8{0}pub fn name(&self)->u8{0}pub fn payload(&self)->&[u8]{&[]}} + pub struct NoExistingCatacomb;impl NoExistingCatacomb{pub fn from_zero_identities(_:usize)->Option{None}} + pub enum ContextScope{SYSTEM}pub fn sbio_select_context(_:ContextScope,_:&NoExistingCatacomb){} + pub struct ComponentStates;impl ComponentStates{pub fn new(_:&[u8])->Self{Self}pub fn state_for(&self,_:i32)->Option{None}} + pub struct CatacombUser;impl CatacombUser{pub const OWNER:Self=Self;pub fn value(&self)->i32{0}} +} +struct Status;impl Status{fn is_ok(&self)->bool{true}fn answered(&self)->Option{None}} +impl std::fmt::Display for Status{fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{Ok(())}} +struct Done{status:Status} +struct Sbio{dev:(),templates_restored:AtomicBool} +impl Sbio{ + fn with_host_store(&self,f:impl FnOnce(&Store)->R)->Option{if HOST.with(|h|h.get())<0{None}else{Some(f(&Store))}} + fn sbio_transfer_raw(&self,_:u8,_:u8,_:&[u8])->Result{TRANSFERRED.with(|t|t.set(true));Ok(Done{status:Status})} + fn confirm_active(&self,_:i32,_:&CStr,_:LoadAnswer)->RestoreOutcome{RestoreOutcome::Restored} + fn log_identity_count(&self,_:&CStr)->Option{Some(1)} + fn enclave_identities_for(&self,_:i32)->Option>{ASKED_ENCLAVE.with(|a|a.set(true));None} + fn sbio_expect_ok(&self,_:&())->Option<()>{None} + fn read_component_states(&self)->Option>{None} + fn save_catacomb(&self,_:sbio::CatacombUser,_:u8,_:&CStr)->bool{false} + @@RESTORE@@ + @@READ_STORED@@ + @@PROBE@@ +} +fn set(files:[i32;3],host:i32){FILE.with(|f|*f.borrow_mut()=files);HOST.with(|h|h.set(host));ASKED_ENCLAVE.with(|a|a.set(false));TRANSFERRED.with(|t|t.set(false));} +fn main(){ + let s=Sbio{dev:(),templates_restored:AtomicBool::new(false)}; + let user=PRIVATE_TYPE_CATACOMB_USER; + set([0,0,EIO],-1); + assert_eq!(s.restore_catacomb(Some(0),1,user,c"user"),RestoreOutcome::Failed,"unreadable catacomb restored as missing"); + assert!(!TRANSFERRED.with(|t|t.get()),"unreadable catacomb sent to the enclave"); + set([0,0,ENOENT],0); + assert_eq!(s.restore_catacomb(Some(0),1,user,c"user"),RestoreOutcome::Restored,"host-store copy not used"); + set([0,0,ENOENT],EIO); + assert_eq!(s.restore_catacomb(Some(0),1,user,c"user"),RestoreOutcome::Failed,"unreadable host store restored as missing"); + set([0,0,ENOENT],-1); + assert_eq!(s.restore_catacomb(Some(0),1,user,c"user"),RestoreOutcome::NoStoredFile,"genuine absence not reported"); + // Owner probe: positive control reaches the enclave; an unreadable owner file must not. + set([0,ENOENT,0],-1);s.probe_owner_export(); + assert!(ASKED_ENCLAVE.with(|a|a.get()),"owner probe never reached its enclave check"); + for e in [EIO,13]{set([0,e,0],-1);s.probe_owner_export();assert!(!ASKED_ENCLAVE.with(|a|a.get()),"unreadable owner catacomb treated as absent");} + set([EIO,ENOENT,0],-1);s.probe_owner_export(); + assert!(!ASKED_ENCLAVE.with(|a|a.get()),"unreadable master catacomb treated as saved"); + println!("caller cases passed"); +} +''' + @unittest.skipUnless(shutil.which('rustc'), 'rustc required') class Preservation(unittest.TestCase): @@ -214,6 +330,33 @@ def test_refused_keybag_requires_canonical_writer_record(self): old=method[:start]+' if state == STATE_REFUSED { return Ok(State::Absent(NoStoredKeyBag { slot })); }'+method[stop:] self.run_rust(make(old),should_pass=False) + def test_catacomb_read_error_is_not_absence(self): + text=(ROOT/'catacomb.rs').read_text() + method=body(text,'pub(crate) fn read(kind: u8)') + path=body(text,'fn path_for(kind: u8)').replace('crate::','') + crc=body((ROOT/'store.rs').read_text(),'pub(crate) const fn crc16_ccitt_false(') + make=lambda m:CATACOMB.replace('METHOD',m).replace('PATH',path).replace('@@CRC@@',crc) + self.run_rust(make(method)) + old=method.replace("Err(e) => return Err(e),","Err(e) => return Ok(None),",1) + self.assertNotEqual(old,method) + self.run_rust(make(old),should_pass=False) + + def test_callers_refuse_unreadable_catacombs(self): + cat=(ROOT/'catacomb.rs').read_text();sb=(ROOT/'sbio.rs').read_text() + parts={'IS_SAVED':body(cat,'pub(crate) fn is_saved('),'IS_ABSENT':body(cat,'pub(crate) fn is_absent('), + 'RESTORE':body(sb,' fn restore_catacomb('),'READ_STORED':body(sb,' fn read_stored('), + 'PROBE':body(sb,' fn probe_owner_export(')} + def make(**over): + text=CALLERS + for key,value in {**parts,**over}.items(): + text=text.replace('@@'+key+'@@',value.replace('crate::','')) + return text + self.run_rust(make()) + restore=parts['RESTORE'].replace('Err(_) => return RestoreOutcome::Failed,','Err(_) => return RestoreOutcome::NoStoredFile,',1) + absent=parts['IS_ABSENT'].replace('matches!(read(kind), Ok(None))','!matches!(read(kind), Ok(Some(_)))',1) + for name,old in (('RESTORE',restore),('IS_ABSENT',absent)): + self.assertNotEqual(old,parts[name]) + self.run_rust(make(**{name:old}),should_pass=False) if __name__=='__main__': unittest.main() diff --git a/tools/aurora-sep/tests/test_store_open_errors.py b/tools/aurora-sep/tests/test_store_open_errors.py index b55626f14f536..7625f1ddffea0 100644 --- a/tools/aurora-sep/tests/test_store_open_errors.py +++ b/tools/aurora-sep/tests/test_store_open_errors.py @@ -2,9 +2,8 @@ """The regular-file C/Rust boundary must not turn an I/O error into absence. Compile the actual C wrapper bodies with a fake open_as_kernel(). No kernel, -files, enclave, or stored identity is opened. Rust assertions cover the source -wiring to the same from_err_ptr API already used by open_host(). When rustc is -available, also compile the unchanged Rust module against a tiny kernel API +files, enclave, or stored identity is opened. When rustc is available, also +compile the unchanged Rust module against a tiny kernel API facade and those C wrappers; this is a userspace boundary fixture, not a kernel build or a real filesystem test. """ @@ -111,15 +110,6 @@ def test_old_error_collapse_is_rejected(self): self.assertEqual(result.returncode, 1) self.assertIn('lost errno', result.stderr) - def test_rust_propagates_each_regular_open_error(self): - source = RUST_SOURCE.read_text() - for method, call in zip(('open', 'open_trunc', 'open_readonly'), OPENS): - with self.subTest(method=method): - body = function(source, 'pub(crate) fn ' + method + '(') - self.assertIn('kernel::error::from_err_ptr(' + call + '(path.as_char_ptr()))?', body) - self.assertNotIn('is_null()', body) - self.assertNotIn('Err(ENOENT)', body) - def test_block_device_open_contract_unchanged(self): body = function(C_SOURCE.read_text(), 'void *sep_store_open_block(') self.assertRegex(body, r'if \(IS_ERR\(f\)\)\s+return NULL;') From 6b46c9f34436cf2bb1bcdc085402ec54353c80ed Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 10:59:42 -0500 Subject: [PATCH 026/392] soc: apple: sep: preserve the J613 26.6.2 existing-bag policy Keep the existing T8122 transport and sensor profile. On the owner-qualified OS-firmware lane, refuse writable attachment without the boot-forwarded identity and guard, and fail closed on host-store/index read errors. Add opt-outs for trusted-key and FileVault API registration with existing defaults preserved. No new provisioning or firmware protocol is introduced. Signed-off-by: Satya Benson --- .../ABI/testing/sysfs-driver-apple-sep | 20 ++++++++++ drivers/soc/apple/profile.rs | 27 ++++++++++++++ drivers/soc/apple/sep.rs | 37 +++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-apple-sep b/Documentation/ABI/testing/sysfs-driver-apple-sep index b48d26187f67d..5ee6cd0df614d 100644 --- a/Documentation/ABI/testing/sysfs-driver-apple-sep +++ b/Documentation/ABI/testing/sysfs-driver-apple-sep @@ -30,6 +30,26 @@ Description: by the boot-time bring-up, so the sensor and touchid attributes only move on once something has opened it. + This deployment uses the existing T8122 profile and v4 + /dev/sep-bio interface. It does not add a new sensor protocol. + OS firmware and system firmware are separate: qualify both. + + For an existing bag, select xart_writes=1, provision_keybag=0, + xart_guard=1 and xarm_trace=1. trusted_keys=0 and + filevault_api=0 omit the optional kernel APIs; both default + to 1 to preserve existing deployments. There is no boot_uuid + parameter: the OS identity comes from /chosen/apfs-preboot-uuid. + Writable J613 26.6.2 attachment rejects a missing DT identity, + explicit UUID override or disabled guard before opening xART. + Host-store or identity-index read failure refuses attachment. + + Preserve the current keybag, host store, reference key, three + Catacombs, ownership journal and fprint records. Different + Linux roots must share the same current files globally, before + module load; the kernel opens them from init_task's root. + Never replace them with a snapshot or enrol again to migrate. + Readiness diagnostics do not prove a successful finger match. + What: /sys/bus/platform/devices/.sep/diag/profile Date: October 2026 KernelVersion: 7.1 diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index aaefc62b9528b..2b9a1c9105953 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -710,3 +710,30 @@ mod dart_window_tests { } } } + +/// The owner-qualified J613 OS-firmware lane; system firmware may differ. +pub(crate) fn j613_26_6_2() -> bool { + machine_has(b"apple,j613") + && chosen_string(c"asahi,os-fw-version") + .as_deref() + .is_some_and(os_firmware_26_6_2) +} + +fn os_firmware_26_6_2(raw: &[u8]) -> bool { + raw == b"26.6.2" || raw == b"26.6.2\0" +} + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[kernel::macros::kunit_tests(apple_sep_j613_firmware)] +mod j613_firmware_tests { + use super::*; + + #[test] + fn exact_os_firmware_only() { + assert!(os_firmware_26_6_2(b"26.6.2")); + assert!(os_firmware_26_6_2(b"26.6.2\0")); + for raw in [&b"14.8.3"[..], &b"26.6.20"[..], &b"27.0"[..], &b""[..]] { + assert!(!os_firmware_26_6_2(raw)); + } + } +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5fd41c5bebf60..10fe16a9b79d4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -857,6 +857,18 @@ impl SepData { let xart_start = module_parameters::xart_start_sector.value(); let xart_guard = xart_guard::level(module_parameters::xart_guard.value(), profile.xart_guard); + // The qualified 26.6.2 lane uses only the boot-forwarded identity. + // Check before opening the shared xART store writable. + if profile::j613_26_6_2() && xart_writes { + if dt::preboot_uuid().is_none() + || module_parameters::os_uuid_hi.value() != 0 + || module_parameters::os_uuid_lo.value() != 0 + || xart_guard.is_none() + { + dev_err!(dev, "J613 26.6.2 requires a DT identity and xART guard\n"); + return Err(EINVAL); + } + } let store = match xart_store::Store::open_owner(xart_writes, xart_start) { Ok(mut store) => { match xart_guard { @@ -897,6 +909,10 @@ impl SepData { let mut host_store = match store::Store::open() { Ok(store) => Some(store), Err(e) => { + if profile::j613_26_6_2() { + dev_err!(dev, "J613 26.6.2 host-state store unavailable: {:?}\n", e); + return Err(e); + } dev_warn!(dev, "Linux host-state store unavailable: {:?}\n", e); None } @@ -904,6 +920,7 @@ impl SepData { let bio_index = match host_store.as_mut().map(bio::IdentityIndex::load) { Some(Ok(index)) => index, + Some(Err(e)) if profile::j613_26_6_2() => return Err(e), Some(Err(_)) => bio::IdentityIndex::new(), None => bio::IdentityIndex::new(), }; @@ -2668,12 +2685,16 @@ impl platform::Driver for SepDriver { data.attach(&sep_node)?; - if let Err(e) = data.register_fv_kernel() { - dev_err!(dev, "could not register the FileVault kernel API: {:?}\n", e); + if module_parameters::filevault_api.value() != 0 { + if let Err(e) = data.register_fv_kernel() { + dev_err!(dev, "could not register the FileVault kernel API: {:?}\n", e); + } } - if let Err(e) = trusted::register(data.clone()) { - dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); + if module_parameters::trusted_keys.value() != 0 { + if let Err(e) = trusted::register(data.clone()) { + dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); + } } if let Err(e) = pm::register(data.clone()) { @@ -2740,6 +2761,14 @@ module! { description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", license: "Dual MIT/GPL", params: { + trusted_keys: u8 { + default: 1, + description: "Register the SEP trusted-key source; 0 leaves it disabled", + }, + filevault_api: u8 { + default: 1, + description: "Register the SEP FileVault kernel API; 0 leaves it disabled", + }, xart_writes: u8 { default: 1, description: "Allow shared xART writes only if APFS proves a single unsnapshotted .gl extent; set to 0 for read-only operation", From b40f6bb15e73589d7b68deddcdb671638a2b224d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 13:23:57 -0500 Subject: [PATCH 027/392] soc: apple: sep: expose passive selected-policy diagnostics Signed-off-by: Satya Benson --- .../ABI/testing/sysfs-driver-apple-sep | 29 ++++++- drivers/soc/apple/diag.rs | 44 ++++++++++ drivers/soc/apple/diag_shim.c | 58 ++++++++++++- drivers/soc/apple/sep.rs | 14 +++- drivers/soc/apple/shim.h | 4 + tools/aurora-sep/tests/test_diag_policy.py | 82 +++++++++++++++++++ 6 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 tools/aurora-sep/tests/test_diag_policy.py diff --git a/Documentation/ABI/testing/sysfs-driver-apple-sep b/Documentation/ABI/testing/sysfs-driver-apple-sep index 5ee6cd0df614d..634f67162bc5c 100644 --- a/Documentation/ABI/testing/sysfs-driver-apple-sep +++ b/Documentation/ABI/testing/sysfs-driver-apple-sep @@ -3,9 +3,10 @@ Date: October 2026 KernelVersion: 7.1 Contact: Chris Kearney Description: - (RO) Version of the diag interface: "2" for the attributes + (RO) Version of the diag interface: "3" for the attributes and values described in this file. Version 2 added - sensor_attach and restore; version 1 had neither. + sensor_attach and restore; version 1 had neither. Version 3 + adds the policy snapshot. Within a version, attributes are only added, and an existing attribute keeps its values and their meaning. A reader tests @@ -19,7 +20,8 @@ Description: bring-up got without reading the kernel log. It exists while the driver is bound to the SEP: it is created at probe and removed at unbind. Every attribute in it is read-only and - reads as one ASCII token followed by a newline. + reads as ASCII followed by a newline. The policy snapshot + contains space-separated key=value tokens. Each value is the state the driver recorded when it made the corresponding decision. Reading an attribute sends nothing @@ -379,3 +381,24 @@ Description: tools/aurora-sep/XART-SAFETY.md. During a boot the value can only move from "unverified" to "blocked", once the guard refuses a write. + +What: /sys/bus/platform/devices/.sep/diag/policy +Date: November 2026 +KernelVersion: 7.2 +Contact: Satya Benson +Description: + (RO, ABI 3) A space-separated key=value snapshot of the + driver policy selected at probe: xart_writes (effective store + writability), provision_keybag, xart_guard (effective level), + trusted_keys, filevault_api, xarm_trace, refkey_v2 and + identity_override (0 or 1). API flags describe the selected + policy, not successful registration if that API later fails. + + identity_source is pending before the driver selects it, dt + for /chosen/apfs-preboot-uuid, override for explicit parameters, + or missing. It is updated at the actual identity decision. + No UUID value, state, identity or key bytes are exposed. + Reads only use cached facts and never contact the enclave. + Module-parameter sysfs files need not exist; their absence + is not evidence of a mismatched policy. Older diagnostic + ABIs retain xart/xart_guard and kernel decision messages. diff --git a/drivers/soc/apple/diag.rs b/drivers/soc/apple/diag.rs index 1b6fdd2288c52..b05b7b90c6e25 100644 --- a/drivers/soc/apple/diag.rs +++ b/drivers/soc/apple/diag.rs @@ -30,6 +30,15 @@ extern "C" { sepos13: bool, xart: bool, ) -> c_int; + fn sep_diag_set_policy( + provision_keybag: bool, + trusted_keys: bool, + filevault_api: bool, + xarm_trace: bool, + refkey_v2: bool, + identity_override: bool, + ); + fn sep_diag_set_identity_source(source: c_uint); fn sep_diag_set_attach(attached: bool); fn sep_diag_set_endpoints(count: c_uint); fn sep_diag_set_keystore_open(); @@ -66,6 +75,28 @@ pub(crate) enum Restore { Unproven = 3, } +/// Boot-time policy, without identity values or key material. +pub(crate) struct Policy { + pub provision_keybag: bool, + pub trusted_keys: bool, + pub filevault_api: bool, + pub xarm_trace: bool, + pub refkey_v2: bool, + pub identity_override: bool, +} + +/// The source selected by prepare_os_uuid, not the UUID itself. +pub(crate) enum IdentitySource { + DeviceTree = 1, + Override = 2, + Missing = 3, +} + +pub(crate) fn identity_source(source: IdentitySource) { + // SAFETY: this only stores the source enum in the passive cache. + unsafe { sep_diag_set_identity_source(source as c_uint) }; +} + /// Adds the `diag` group to the SEP device, with the facts fixed at probe. The /// group goes away when the driver unbinds. /// @@ -76,6 +107,7 @@ pub(crate) fn register( profile: &profile::PlatformProfile, xart_writable: bool, guard: Option<(u8, bool)>, + policy: Policy, ) -> Result { // Exhaustive, so a new bootstrap or key store fails to build here until // the ABI has a value for it. @@ -97,6 +129,18 @@ pub(crate) fn register( // SAFETY: no preconditions; stores three words. unsafe { sep_diag_set_xart_guard(c_uint::from(level), list_loaded, matching() as c_int) }; + // SAFETY: the facts are fixed before the attribute group is published. + unsafe { + sep_diag_set_policy( + policy.provision_keybag, + policy.trusted_keys, + policy.filevault_api, + policy.xarm_trace, + policy.refkey_v2, + policy.identity_override, + ) + }; + // SAFETY: `dev` is bound for the duration of the call, which is what devres // needs. `profile.name` is a `str`, valid for reads of `profile.name.len()` // bytes; the shim copies at most that many, needs no NUL terminator, and diff --git a/drivers/soc/apple/diag_shim.c b/drivers/soc/apple/diag_shim.c index 5c54d5d8410b3..9b8773b011260 100644 --- a/drivers/soc/apple/diag_shim.c +++ b/drivers/soc/apple/diag_shim.c @@ -23,7 +23,7 @@ #include "shim.h" /* The contract version; see Documentation/ABI/testing/sysfs-driver-apple-sep. */ -#define SEP_DIAG_ABI 2 +#define SEP_DIAG_ABI 3 #define SEP_DIAG_ATTACH_PENDING 0 #define SEP_DIAG_ATTACH_ATTACHED 1 @@ -64,6 +64,13 @@ static struct { bool cold; bool sepos13; bool xart; + bool provision_keybag; + bool trusted_keys; + bool filevault_api; + bool xarm_trace; + bool refkey_v2; + bool identity_override; + atomic_t identity_source; /* The xART guard's effective level (0: none runs), and its list. */ unsigned int xart_guard; bool xart_guard_list; @@ -120,6 +127,54 @@ static ssize_t xart_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(xart); +/* Parameter policy is captured at probe; the identity choice is cached later. */ +static ssize_t policy_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *identity; + + switch (atomic_read(&sep_diag.identity_source)) { + case 1: + identity = "dt"; + break; + case 2: + identity = "override"; + break; + case 3: + identity = "missing"; + break; + default: + identity = "pending"; + break; + } + return sysfs_emit(buf, + "xart_writes=%u provision_keybag=%u xart_guard=%u " + "trusted_keys=%u filevault_api=%u xarm_trace=%u " + "refkey_v2=%u identity_override=%u identity_source=%s\n", + sep_diag.xart, sep_diag.provision_keybag, sep_diag.xart_guard, + sep_diag.trusted_keys, sep_diag.filevault_api, sep_diag.xarm_trace, + sep_diag.refkey_v2, sep_diag.identity_override, identity); +} +static DEVICE_ATTR_RO(policy); + +void sep_diag_set_policy(bool provision_keybag, bool trusted_keys, + bool filevault_api, bool xarm_trace, bool refkey_v2, + bool identity_override) +{ + sep_diag.provision_keybag = provision_keybag; + sep_diag.trusted_keys = trusted_keys; + sep_diag.filevault_api = filevault_api; + sep_diag.xarm_trace = xarm_trace; + sep_diag.refkey_v2 = refkey_v2; + sep_diag.identity_override = identity_override; + atomic_set(&sep_diag.identity_source, 0); +} + +void sep_diag_set_identity_source(unsigned int source) +{ + atomic_set(&sep_diag.identity_source, source); +} + static ssize_t attach_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -346,6 +401,7 @@ static struct attribute *sep_diag_attrs[] = { &dev_attr_boot.attr, &dev_attr_protocol.attr, &dev_attr_xart.attr, + &dev_attr_policy.attr, &dev_attr_attach.attr, &dev_attr_endpoints.attr, &dev_attr_keystore.attr, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 10fe16a9b79d4..e0f9f9a766e2f 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1568,14 +1568,17 @@ impl SepData { match os_identity::select(hi, lo, dt::preboot_uuid) { os_identity::Selection::Explicit(uuid) => { self.xarm.lock().os_uuid = Some(uuid); + diag::identity_source(diag::IdentitySource::Override); dev_info!(self.dev, "xART: using explicit OS identity\n"); } os_identity::Selection::Chosen(uuid) => { self.xarm.lock().os_uuid = Some(uuid); + diag::identity_source(diag::IdentitySource::DeviceTree); dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); } os_identity::Selection::Unavailable => { self.xarm.lock().os_uuid = None; + diag::identity_source(diag::IdentitySource::Missing); dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); } } @@ -2677,7 +2680,16 @@ impl platform::Driver for SepDriver { // As soon as the fixed facts are known, so a tool can watch the attach // from "pending" on. Diagnostics are not worth failing the probe for. let guard = data.with_store(|store| store.guard_state()).flatten(); - if let Err(e) = diag::register(dev, data.profile, data.xart_writable(), guard) { + let policy = diag::Policy { + provision_keybag: module_parameters::provision_keybag.value() != 0, + trusted_keys: module_parameters::trusted_keys.value() != 0, + filevault_api: module_parameters::filevault_api.value() != 0, + xarm_trace: module_parameters::xarm_trace.value() != 0, + refkey_v2: module_parameters::refkey_v2.value() != 0, + identity_override: module_parameters::os_uuid_hi.value() != 0 + || module_parameters::os_uuid_lo.value() != 0, + }; + if let Err(e) = diag::register(dev, data.profile, data.xart_writable(), guard, policy) { dev_warn!(dev, "diag: could not add the sysfs attributes ({:?})\n", e); } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index ca0d3d8a797ee..e478deaddd0e7 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -179,6 +179,10 @@ struct device; int sep_diag_register(struct device *dev, const char *profile, size_t profile_len, bool cold, bool sepos13, bool xart); +void sep_diag_set_policy(bool provision_keybag, bool trusted_keys, + bool filevault_api, bool xarm_trace, bool refkey_v2, + bool identity_override); +void sep_diag_set_identity_source(unsigned int source); void sep_diag_set_attach(bool attached); void sep_diag_set_endpoints(unsigned int count); void sep_diag_set_keystore_open(void); diff --git a/tools/aurora-sep/tests/test_diag_policy.py b/tools/aurora-sep/tests/test_diag_policy.py new file mode 100644 index 0000000000000..be4b93f55c2bf --- /dev/null +++ b/tools/aurora-sep/tests/test_diag_policy.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the real passive policy cache/show bodies against synthetic flags.""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +SOURCE = ROOT / 'drivers/soc/apple/diag_shim.c' + + +def function(text, name): + match = re.search(r'^(?:static ssize_t|void) ' + name + r'\(', text, re.M) + if not match: + raise AssertionError(name) + start = text.index('{', match.start()) + depth = 1 + at = start + 1 + while depth: + depth += (text[at] == '{') - (text[at] == '}') + at += 1 + return text[match.start():at] + + +class Tests(unittest.TestCase): + def test_selected_policy_and_identity_source_only(self): + compiler = shutil.which('cc') + self.assertIsNotNone(compiler, 'C compiler required; do not count a skip as a pass') + text = SOURCE.read_text() + state = re.search(r'static struct \{.*?\} sep_diag;', text, re.S).group(0) + body = '\n'.join(function(text, name) for name in + ('policy_show', 'sep_diag_set_policy', 'sep_diag_set_identity_source')) + prefix = '''#include +#include +#include +#include +typedef int atomic_t; +struct device { int unused; }; +struct device_attribute { int unused; }; +#define atomic_read(p) (*(p)) +#define atomic_set(p, v) (*(p) = (v)) +static int sysfs_emit(char *out, const char *fmt, ...) __attribute__((format(printf, 2, 3))); +static int sysfs_emit(char *out, const char *fmt, ...) { + va_list args; va_start(args, fmt); int n = vsnprintf(out, 4096, fmt, args); va_end(args); return n; +} +''' + main = '''int main(void) { + char out[4096]; sep_diag.xart = true; sep_diag.xart_guard = 1; + sep_diag_set_policy(false, false, false, true, false, false); + for (unsigned int i = 0; i < 4; i++) { + sep_diag_set_identity_source(i); policy_show(NULL, NULL, out); fputs(out, stdout); + } + sep_diag_set_policy(true, true, true, false, true, true); + sep_diag.xart = false; sep_diag.xart_guard = 0; + policy_show(NULL, NULL, out); fputs(out, stdout); return 0; +} +''' + with tempfile.TemporaryDirectory() as tmp: + source, program = Path(tmp) / 'policy.c', Path(tmp) / 'policy' + source.write_text(prefix + state + body + main) + subprocess.run([compiler, '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', str(source), '-o', str(program)], check=True) + lines = subprocess.check_output([str(program)], text=True).splitlines() + expected = dict(xart_writes='1', provision_keybag='0', xart_guard='1', + trusted_keys='0', filevault_api='0', xarm_trace='1', + refkey_v2='0', identity_override='0') + for line, identity in zip(lines[:4], ('pending', 'dt', 'override', 'missing')): + self.assertEqual(dict(word.split('=') for word in line.split()), + {**expected, 'identity_source': identity}) + self.assertEqual(dict(word.split('=') for word in lines[4].split()), + dict(xart_writes='0', provision_keybag='1', xart_guard='0', + trusted_keys='1', filevault_api='1', xarm_trace='0', + refkey_v2='1', identity_override='1', identity_source='pending')) + self.assertNotIn('os_uuid_hi=', '\n'.join(lines)) + self.assertNotIn('os_uuid_lo=', '\n'.join(lines)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 4a3e51627f70feaed5ad32c05b6ced7b91463572 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 13:54:50 -0500 Subject: [PATCH 028/392] soc: apple: sep: update sensor host tests for diagnostic ABI 3 Signed-off-by: Satya Benson --- tools/aurora-sep/tests/test_diag_sensor.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/aurora-sep/tests/test_diag_sensor.py b/tools/aurora-sep/tests/test_diag_sensor.py index 01c7a8c1d97ba..16623f1f3e358 100644 --- a/tools/aurora-sep/tests/test_diag_sensor.py +++ b/tools/aurora-sep/tests/test_diag_sensor.py @@ -160,7 +160,7 @@ sep_diag_set_xart_guard(0, false, SEP_DIAG_MATCHING_USABLE); sep_diag_register(&dev, "T8140/J700", 10, false, false, true); /* The guard's attributes were added within version 2. */ - expect("probe", "abi", "2"); + expect("probe", "abi", "3"); expect("probe", "sensor", "unbound"); expect("probe", "touchid", "unknown"); expect("probe", "xart_guard", "0"); @@ -295,14 +295,14 @@ def test_matching_follows_the_guard(self): usable, unverified, blocked = 1, 2, 3 # SEP_DIAG_MATCHING_* in shim.h cases = [ # level, list, matching at probe, refusals, matching after them - ((0, 0, usable, 0, 0), '2 0 none 0 usable'), # Neo, M1 Pro (pinned 0) - ((1, 1, unverified, 0, 0), '2 1 loaded 0 unverified'), # M2 under the guard - ((1, 1, unverified, 2, blocked), '2 1 loaded 2 blocked'), # ... after refusals - ((2, 1, unverified, 0, 0), '2 2 loaded 0 unverified'), - ((1, 0, blocked, 0, 0), '2 1 unavailable 0 blocked'), # no list - ((1, 1, blocked, 0, 0), '2 1 loaded 0 blocked'), # blocked at probe - ((0, 0, 0, 0, 0), '2 0 none 0 unknown'), # never set - ((0, 0, 7, 0, 0), '2 0 none 0 unknown'), # out of range + ((0, 0, usable, 0, 0), '3 0 none 0 usable'), # Neo, M1 Pro (pinned 0) + ((1, 1, unverified, 0, 0), '3 1 loaded 0 unverified'), # M2 under the guard + ((1, 1, unverified, 2, blocked), '3 1 loaded 2 blocked'), # ... after refusals + ((2, 1, unverified, 0, 0), '3 2 loaded 0 unverified'), + ((1, 0, blocked, 0, 0), '3 1 unavailable 0 blocked'), # no list + ((1, 1, blocked, 0, 0), '3 1 loaded 0 blocked'), # cold boot, level 1 + ((0, 0, 0, 0, 0), '3 0 none 0 unknown'), # never set + ((0, 0, 7, 0, 0), '3 0 none 0 unknown'), # out of range ] with tempfile.TemporaryDirectory() as directory: harness = self.build(Path(directory), GUARD_HARNESS, 'guard') From d3f6486fd3fddea01310e834b7e261e339de7da4 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:18:41 -0400 Subject: [PATCH 029/392] drm/apple: accept automatic PMGR activity in J613 coldplug Use the PMGR provider active-state predicate when checking inherited display power. DISP_CPU can report ACTUAL=4 with AUTO_ENABLE and TARGET=15 after automatic clock gating. Keep the checks read-only and preserve all other handoff requirements. Test all actual, target and automatic-state combinations and the captured J613 state through the production helper. Signed-off-by: Satya Benson --- drivers/gpu/drm/apple/j613_dcp_activate.c | 9 ++++- tools/asahi/run-host-tests.py | 1 + tools/asahi/test-j613-dcp-power.py | 45 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 tools/asahi/test-j613-dcp-power.py diff --git a/drivers/gpu/drm/apple/j613_dcp_activate.c b/drivers/gpu/drm/apple/j613_dcp_activate.c index 6e20a5afc914c..6d0a262069469 100644 --- a/drivers/gpu/drm/apple/j613_dcp_activate.c +++ b/drivers/gpu/drm/apple/j613_dcp_activate.c @@ -40,6 +40,13 @@ bool apple_t8122_25g83_board(void) return ok; } +/* Match the PMGR provider: automatic domains can be clock-gated while active. */ +static bool j613_power_state_active(u32 state) +{ + return FIELD_GET(GENMASK(7, 4), state) == 15 || + ((state & BIT(28)) && FIELD_GET(GENMASK(3, 0), state) == 15); +} + /* These are DT domain nodes, carrying the PMGR offsets, not copied registers. */ static int j613_power_ready(void) { @@ -82,7 +89,7 @@ static int j613_power_ready(void) return -ENOMEM; state = readl(pmgr + offset); iounmap(pmgr); - if (FIELD_GET(GENMASK(7, 4), state) != 15) + if (!j613_power_state_active(state)) return -EBUSY; } return 0; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..00b9d65f68575 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-j613-dcp-power.py': [([], PASS)], 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], diff --git a/tools/asahi/test-j613-dcp-power.py b/tools/asahi/test-j613-dcp-power.py new file mode 100644 index 0000000000000..4e93bcc8b1bdb --- /dev/null +++ b/tools/asahi/test-j613-dcp-power.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production J613 PMGR predicate and test all actual/target/auto states.""" +from pathlib import Path +import os +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/j613_dcp_activate.c").read_text() +start = source.index("static bool j613_power_state_active(") +end = source.index("\n}\n", start) + 3 +stub = r""" +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(a,b) (((1U << ((a)-(b)+1))-1) << (b)) +#define FIELD_GET(mask,v) (((v) & (mask)) >> __builtin_ctz(mask)) +""" +main = r""" +int main(void) { + for (u32 actual = 0; actual < 16; actual++) + for (u32 target = 0; target < 16; target++) + for (u32 automatic = 0; automatic < 2; automatic++) { + u32 word = (actual << 4) | target | (automatic << 28); + bool expected = actual == 15 || (target == 15 && automatic); + if (j613_power_state_active(word) != expected) return 1; + } + if (!j613_power_state_active(0x1404024f)) return 2; + if (j613_power_state_active(0x0404024f)) return 3; + if (j613_power_state_active(0x14040240)) return 4; + puts("PASS: 512 PMGR combinations and captured-state regression controls"); + return 0; +} +""" +with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "power.c" + path.write_text(stub + source[start:end] + main) + for flags in ([], ["-O2"]): + exe = str(path.with_suffix("")) + subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", + *flags, str(path), "-o", exe], check=True) + subprocess.run([exe], check=True) From 31a2f5de22bbcf15668f864183b2c37fec43913b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 22:17:26 -0400 Subject: [PATCH 030/392] test: vary unrelated bits in J613 PMGR state words Cross all 512 ACTUAL/TARGET/AUTO combinations with zero, all-set, alternating and individual unrelated-bit patterns. This verifies that readiness ignores every bit outside the three PMGR fields. A temporary ACTUAL mask widened from GENMASK(7, 4) to GENMASK(8, 4) passes the old cases but fails the new ones at both -O0 and -O2. Make the unoptimized compiler flag explicit and report failing words. Signed-off-by: Satya Benson --- tools/asahi/test-j613-dcp-power.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/asahi/test-j613-dcp-power.py b/tools/asahi/test-j613-dcp-power.py index 4e93bcc8b1bdb..3bc1a3379c253 100644 --- a/tools/asahi/test-j613-dcp-power.py +++ b/tools/asahi/test-j613-dcp-power.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Compile the production J613 PMGR predicate and test all actual/target/auto states.""" +"""Compile the production J613 PMGR predicate and test PMGR fields and unrelated bits.""" from pathlib import Path import os import subprocess @@ -21,24 +21,36 @@ """ main = r""" int main(void) { + /* Only ACTUAL[7:4], TARGET[3:0] and AUTO_ENABLE[28] affect readiness. */ + const u32 unrelated_mask = ~(GENMASK(7, 0) | BIT(28)); + u32 unrelated[32] = {0, unrelated_mask, + 0xaaaaaaaaU & unrelated_mask, + 0x55555555U & unrelated_mask}; + unsigned int count = 4; + for (unsigned int bit = 8; bit < 32; bit++) + if (bit != 28) unrelated[count++] = BIT(bit); + for (unsigned int i = 0; i < count; i++) for (u32 actual = 0; actual < 16; actual++) for (u32 target = 0; target < 16; target++) for (u32 automatic = 0; automatic < 2; automatic++) { - u32 word = (actual << 4) | target | (automatic << 28); + u32 word = unrelated[i] | (actual << 4) | target | (automatic << 28); bool expected = actual == 15 || (target == 15 && automatic); - if (j613_power_state_active(word) != expected) return 1; + if (j613_power_state_active(word) != expected) { + fprintf(stderr, "PMGR state 0x%08x: expected active=%d\n", word, expected); + return 1; + } } if (!j613_power_state_active(0x1404024f)) return 2; if (j613_power_state_active(0x0404024f)) return 3; if (j613_power_state_active(0x14040240)) return 4; - puts("PASS: 512 PMGR combinations and captured-state regression controls"); + puts("PASS: 13824 PMGR combinations and captured-state regression controls"); return 0; } """ with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "power.c" path.write_text(stub + source[start:end] + main) - for flags in ([], ["-O2"]): + for flags in (["-O0"], ["-O2"]): exe = str(path.with_suffix("")) subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", *flags, str(path), "-o", exe], check=True) From 8bf9ec2ca3171eac7386853ff3372d307e4abbed Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:18:41 -0400 Subject: [PATCH 031/392] drm/asahi: accept the T8122 25G83 mailbox compatible list The 25G83 device tree advertises the AGX-specific compatible followed by asc-mailbox-v4. Both bind the same mailbox operations. Admit this exact pair while retaining all resource and identity checks. Exercise the production rejection predicate with public-DTS fixtures and resource mutations; allow optional local DT captures for reproduction. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_soc.rs | 1 + tools/asahi/run-host-tests.py | 1 + tools/asahi/test-m3-mailbox.py | 90 ++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 tools/asahi/test-m3-mailbox.py diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index 4b81475612b6f..7500822c4124c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -291,6 +291,7 @@ pub(crate) static T8122: Soc = Soc { cores_per_cluster: 10, mailbox_compatibles: &[ b"apple,t8122-agx-asc-mailbox\0", + b"apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0", b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0", ], mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4], diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..c4c6e21383fbf 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-mailbox.py': [([], PASS)], 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], diff --git a/tools/asahi/test-m3-mailbox.py b/tools/asahi/test-m3-mailbox.py new file mode 100644 index 0000000000000..2320d17f10f6f --- /dev/null +++ b/tools/asahi/test-m3-mailbox.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production T8122 mailbox predicate with captured DTB properties. + +Usage: python3 tools/asahi/test-m3-mailbox.py [runtime.dtb daily.dtb] +Requires dtc's fdtget. No target access or MMIO. +""" +import hashlib +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +board = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_board.rs').read_text() +soc = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_soc.rs').read_text() +t8122 = soc[soc.index('pub(crate) static T8122:'):] +compatibles = re.search(r'mailbox_compatibles: &(\[.*?\]),', t8122, re.S)[1] +irqs = re.search(r'mailbox_interrupts: &(\[.*?\]),', t8122, re.S)[1] +base = re.search(r'mailbox: (0x[0-9a-fA-F_]+),', t8122)[1] +regfn = re.search(r'fn expected_mailbox_reg\(.*?\n}', board, re.S)[0] +predicate = board[board.index(' if !soc.mailbox_compatibles'):] +predicate = predicate[:predicate.index('\n {')].strip()[3:] + + +def prop(dtb, node, name, kind): + result = subprocess.check_output(['fdtget', '-t', kind, str(dtb), node, name], text=True) + return [int(v, 16) for v in result.split()] + + +def vec(values): + return 'vec![' + ','.join(map(str, values)) + ']' + + +fixtures = [] +for argument in sys.argv[1:]: + dtb = Path(argument) + gpu = '/soc/gpu@292400000' + # Both retained captures use this mailbox path; verify the phandle link, + # not its capture-specific numeric value. + mbox = '/soc/mbox@292408000' + if prop(dtb, gpu, 'mboxes', 'x') != prop(dtb, mbox, 'phandle', 'x'): + raise RuntimeError('GPU mailbox phandle mismatch') + values = [prop(dtb, mbox, name, kind) for name, kind in + [('compatible', 'bx'), ('reg', 'x'), ('interrupt-names', 'bx'), + ('interrupts', 'x'), ('#mbox-cells', 'x')]] + if len(values[4]) != 1: + raise RuntimeError('bad #mbox-cells length') + fixtures.append('check(' + ','.join(vec(v) for v in values[:4]) + ',' + str(values[4][0]) + ');') + print(dtb.name, hashlib.sha256(dtb.read_bytes()).hexdigest()) +if not sys.argv[1:]: + # Public t8122-j613-dcp-25g83.dtsi and t8122-gpu.dtsi resource contract. + names = list(b'send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0') + for compat in (b'apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0', + b'apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0'): + values = [list(compat), [2, 0x92408000, 0, 0x4000], names, + [0,723,4,0,724,4,0,725,4,0,726,4]] + fixtures.append('check(' + ','.join(vec(v) for v in values) + ',0);') +if len(fixtures) != 2: + raise SystemExit('supply runtime.dtb and daily.dtb') + +source = ''' +struct Windows { mailbox: u64 } +struct Soc { mailbox_compatibles: &'static [&'static [u8]], mailbox_interrupts: &'static [u32], windows: Windows } +''' + regfn + ''' +fn accepts(soc: &Soc, compat: &Vec, mbox_reg: &Vec, irq_names: &Vec, irqs: &Vec, cells: u32) -> bool { + !(''' + predicate + ''') +} +fn check(compat: Vec, reg: Vec, names: Vec, irqs: Vec, cells: u32) { + let soc = Soc { mailbox_compatibles: &''' + compatibles + ''', mailbox_interrupts: &''' + irqs + ''', windows: Windows { mailbox: ''' + base + ''' } }; + let ok = |c: &Vec, r: &Vec, n: &Vec, i: &Vec, x| accepts(&soc,c,r,n,i,x); + assert!(ok(&compat,®,&names,&irqs,cells)); + for i in 0..reg.len() { let mut v=reg.clone(); v[i]^=1; assert!(!ok(&compat,&v,&names,&irqs,cells)); } + for i in 0..irqs.len() { let mut v=irqs.clone(); v[i]^=1; assert!(!ok(&compat,®,&names,&v,cells)); } + assert!(!ok(&compat,®,&names,&irqs,1)); + let mut v=names.clone(); v[0]^=1; assert!(!ok(&compat,®,&v,&irqs,cells)); + for v in [b"apple,asc-mailbox-v4\\0".to_vec(), b"apple,t8122-agx-asc-mailbox\\0unknown\\0".to_vec(), b"apple,t6031-agx-asc-mailbox\\0".to_vec()] { assert!(!ok(&v,®,&names,&irqs,cells)); } + for v in soc.mailbox_compatibles { assert!(ok(&v.to_vec(),®,&names,&irqs,cells)); } +} +fn main() { +''' + '\n'.join(fixtures) + '\nprintln!("mailbox: both captures, all exact supported lists and resource/identity mutation controls PASS");\n}\n' +with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) + (path / 'test.rs').write_text(source) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path / 'test.rs'), '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) +print('m3_board.rs', hashlib.sha256(board.encode()).hexdigest()) +print('m3_soc.rs', hashlib.sha256(soc.encode()).hexdigest()) From dfb9d8b576a0b6c0364990c4f3487737413524bc Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:38:58 -0400 Subject: [PATCH 032/392] drm/asahi: describe the T8122 extended firmware profile and owned IO records Separate firmware identity, HAL metadata and VM bounds from physical G15G stream selection. Add checked root and owned IO descriptor encoders. Introduce slot 28 at SGX+0xd50000 with a production-writer regression; the shifted T8132 bank is never introduced. Keep VM padding zero. Signed-off-by: Satya Benson --- MAINTAINERS | 2 + drivers/gpu/drm/apple/j613_dcp_activate.c | 9 +- drivers/gpu/drm/asahi/m3/m3_client.rs | 17 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 20 +- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 146 +++- drivers/gpu/drm/asahi/m3/m3_init_26.rs | 275 +++++++ drivers/gpu/drm/asahi/m3/m3_init_layout.rs | 139 +++- drivers/gpu/drm/asahi/m3/m3_mmio_26.rs | 265 +++++++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 69 ++ drivers/gpu/drm/asahi/m3/mod.rs | 4 + drivers/gpu/drm/asahi/m3/t8122_admission.rs | 49 +- tools/asahi/m3-firmware-tests.rs | 14 + tools/asahi/m3-runtime-tests.rs | 9 + tools/asahi/m3-uapi-fixture.c | 363 +++++++++ tools/asahi/m3-uapi-tests.rs | 832 ++++++++++++++++++++ tools/asahi/run-host-tests.py | 3 + tools/asahi/test-j613-dcp-power.py | 57 ++ tools/asahi/test-m3-mailbox.py | 11 + tools/asahi/test-m3-uapi.py | 73 ++ 19 files changed, 2333 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_init_26.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_mmio_26.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_profile.rs create mode 100644 tools/asahi/m3-firmware-tests.rs create mode 100644 tools/asahi/m3-uapi-fixture.c create mode 100644 tools/asahi/m3-uapi-tests.rs create mode 100644 tools/asahi/test-j613-dcp-power.py create mode 100755 tools/asahi/test-m3-uapi.py diff --git a/MAINTAINERS b/MAINTAINERS index e0a0070243ae6..dbe76f9d15eaf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2789,8 +2789,10 @@ F: tools/asahi/apple-pwm-pm-tests.py F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* +F: tools/asahi/m3-uapi-* F: tools/asahi/pmp-bootargs-tests.rs F: drivers/spi/spi-apple-test.c +F: tools/asahi/test-m3-uapi.py F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thermal/apple-pmp* diff --git a/drivers/gpu/drm/apple/j613_dcp_activate.c b/drivers/gpu/drm/apple/j613_dcp_activate.c index 6e20a5afc914c..6d0a262069469 100644 --- a/drivers/gpu/drm/apple/j613_dcp_activate.c +++ b/drivers/gpu/drm/apple/j613_dcp_activate.c @@ -40,6 +40,13 @@ bool apple_t8122_25g83_board(void) return ok; } +/* Match the PMGR provider: automatic domains can be clock-gated while active. */ +static bool j613_power_state_active(u32 state) +{ + return FIELD_GET(GENMASK(7, 4), state) == 15 || + ((state & BIT(28)) && FIELD_GET(GENMASK(3, 0), state) == 15); +} + /* These are DT domain nodes, carrying the PMGR offsets, not copied registers. */ static int j613_power_ready(void) { @@ -82,7 +89,7 @@ static int j613_power_ready(void) return -ENOMEM; state = readl(pmgr + offset); iounmap(pmgr); - if (FIELD_GET(GENMASK(7, 4), state) != 15) + if (!j613_power_state_active(state)) return -EBUSY; } return 0; diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index dff1af0783965..c1a07d61120d1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -35,7 +35,7 @@ use crate::{ const AGX3_PARAMS_SIZE: u64 = (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + core::mem::size_of::()) as u64; -const _: () = assert!(AGX3_PARAMS_SIZE == 600); +const _: () = assert!(AGX3_PARAMS_SIZE == super::m3_profile::PARAMS_REQUIRED); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; @@ -45,10 +45,19 @@ static REFUSALS: AtomicU32 = AtomicU32::new(0); /// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { + admit_vm_for(device, super::m3_profile::Profile::Legacy, size, flags) +} + +pub(crate) fn admit_vm_for( + device: &kernel::device::Device, + profile: super::m3_profile::Profile, + size: u64, + flags: u32, +) -> Result { { // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`): // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime. - if size >= AGX3_PARAMS_SIZE && flags == 0 { + if profile.vm_acknowledged(size, flags) { return Ok(()); } let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); @@ -64,11 +73,13 @@ pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) - }; dev_info!( device, - "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", + "M3: refusing a VM for {}[{}]: GET_PARAMS bytes={}/{} HAL={} flags={:#x}{}\n", name, task.group_leader().pid(), size, AGX3_PARAMS_SIZE, + profile.hal(), + flags, last ); } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809..796abc7793f90 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -86,6 +86,7 @@ pub(crate) struct Backend { core_mask: u64, max_frequency_khz: u32, soc: &'static crate::m3::m3_soc::Soc, + profile: super::m3_profile::Profile, _progress: Pin>>, _activity: Pin>>, _memory: Pin>>, @@ -125,6 +126,7 @@ impl Backend { let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment) .inspect_err(refused("the firmware identity"))?; dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); + let profile = firmware.profile(); // InitData source and contents, before any GPU register access. let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment) .inspect_err(refused("the InitData contents"))?; @@ -160,7 +162,7 @@ impl Backend { let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?; // From here on every failure drops the backend, which stops the runtime. let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(), - scheduler, core_mask, max_frequency_khz, soc, + scheduler, core_mask, max_frequency_khz, soc, profile, _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }, GFP_KERNEL)?; // The completion worker services this runtime from now on; the RTKit callbacks and @@ -208,7 +210,9 @@ impl gpu::Gpu for Backend { } params.max_frequency_khz = self.max_frequency_khz; params.usc_generation = 3; - params.gpu_hal_generation = hw::GpuHalGeneration::Legacy as u32; + params.gpu_hal_generation = self.profile.hal(); + params.max_commands_per_submission = super::agx_uapi::UAPI_MAX_HARDWARE_COMMANDS; + params.max_attachments = super::agx_uapi::UAPI_MAX_ATTACHMENTS as u32; Ok(()) } fn supports_vm_status(&self) -> bool { true } @@ -217,10 +221,12 @@ impl gpu::Gpu for Backend { fn feature_flags(&self) -> u64 { let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; - if self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { + if self.profile == super::m3_profile::Profile::Legacy + && self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64; } - if self.soc.features.compute_wide_visibility { + if self.profile == super::m3_profile::Profile::Legacy + && self.soc.features.compute_wide_visibility { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64; } flags @@ -233,17 +239,17 @@ impl gpu::Gpu for Backend { } fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } fn user_range(&self) -> Range { - mmu::UAT_PGSZ as u64..crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + mmu::UAT_PGSZ as u64..self.profile.user_end() } /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the /// 42-bit geometry's own unk_page() is a different address. fn unknown_page(&self) -> u64 { - crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + self.profile.user_end() } fn tracked_submission(&self) -> bool { true } /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { - crate::m3::m3_client::admit_vm(dev, params_size, flags) + crate::m3::m3_client::admit_vm_for(dev, self.profile, params_size, flags) } fn new_vm(&self, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 0d2835174c677..c9aa45cc02864 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,6 +62,19 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; +/// Image record for the extended format. The SoC's runtime selection does not use it yet. +pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 26.6.2", + uuid: None, + stkg_sha256: Some([ + 0x3a, 0x6b, 0x8b, 0x5e, 0x83, 0x38, 0x4d, 0x5c, + 0xc1, 0x92, 0x9d, 0x91, 0xb6, 0x36, 0x40, 0xfe, + 0xe8, 0xf5, 0x52, 0xb5, 0x10, 0x8c, 0x4e, 0xa6, + 0x20, 0xbe, 0xdb, 0x7b, 0xbc, 0xa0, 0x31, 0x64, + ]), + initdata_magic: Some(0x0c89_c357_8392_04b8), +}]; + /// The same T8122 image for the start experiment (`t8122_start`, only with /// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version. pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage { @@ -117,6 +130,10 @@ pub(crate) struct Layout { /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing. pub(crate) entropy_tag_offset: usize, pub(crate) entropy_tag: [u8; 8], + /// An additional independent boot value, when required by this image. + extra_boot_value: Option<(usize, [u8; 8])>, + /// Expected boot-table location and byte count in the image header. + boot_table: Option<(u32, u32)>, /// The firmware version, for logs. pub(crate) version: &'static str, } @@ -127,6 +144,8 @@ pub(crate) static T6030_LAYOUT: Layout = Layout { vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000], entropy_tag_offset: 0x58e88, entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: None, + boot_table: None, version: "RTKit-2419.140.12.release", }; @@ -137,23 +156,73 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], entropy_tag_offset: 0x5d54a, entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: None, + boot_table: None, version: "RTKit-2419.140.12.release", }; pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; +/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +pub(crate) static T8122_LAYOUT_26: Layout = Layout { + text_size: 393216, + data_size: 1228800, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_0000], + entropy_tag_offset: 380492, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: Some((380719, *b"ECAP\x08\x00\x00\x00")), + boot_table: Some((380492, 561)), + version: "RTKit-3255.160.4.release", +}; + fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool { - let tag = layout.entropy_tag_offset; if text.len() != layout.text_size as usize { return false; } - let Some(end) = tag.checked_add(16) else { return false; }; - let Some(entropy) = text.get_mut(tag..end) else { return false; }; - if entropy[..8] != layout.entropy_tag { - return false; + if let Some((offset, count)) = layout.boot_table { + let Some(header) = text.get(0x22c..0x234) else { + return false; + }; + if header[..4] != offset.to_le_bytes() || header[4..] != count.to_le_bytes() { + return false; + } + let Some(end) = (offset as usize).checked_add(count as usize) else { + return false; + }; + if text.get(offset as usize..end).is_none() { + return false; + } + } + let fields = [ + Some((layout.entropy_tag_offset, layout.entropy_tag)), + layout.extra_boot_value, + ]; + // Validate every field before touching even the first boot value. + for (offset, tag) in fields.into_iter().flatten() { + let Some(end) = offset.checked_add(16) else { + return false; + }; + let Some(field) = text.get(offset..end) else { + return false; + }; + if field[..8] != tag { + return false; + } + if let Some((base, count)) = layout.boot_table { + if offset < base as usize || end > base as usize + count as usize { + return false; + } + } + } + if let Some((other, _)) = layout.extra_boot_value { + if other < layout.entropy_tag_offset + 16 && layout.entropy_tag_offset < other + 16 { + return false; + } + } + for (offset, _) in fields.into_iter().flatten() { + text[offset + 8..offset + 16].fill(0); } - entropy[8..].fill(0); true } @@ -167,6 +236,14 @@ pub(crate) struct Firmware { } impl Firmware { + pub(crate) fn profile(&self) -> super::m3_profile::Profile { + if core::ptr::eq(self.layout, &T8122_LAYOUT_26) { + super::m3_profile::Profile::Extended + } else { + super::m3_profile::Profile::Legacy + } + } + pub(crate) const fn version(&self) -> &'static str { self.layout.version } @@ -393,4 +470,61 @@ mod tests { assert!(!normalize_boot_entropy(&layout, &mut text)); assert_eq!(text, vec![0xa5; 15]); } + + fn image_26() -> Vec { + let mut image = vec![0x5a; 393216]; + image[0x22c..0x230].copy_from_slice(&380492u32.to_le_bytes()); + image[0x230..0x234].copy_from_slice(&561u32.to_le_bytes()); + image[380492..380500].copy_from_slice(b"GKTS\x08\x00\x00\x00"); + image[380719..380727].copy_from_slice(b"ECAP\x08\x00\x00\x00"); + image + } + + #[test] + fn newer_image_masks_only_both_boot_values() { + let mut image = image_26(); + let before = image.clone(); + assert!(normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + for index in 0..image.len() { + let expected = if (380500..380508).contains(&index) || (380727..380735).contains(&index) + { + 0 + } else { + before[index] + }; + assert_eq!(image[index], expected, "byte {index}"); + } + } + + #[test] + fn invalid_second_field_or_table_does_not_mask_first_value() { + for index in [0x22c, 0x230, 380492, 380496, 380719, 380723] { + let mut image = image_26(); + image[index] ^= 1; + let before = image.clone(); + assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + assert_eq!(image, before); + } + let mut image = image_26(); + image.pop(); + let before = image.clone(); + assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + assert_eq!(image, before); + } + + #[test] + fn extended_record_cannot_replace_legacy_identity() { + let newer = &T8122_IMAGES_26[0]; + let digest = newer.stkg_sha256.unwrap(); + assert!(newer.accepts_runtime(None, &digest)); + assert_eq!(newer.initdata_magic, Some(0x0c89_c357_8392_04b8)); + for index in 0..digest.len() { + let mut wrong = digest; + wrong[index] ^= 1; + assert!(!newer.accepts_runtime(None, &wrong)); + } + assert!(!KNOWN_IMAGES_T8122[0].accepts_runtime(None, &digest)); + assert!(!newer.accepts_runtime(KNOWN_IMAGES_T8122[0].uuid, &[0; 32])); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_init_26.rs b/drivers/gpu/drm/asahi/m3/m3_init_26.rs new file mode 100644 index 0000000000000..f08f16200b070 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_init_26.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Allocation and pointer records for the extended M3 initialization format. +//! Owners supply mapped regions; these builders never allocate or publish DMA. + +use super::m3_init_layout::{Error, Region, Root}; +use super::m3_init_storage::{self as storage, Allocation, Initial}; + +pub(crate) const COUNT: usize = 56; +pub(crate) const DYNAMIC_STATE: usize = 49; +pub(crate) const PERFORMANCE_CONTROL: usize = 50; +pub(crate) const RECOVERY_ACTIVITY: usize = 51; +pub(crate) const SENSOR_VALUE: usize = 52; +pub(crate) const RETENTION_STATE: usize = 53; +pub(crate) const GLOBAL_POOL: usize = 54; +pub(crate) const GLOBAL_PARAMETERS: usize = 55; +pub(crate) const RUNTIME_SIZE: usize = 0x4b8; +pub(crate) const LOG_THREADS: usize = 9; +pub(crate) const RX_BYTES: usize = 0x48; +pub(crate) const LOG_PAYLOAD_BYTES: usize = 0xd8; + +/// Extend the existing M3 owner indices instead of introducing another owner graph. +pub(crate) fn allocation(index: usize) -> Result { + let size = match index { + 0..=23 if index % 2 == 0 => 0x30, + 0..=23 => 0x1800, + 24 | 26 | 30 | 32 => 0x30, + 25 => 0x4000, + 27 | 33 => 0x4800, + 28 => LOG_THREADS * 0x30, + 29 => LOG_THREADS * 256 * RX_BYTES, + 31 => 512 * RX_BYTES, + 34 => 0x40, + 35 => 0x3fc0, + 36 => 0xc8, + 37 => 0x4000, + 38 => RUNTIME_SIZE, + 39 => 0x2658, + 40 => 0x88, + 41 => LOG_THREADS * 256 * LOG_PAYLOAD_BYTES, + 42 => 0xc18, + 43 => 0x1248, + 44 => 0xe10, + 45 => 0xe48, + 46 => 0x670, + 47 => 0xeaf0, + 48 => 0x20, + DYNAMIC_STATE | RETENTION_STATE | GLOBAL_POOL | GLOBAL_PARAMETERS => 0x4000, + PERFORMANCE_CONTROL => 0x6578, + RECOVERY_ACTIVITY => 0x60, + SENSOR_VALUE => 4, + _ => return Err(Error::Bounds), + }; + // Separate page-aligned owners, with room for the largest receive allocation. + // This namespace is disjoint from the existing fixed work and timestamp arenas. + Ok(Allocation { + address: 0xffff_fc21_0000_0000 + (index as u64) * 0x10_0000, + size, + gpu_shared: matches!(index, GLOBAL_POOL | GLOBAL_PARAMETERS), + initial: Initial::Zero, + }) +} + +pub(crate) fn cached(index: usize) -> bool { + matches!(index, 38 | 39 | 42 | 43 | 44 | PERFORMANCE_CONTROL) +} + +pub(crate) struct PointerGraph<'a> { + pub(crate) owners: &'a [Region; COUNT], + pub(crate) scratch: Region, + pub(crate) pool_alias: u64, + pub(crate) parameter_alias: u64, +} + +impl PointerGraph<'_> { + /// Check the complete graph, including padded mapping ownership, before encoding it. + fn addresses(&self) -> Result<[u64; COUNT], Error> { + let mut addresses = [0; COUNT]; + for (index, owner) in self.owners.iter().enumerate() { + let size = allocation(index)?.size; + addresses[index] = owner.at(0, size)?; + if addresses[index] & 0x3fff != 0 + || addresses[index] < 0xffff_fc20_0000_0000 + || addresses[index] >= 0xffff_fc2f_fc00_0000 + { + return Err(Error::Address); + } + let mapped = (size + 0x3fff) & !0x3fff; + let end = addresses[index] + .checked_add(mapped as u64) + .ok_or(Error::Address)?; + if end > 0xffff_fc2f_fc00_0000 { + return Err(Error::Address); + } + for prior in 0..index { + let prior_size = (allocation(prior)?.size + 0x3fff) & !0x3fff; + if addresses[index] < addresses[prior] + prior_size as u64 + && addresses[prior] < addresses[index] + mapped as u64 + { + return Err(Error::Address); + } + } + } + let scratch = self.scratch.at(0, 0x8000)?; + let scratch_end = scratch.checked_add(0x8000).ok_or(Error::Address)?; + if scratch & 0x3fff != 0 || scratch_end > 0xffff_fc2f_fc00_0000 { + return Err(Error::Address); + } + for (index, address) in addresses.iter().enumerate() { + let bytes = (allocation(index)?.size + 0x3fff) & !0x3fff; + if scratch < *address + bytes as u64 && *address < scratch_end { + return Err(Error::Address); + } + } + for alias in [self.pool_alias, self.parameter_alias] { + if alias & 0x3fff != 0 || !(0x70_0000_0000..0x70_0200_0000).contains(&alias) { + return Err(Error::Address); + } + } + if self.pool_alias == self.parameter_alias { + return Err(Error::Address); + } + Ok(addresses) + } + + pub(crate) fn root(&self, output: &mut [u8]) -> Result<(), Error> { + self.addresses()?; + Root { + region_a: self.owners[storage::REGION_A], + runtime: self.owners[storage::RUNTIME_POINTERS], + globals: self.owners[storage::GLOBALS], + new_a: self.owners[storage::RUNTIME_FLAGS], + control: self.owners[storage::CONTROL_REGION], + power: self.owners[storage::GLOBALS_POWER], + } + .encode_26(output, self.owners[DYNAMIC_STATE]) + } + + pub(crate) fn runtime(&self, output: &mut [u8]) -> Result<(), Error> { + let addresses = self.addresses()?; + let scratch = self.scratch.at(0, 0x8000)?; + let bytes = output.get_mut(..RUNTIME_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let pointer = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + pointer(bytes, 0x18, scratch); + pointer(bytes, 0x2d0, self.parameter_alias); + pointer(bytes, 0x2e0, self.pool_alias); + bytes[0x250..0x254].copy_from_slice(&1u32.to_le_bytes()); + bytes[0x2f8..0x2fc].copy_from_slice(&4u32.to_le_bytes()); + for (offset, owner) in [ + (0, storage::HARDWARE_DATA), + (0x10, storage::UNKNOWN_PAIR), + (0x1c0, 26), + (0x1c8, 27), + (0x1d0, 28), + (0x1d8, 29), + (0x1e0, 30), + (0x1e8, 31), + (0x1f0, 32), + (0x1f8, 33), + (0x200, storage::FWLOG_PAYLOAD), + (0x254, storage::UNKNOWN_C0), + (0x25c, storage::UNKNOWN_C1), + (0x264, storage::UNKNOWN_C3), + (0x26c, RECOVERY_ACTIVITY), + (0x2d8, GLOBAL_PARAMETERS), + (0x2e8, GLOBAL_POOL), + (0x469, PERFORMANCE_CONTROL), + ] { + pointer(bytes, offset, addresses[owner]); + } + for channel in 0..13 { + let offset = 0x20 + channel * 0x20; + let state = addresses[channel * 2]; + pointer(bytes, offset, state); + pointer(bytes, offset + 8, state + 0x10); + pointer(bytes, offset + 16, state + 0x20); + pointer(bytes, offset + 24, addresses[channel * 2 + 1]); + } + Ok(()) + } + + pub(crate) fn firmware_control(&self, output: &mut [u8]) -> Result<(), Error> { + let addresses = self.addresses()?; + let bytes = output + .get_mut(..allocation(storage::CONTROL_REGION)?.size) + .ok_or(Error::Size)?; + bytes.fill(0); + bytes[0x4f88..0x4f90].copy_from_slice(&addresses[storage::FW_CONTROL_STATE].to_le_bytes()); + bytes[0x4f90..0x4f98].copy_from_slice(&addresses[storage::FW_CONTROL_RING].to_le_bytes()); + bytes[0x4fd4..0x4fd8].copy_from_slice(&1u32.to_le_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owners(delta: u64) -> [Region; COUNT] { + core::array::from_fn(|index| { + let item = allocation(index).unwrap(); + Region::new(item.address + delta, item.size).unwrap() + }) + } + + fn graph(owners: &[Region; COUNT]) -> PointerGraph<'_> { + PointerGraph { + owners, + scratch: Region::new(0xffff_fc22_0000_0000, 0x8000).unwrap(), + pool_alias: 0x70_0000_0000, + parameter_alias: 0x70_0000_8000, + } + } + + #[test] + fn complete_plan_has_disjoint_page_owners_and_nine_log_threads() { + let owners = owners(0); + assert!(graph(&owners).addresses().is_ok()); + assert_eq!(allocation(28).unwrap().size, 0x1b0); + assert_eq!(allocation(29).unwrap().size, 0x28800); + assert_eq!(allocation(41).unwrap().size, 0x79800); + assert!(allocation(COUNT).is_err()); + } + + #[test] + fn runtime_references_relocate_and_keep_the_packed_power_pointer() { + for delta in [0, 0x1000_0000] { + let owners = owners(delta); + let mut output = [0xa5; RUNTIME_SIZE + 8]; + graph(&owners).runtime(&mut output).unwrap(); + let read = |at| u64::from_le_bytes(output[at..at + 8].try_into().unwrap()); + assert_eq!( + read(0x469), + allocation(PERFORMANCE_CONTROL).unwrap().address + delta + ); + assert_eq!(read(0x1a0), allocation(24).unwrap().address + delta); + assert_eq!(read(0x1b8), allocation(25).unwrap().address + delta); + assert_eq!(read(0x2d0), 0x70_0000_8000); + assert_eq!(&output[RUNTIME_SIZE..], &[0xa5; 8]); + } + } + + #[test] + fn every_short_runtime_rejects_before_mutation() { + let owners = owners(0); + for length in 0..RUNTIME_SIZE { + let mut output = [0xa5; RUNTIME_SIZE]; + assert_eq!( + graph(&owners).runtime(&mut output[..length]), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } + } + + #[test] + fn every_owner_is_checked_before_any_record_is_written() { + for index in 0..COUNT { + let mut owners = owners(0); + let item = allocation(index).unwrap(); + owners[index] = Region::new(item.address, item.size - 1).unwrap(); + let mut output = [0xa5; RUNTIME_SIZE]; + assert!(graph(&owners).runtime(&mut output).is_err()); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } + let mut owners = owners(0); + owners[1] = owners[0]; + let mut output = [0xa5; RUNTIME_SIZE]; + assert!(graph(&owners).runtime(&mut output).is_err()); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs index 92cc818e030cc..75f06abe6f411 100644 --- a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs @@ -78,13 +78,38 @@ impl Root { (root::CONTROL, self.control.at(0, CONTROL_SIZE)?), (root::POWER, self.power.at(0, 2292)?), ]; - let bytes = out.get_mut(..ROOT_SIZE).ok_or(Error::Size)?; + Self::write_root(out, ROOT_SIZE, 0x0c08_e21e_8380_0490, &references) + } + + /// Encode the extended root without admitting a firmware image or publishing it. + pub(crate) fn encode_26(&self, out: &mut [u8], extra: Region) -> Result<(), Error> { + let references = [ + (root::REGION_A, self.region_a.at(0, 0x4000)?), + (root::RUNTIME, self.runtime.at(0, 0x4b8)?), + (root::GLOBALS, self.globals.at(0, 0xe48)?), + (root::NEW_A, self.new_a.at(0, 0x20)?), + (root::CONTROL, self.control.at(0, 0xeaf0)?), + (root::POWER, self.power.at(0, 0x670)?), + (0xc0, extra.at(0, 0x4000)?), + ]; + if references.iter().any(|(_, address)| address & 7 != 0) { + return Err(Error::Address); + } + Self::write_root(out, 0xc8, 0x0c89_c357_8392_04b8, &references) + } + + fn write_root( + out: &mut [u8], + size: usize, + version: u64, + references: &[(usize, u64)], + ) -> Result<(), Error> { + let bytes = out.get_mut(..size).ok_or(Error::Size)?; bytes.fill(0); - // Opaque qualified G15/V14_8_3 version tuple, not a semantic version. - for (i, value) in [0x490u16, 0x8380, 0xe21e, 0xc08].iter().enumerate() { - bytes[root::VERSION + i * 2..root::VERSION + i * 2 + 2].copy_from_slice(&value.to_le_bytes()); + pointer(bytes, root::VERSION, version); + for &(offset, value) in references { + pointer(bytes, offset, value); } - for (offset, value) in references { pointer(bytes, offset, value); } word(bytes, root::HOST_ALLOCATIONS, 1); bytes[root::PAGE_SIZE..root::PAGE_SIZE + 2].copy_from_slice(&0x4000u16.to_le_bytes()); bytes[root::PAGE_BITS] = 14; @@ -193,3 +218,107 @@ impl ControlRegion { Ok(()) } } + +#[cfg(test)] +mod root_format_tests { + use super::*; + + fn allocations() -> (Root, Region) { + let region = + |index: u64| Region::new(0xffff_fc01_0000_0000 + index * 0x10000, 0x10000).unwrap(); + ( + Root { + region_a: region(1), + runtime: region(2), + globals: region(3), + new_a: region(4), + control: region(5), + power: region(6), + }, + region(7), + ) + } + + #[test] + fn root_revision_changes_only_the_version_and_extension() { + let (root, extra) = allocations(); + let mut older = [0x55; 0xd0]; + let mut newer = older; + root.encode(&mut older).unwrap(); + root.encode_26(&mut newer, extra).unwrap(); + assert_eq!(&older[8..0xc0], &newer[8..0xc0]); + assert_eq!( + u64::from_le_bytes(newer[..8].try_into().unwrap()), + 0x0c89_c357_8392_04b8 + ); + assert_eq!( + u64::from_le_bytes(newer[0xc0..0xc8].try_into().unwrap()), + 0xffff_fc01_0007_0000 + ); + assert_eq!(&older[0xc0..], &[0x55; 16]); + assert_eq!(&newer[0xc8..], &[0x55; 8]); + for (offset, index) in [ + (8, 1), + (0x18, 2), + (0x20, 3), + (0xa8, 4), + (0xb0, 5), + (0xb8, 6), + ] { + assert_eq!( + u64::from_le_bytes(newer[offset..offset + 8].try_into().unwrap()), + 0xffff_fc01_0000_0000 + index * 0x10000 + ); + } + } + + #[test] + fn every_short_destination_is_unchanged() { + let (root, extra) = allocations(); + for length in 0..0xc8 { + let mut output = [0xa5; 0xc8]; + assert_eq!( + root.encode_26(&mut output[..length], extra), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; 0xc8]); + } + } + + #[test] + fn every_reference_requires_its_own_complete_allocation() { + for (index, required) in [0x4000, 0x4b8, 0xe48, 0x20, 0xeaf0, 0x670, 0x4000] + .into_iter() + .enumerate() + { + for (size, aligned, valid) in [ + (required - 1, true, false), + (required, false, false), + (required, true, true), + ] { + let (mut root, mut extra) = allocations(); + let reference = match index { + 0 => &mut root.region_a, + 1 => &mut root.runtime, + 2 => &mut root.globals, + 3 => &mut root.new_a, + 4 => &mut root.control, + 5 => &mut root.power, + _ => &mut extra, + }; + *reference = + Region::new(0xffff_fc02_0000_0000 + u64::from(!aligned), size).unwrap(); + let mut output = [0xa5; 0xc8]; + let result = root.encode_26(&mut output, extra); + assert_eq!( + result.is_ok(), + valid, + "reference {index}, size {size}, aligned {aligned}" + ); + if !valid { + assert_eq!(output, [0xa5; 0xc8]); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs new file mode 100644 index 0000000000000..e4008f239c4a0 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8122 firmware MMIO descriptors. Mapping ownership stays with the M3 runtime. + +use super::m3_init_layout::{Error, Region}; + +pub(crate) const SLOTS: usize = 53; +pub(crate) const RECORD_BYTES: usize = 40; +pub(crate) const TABLE_OFFSET: usize = 0x640; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Area { + pub(crate) physical: u64, + pub(crate) bytes: usize, + relative_register: u64, + writable: bool, + split: bool, +} + +impl Area { + pub(crate) fn page_offset(self) -> usize { + (self.physical & 0x3fff) as usize + } + + pub(crate) fn mapped_bytes(self) -> usize { + (self.page_offset() + self.bytes + 0x3fff) & !0x3fff + } + + /// At most two physical banks; the second bank occupies the next virtual pages. + pub(crate) fn banks(self) -> [(u64, usize, usize); 2] { + if self.split { + [ + (self.physical, self.bytes / 2, 0), + (self.physical + 0x200_0000, self.bytes / 2, self.bytes / 2), + ] + } else { + [(self.physical & !0x3fff, self.mapped_bytes(), 0), (0, 0, 0)] + } + } +} + +/// Register resources obtained from the admitted T8122 hardware configuration. +/// Their physical addresses are supplied by the owner, not duplicated in a wire table. +#[derive(Clone, Copy)] +pub(crate) struct RegisterBases { + pub(crate) gpu: u64, + pub(crate) interrupt: u64, + pub(crate) memory: u64, + pub(crate) telemetry_write: u64, + pub(crate) telemetry_read: u64, + pub(crate) neural: u64, + pub(crate) metrology: u64, +} + +/// Select the wire slot and map its logical hardware block independently of its VA. +pub(crate) fn area(slot: usize, bases: RegisterBases) -> Result, Error> { + let (base, offset, bytes, tagged, writable, split) = match slot { + 0 => (bases.interrupt, 0, 0x4000, false, true, false), + 3 => (bases.memory, 0, 0x30000, false, true, true), + 9 => (bases.telemetry_write, 0, 0x1000, false, true, false), + 10 => (bases.telemetry_read, 0, 0x2000, false, false, false), + 12 => (bases.neural, 0, 0x4000, false, true, false), + 14 => (bases.metrology, 0, 0x8000, false, false, false), + 17 => (bases.gpu, 0, 0x20000, false, true, false), + 26 => (bases.gpu, 0xd04000, 0x8000, true, true, false), + 27 => (bases.gpu, 0xd0d000, 0x1000, true, true, false), + 28 => (bases.gpu, 0xd50000, 0x8000, true, true, false), + 29 => (bases.gpu, 0xd10000, 0x4000, true, true, false), + 31 => (bases.gpu, 0xd40000, 0x4000, true, true, false), + 32 => (bases.gpu, 0xd60000, 0x8000, true, true, false), + 35 => (bases.gpu, 0xe00000, 0x4000, true, true, false), + 39 => (bases.gpu, 0xe08000, 0x8000, false, true, false), + 40 => (bases.gpu, 0xe1c000, 0x4000, true, true, false), + 0..SLOTS => return Ok(None), + _ => return Err(Error::Bounds), + }; + let physical = base.checked_add(offset).ok_or(Error::Address)?; + if physical == 0 || physical.checked_add(bytes as u64).ok_or(Error::Address)? > 1 << 42 { + return Err(Error::Address); + } + Ok(Some(Area { + physical, + bytes, + relative_register: if tagged { offset } else { 0 }, + writable, + split, + })) +} + +fn empty_mode(slot: usize) -> u64 { + if matches!(slot, 2 | 5..=8 | 30 | 33 | 34 | 37 | 38 | 41..=43 | 46..=49 | 52) { + 2 + } else { + 0 + } +} + +/// Encode a complete record after checking the owned mapping; errors preserve all output. +pub(crate) fn encode( + slot: usize, + bases: RegisterBases, + mapping: Option, + output: &mut [u8], +) -> Result<(), Error> { + let descriptor = area(slot, bases)?; + let (physical, va, bytes, requested, register, mode) = match (descriptor, mapping) { + (None, None) => (0, 0, 0, 0, 0, empty_mode(slot)), + (Some(item), Some(mapping)) => { + let page = mapping.at(0, item.mapped_bytes())?; + if page & 0x3fff != 0 { + return Err(Error::Address); + } + let va = page + .checked_add(item.page_offset() as u64) + .ok_or(Error::Address)?; + ( + item.physical, + va, + item.bytes as u32, + if item.split { + (item.bytes / 2) as u32 + } else { + item.bytes as u32 + }, + item.relative_register, + if item.writable { 2 } else { 0 }, + ) + } + _ => return Err(Error::Address), + }; + let bytes_out = output.get_mut(..RECORD_BYTES).ok_or(Error::Size)?; + bytes_out.fill(0); + bytes_out[0..8].copy_from_slice(&physical.to_le_bytes()); + bytes_out[8..16].copy_from_slice(&va.to_le_bytes()); + bytes_out[16..20].copy_from_slice(&bytes.to_le_bytes()); + bytes_out[20..24].copy_from_slice(&requested.to_le_bytes()); + bytes_out[24..32].copy_from_slice(®ister.to_le_bytes()); + bytes_out[32..40].copy_from_slice(&mode.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Register identities already present in the public T8122 configuration; + // memory-controller bank geometry is shared with the public 25G83 encoder. + const BASES: RegisterBases = RegisterBases { + gpu: 0x2_9000_0000, + interrupt: 0x2_d101_4000, + memory: 0x2_201c_4000, + telemetry_write: 0x2_d03d_0000, + telemetry_read: 0x2_d03c_0000, + neural: 0x3_1145_c000, + metrology: 0x2_d028_0000, + }; + + #[test] + fn j613_slot28_bank_and_wire_tag_agree() { + let bank = area(28, BASES).unwrap().unwrap(); + assert_eq!(bank.physical, BASES.gpu + 0xd50000); + assert_eq!(bank.bytes, 0x8000); + let mapped = Region::new(0xffff_fc23_01c0_0000, 0x8000).unwrap(); + let mut bytes = [0; RECORD_BYTES]; + encode(28, BASES, Some(mapped), &mut bytes).unwrap(); + assert_eq!(&bytes[0..8], &(BASES.gpu + 0xd50000).to_le_bytes()); + assert_eq!(&bytes[24..32], &0xd50000u64.to_le_bytes()); + assert_ne!(bank.physical, BASES.gpu + 0xd58000); + } + + #[test] + fn descriptor_inventory_ends_before_the_retention_pointer() { + assert_eq!(TABLE_OFFSET + SLOTS * RECORD_BYTES, 0xe88); + assert_eq!( + (0..SLOTS) + .filter(|slot| area(*slot, BASES).unwrap().is_some()) + .count(), + 16 + ); + assert!(area(SLOTS, BASES).is_err()); + } + + #[test] + fn discontiguous_memory_controller_banks_have_distinct_physical_owners() { + let item = area(3, BASES).unwrap().unwrap(); + assert_eq!( + item.banks(), + [ + (0x2_201c_4000, 0x18000, 0), + (0x2_221c_4000, 0x18000, 0x18000) + ] + ); + let mapping = Region::new(0xffff_fc23_0000_0000, 0x30000).unwrap(); + let mut bytes = [0xa5; RECORD_BYTES]; + encode(3, BASES, Some(mapping), &mut bytes).unwrap(); + assert_eq!( + u32::from_le_bytes(bytes[16..20].try_into().unwrap()), + 0x30000 + ); + assert_eq!( + u32::from_le_bytes(bytes[20..24].try_into().unwrap()), + 0x18000 + ); + } + + #[test] + fn subpage_mapping_keeps_the_register_offset() { + let item = area(27, BASES).unwrap().unwrap(); + assert_eq!(item.page_offset(), 0x1000); + assert_eq!(item.banks()[0], (0x2_90d0_c000, 0x4000, 0)); + let mut output = [0xa5; RECORD_BYTES + 8]; + encode( + 27, + BASES, + Some(Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap()), + &mut output, + ) + .unwrap(); + assert_eq!( + u64::from_le_bytes(output[8..16].try_into().unwrap()), + 0xffff_fc23_0000_1000 + ); + assert_eq!(&output[RECORD_BYTES..], &[0xa5; 8]); + } + + #[test] + fn empty_slots_keep_modes_without_fabricating_mappings() { + for slot in 0..SLOTS { + if area(slot, BASES).unwrap().is_some() { + continue; + } + let mut output = [0xa5; RECORD_BYTES]; + encode(slot, BASES, None, &mut output).unwrap(); + assert_eq!(&output[..32], &[0; 32]); + assert_eq!( + u64::from_le_bytes(output[32..40].try_into().unwrap()), + empty_mode(slot) + ); + } + } + + #[test] + fn invalid_mapping_or_output_never_partially_changes_a_descriptor() { + let good = Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap(); + for (slot, mapping) in [ + (SLOTS, None), + (0, None), + (1, Some(good)), + (0, Some(Region::new(0xffff_fc23_0000_0001, 0x4000).unwrap())), + (0, Some(Region::new(0xffff_fc23_0000_0000, 0x3fff).unwrap())), + ] { + let mut output = [0xa5; RECORD_BYTES]; + assert!(encode(slot, BASES, mapping, &mut output).is_err()); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + for length in 0..RECORD_BYTES { + let mut output = [0xa5; RECORD_BYTES]; + assert_eq!( + encode(0, BASES, Some(good), &mut output[..length]), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs new file mode 100644 index 0000000000000..c70a88029072b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-interface properties, independent of the physical G15 GPU record format. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Profile { + Legacy, + Extended, +} + +pub(crate) const PARAMS_REQUIRED: u64 = 600; + +impl Profile { + pub(crate) const fn firmware_compat(self) -> [u32; 3] { + match self { + Self::Legacy => [14, 8, 3], + Self::Extended => [26, 6, 2], + } + } + + pub(crate) const fn hal(self) -> u32 { + match self { + Self::Legacy => 0, + Self::Extended => 200, + } + } + + pub(crate) const fn user_end(self) -> u64 { + match self { + Self::Legacy => 0x2ff_ffff_8000, + Self::Extended => (1 << 42) - 2 * 0x4000, + } + } + + pub(crate) const fn vm_acknowledged(self, params_bytes: u64, flags: u32) -> bool { + // Both profiles consume G15 records; the existing VM padding stays zero. + params_bytes >= PARAMS_REQUIRED && flags == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { + for profile in [Profile::Legacy, Profile::Extended] { + let expected = 0; + for size in [0, 599, 600, 616, u64::MAX] { + for flags in [0, 1, 2, 3, u32::MAX] { + assert_eq!( + profile.vm_acknowledged(size, flags), + size >= 600 && flags == expected + ); + } + } + } + } + + #[test] + fn firmware_interface_version_does_not_change_physical_identity() { + assert_eq!(Profile::Legacy.hal(), 0); + assert_eq!(Profile::Extended.hal(), 200); + assert_eq!(Profile::Legacy.firmware_compat(), [14, 8, 3]); + assert_eq!(Profile::Extended.firmware_compat(), [26, 6, 2]); + assert_eq!(Profile::Extended.user_end(), 0x3ff_ffff_8000); + assert_eq!(Profile::Legacy.user_end(), 0x2ff_ffff_8000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 1d10657c02f66..7073ecf294d63 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -198,3 +198,7 @@ mod t8122_admission; pub(crate) mod m3_soc; mod t8122_hwdata; + +mod m3_init_26; +mod m3_profile; +mod m3_mmio_26; diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index 12bea894c373e..fdc41024899f1 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -2,6 +2,8 @@ //! Read-only T8122 identity admission. Firmware startup remains disabled. +use super::m3_profile::Profile; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Identity { pub(crate) active_cores: u32, @@ -16,12 +18,23 @@ pub(crate) fn validate_identity( result: u32, id_version: u32, masks: [u32; 3], +) -> Option { + validate_identity_for(Profile::Legacy, firmware_compat, schema, result, id_version, masks) +} + +pub(crate) fn validate_identity_for( + profile: Profile, + firmware_compat: &[u32], + schema: u32, + result: u32, + id_version: u32, + masks: [u32; 3], ) -> Option { let family = id_version >> 24; let variant = (id_version >> 16) & 0xff; let revision = (id_version >> 8) & 0xff; let active_cores = masks[0].count_ones(); - if firmware_compat != [14, 8, 3] + if firmware_compat != profile.firmware_compat() || schema != 1 || result != 1 || family != 7 @@ -94,6 +107,14 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt #[cfg(not(test))] pub(crate) fn read_identity( pdev: &kernel::platform::Device>, +) -> kernel::error::Result { + read_identity_for(pdev, Profile::Legacy) +} + +#[cfg(not(test))] +pub(crate) fn read_identity_for( + pdev: &kernel::platform::Device>, + profile: Profile, ) -> kernel::error::Result { use kernel::{bindings, c_str, prelude::*}; @@ -146,7 +167,8 @@ pub(crate) fn read_identity( observation.get_u32(c_str!("core-mask-1"))?, observation.get_u32(c_str!("core-mask-2"))?, ]; - validate_identity(firmware_compat.as_slice(), schema, result, id_version, masks).ok_or(ENODEV) + validate_identity_for(profile, firmware_compat.as_slice(), schema, result, id_version, masks) + .ok_or(ENODEV) } #[cfg(test)] @@ -164,6 +186,29 @@ mod tests { } } + #[test] + fn extended_identity_requires_its_actual_version_and_the_same_g15g_bounds() { + for mask in [0xff, 0x3ff] { + let good = || validate_identity_for(Profile::Extended, &[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]); + assert_eq!(good().unwrap().active_cores, mask.count_ones()); + assert!(validate_identity(&[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]).is_none()); + assert!(validate_identity_for(Profile::Extended, &[14, 8, 3], 1, 1, G15G_C0, [mask, 0, 0]).is_none()); + } + for (schema, result, id, masks) in [ + (0, 1, G15G_C0, [0xff, 0, 0]), + (1, 0, G15G_C0, [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 24), [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 16), [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 8), [0xff, 0, 0]), + (1, 1, G15G_C0, [0x1ff, 0, 0]), + (1, 1, G15G_C0, [0x7ff, 0, 0]), + (1, 1, G15G_C0, [0xff, 1, 0]), + (1, 1, G15G_C0, [0xff, 0, 1]), + ] { + assert!(validate_identity_for(Profile::Extended, &[26, 6, 2], schema, result, id, masks).is_none()); + } + } + /// The booted J613 ladder (`perf-states`, frequency then millivolts), in ADT order: fourteen /// states, eight voltages above the idle one, 1338 MHz at the top. const J613_PERF_STATES: [(u64, u32); 14] = [ diff --git a/tools/asahi/m3-firmware-tests.rs b/tools/asahi/m3-firmware-tests.rs new file mode 100644 index 0000000000000..90539441d623e --- /dev/null +++ b/tools/asahi/m3-firmware-tests.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Host tests of production firmware identity and normalization. +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +pub(crate) mod m3_profile; +#[path = "../../drivers/gpu/drm/asahi/m3/agx_resources.rs"] +pub(crate) mod resources; +mod m3 { + pub(crate) use crate::firmware as m3_firmware; + pub(crate) use crate::m3_profile; + pub(crate) use crate::resources as m3_resources; +} +#[path = "../../drivers/gpu/drm/asahi/m3/m3_firmware.rs"] +pub(crate) mod firmware; diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index fd4dc90489e81..f1768a242147b 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -15,3 +15,12 @@ mod m3_init_storage; mod m3_thermal_policy; #[path = "../../drivers/gpu/drm/asahi/m3/t8122_hwdata.rs"] mod t8122_hwdata; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_init_26.rs"] +mod m3_init_26; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +mod m3_profile; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_mmio_26.rs"] +mod m3_mmio_26; diff --git a/tools/asahi/m3-uapi-fixture.c b/tools/asahi/m3-uapi-fixture.c new file mode 100644 index 0000000000000..9ee0994096009 --- /dev/null +++ b/tools/asahi/m3-uapi-fixture.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#ifndef ASAHI_HEADER +#define ASAHI_HEADER "drm/asahi_drm.h" +#endif +#include ASAHI_HEADER + +static_assert(sizeof(struct drm_asahi_cmd_header) == 8, "command header"); +static_assert(sizeof(struct drm_asahi_attachment) == 24, "attachment"); +static_assert(sizeof(struct drm_asahi_timestamps) == 16, "timestamps"); +static_assert(offsetof(struct drm_asahi_cmd_compute, ts_end_copies) == 64, + "compute prefix"); +static_assert(offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies) == 256, + "render prefix"); + +static int emit(unsigned int type, const void *payload, size_t size) +{ + struct drm_asahi_cmd_header header = { + .cmd_type = type, + .size = size, + .vdm_barrier = DRM_ASAHI_BARRIER_NONE, + .cdm_barrier = DRM_ASAHI_BARRIER_NONE, + }; + + return fwrite(&header, sizeof(header), 1, stdout) != 1 || + fwrite(payload, size, 1, stdout) != 1; +} + +/* Named C offsets are exported independently of the Rust decoder. */ +static void layout(void) +{ + printf("profile.params_required %zu 0\n", + offsetof(struct drm_asahi_params_global, gpu_hal_generation) + sizeof(__u32)); + printf("profile.vm_create_size %zu 0\n", sizeof(struct drm_asahi_vm_create)); + printf("profile.hal200 %u 0\n", (unsigned int)DRM_ASAHI_GPU_HAL_200); + printf("profile.features %u 0\n", + DRM_ASAHI_FEATURE_VM_STATUS | DRM_ASAHI_FEATURE_SCHEDULED_QUEUES); + printf("header %zu 0\n", sizeof(struct drm_asahi_cmd_header)); + printf("attachment %zu 0\n", sizeof(struct drm_asahi_attachment)); + printf("timestamps %zu 0\n", sizeof(struct drm_asahi_timestamps)); + printf("cmd_render.flags %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, flags), + sizeof(((struct drm_asahi_cmd_render *)0)->flags)); + printf("cmd_render.isp_zls_pixels %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_zls_pixels), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_zls_pixels)); + printf("cmd_render.vdm_ctrl_stream_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vdm_ctrl_stream_base), + sizeof(((struct drm_asahi_cmd_render *)0)->vdm_ctrl_stream_base)); + printf("cmd_render.vertex_helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.binary), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.binary)); + printf("cmd_render.vertex_helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.cfg), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.cfg)); + printf("cmd_render.vertex_helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.data), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.data)); + printf("cmd_render.fragment_helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.binary), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.binary)); + printf("cmd_render.fragment_helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.cfg), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.cfg)); + printf("cmd_render.fragment_helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.data), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.data)); + printf("cmd_render.isp_scissor_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_scissor_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_scissor_base)); + printf("cmd_render.isp_dbias_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_dbias_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_dbias_base)); + printf("cmd_render.isp_oclqry_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_oclqry_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_oclqry_base)); + printf("cmd_render.depth.base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.base), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.base)); + printf("cmd_render.depth.comp_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.comp_base), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_base)); + printf("cmd_render.depth.stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.stride), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.stride)); + printf("cmd_render.depth.comp_stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.comp_stride), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_stride)); + printf("cmd_render.stencil.base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.base), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.base)); + printf("cmd_render.stencil.comp_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.comp_base), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_base)); + printf("cmd_render.stencil.stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.stride), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.stride)); + printf("cmd_render.stencil.comp_stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.comp_stride), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_stride)); + printf("cmd_render.zls_ctrl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, zls_ctrl), + sizeof(((struct drm_asahi_cmd_render *)0)->zls_ctrl)); + printf("cmd_render.ppp_multisamplectl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ppp_multisamplectl), + sizeof(((struct drm_asahi_cmd_render *)0)->ppp_multisamplectl)); + printf("cmd_render.sampler_heap %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sampler_heap), + sizeof(((struct drm_asahi_cmd_render *)0)->sampler_heap)); + printf("cmd_render.ppp_ctrl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ppp_ctrl), + sizeof(((struct drm_asahi_cmd_render *)0)->ppp_ctrl)); + printf("cmd_render.width_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, width_px), + sizeof(((struct drm_asahi_cmd_render *)0)->width_px)); + printf("cmd_render.height_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, height_px), + sizeof(((struct drm_asahi_cmd_render *)0)->height_px)); + printf("cmd_render.layers %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, layers), + sizeof(((struct drm_asahi_cmd_render *)0)->layers)); + printf("cmd_render.sampler_count %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sampler_count), + sizeof(((struct drm_asahi_cmd_render *)0)->sampler_count)); + printf("cmd_render.utile_width_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, utile_width_px), + sizeof(((struct drm_asahi_cmd_render *)0)->utile_width_px)); + printf("cmd_render.utile_height_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, utile_height_px), + sizeof(((struct drm_asahi_cmd_render *)0)->utile_height_px)); + printf("cmd_render.samples %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, samples), + sizeof(((struct drm_asahi_cmd_render *)0)->samples)); + printf("cmd_render.sample_size_B %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sample_size_B), + sizeof(((struct drm_asahi_cmd_render *)0)->sample_size_B)); + printf("cmd_render.isp_merge_upper_x %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_merge_upper_x), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_x)); + printf("cmd_render.isp_merge_upper_y %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_merge_upper_y), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_y)); + printf("cmd_render.bg.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->bg.usc)); + printf("cmd_render.bg.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->bg.rsrc_spec)); + printf("cmd_render.bg_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_rsrc_spec_hi)); + printf("cmd_render.eot.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, eot.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->eot.usc)); + printf("cmd_render.eot.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, eot.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->eot.rsrc_spec)); + printf("cmd_render.bg_eot_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_eot_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_rsrc_spec_hi)); + printf("cmd_render.partial_bg.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_bg.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.usc)); + printf("cmd_render.partial_bg.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_bg.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.rsrc_spec)); + printf("cmd_render.bg_partial_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_partial_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_partial_rsrc_spec_hi)); + printf("cmd_render.partial_eot.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_eot.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.usc)); + printf("cmd_render.partial_eot.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_eot.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.rsrc_spec)); + printf("cmd_render.bg_eot_partial_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_eot_partial_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_partial_rsrc_spec_hi)); + printf("cmd_render.isp_bgobjdepth %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_bgobjdepth), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjdepth)); + printf("cmd_render.isp_bgobjvals %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_bgobjvals), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjvals)); + printf("cmd_render.ts_vtx.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.start.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.handle)); + printf("cmd_render.ts_vtx.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.start.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.offset)); + printf("cmd_render.ts_vtx.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.end.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.handle)); + printf("cmd_render.ts_vtx.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.end.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.offset)); + printf("cmd_render.ts_frag.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.start.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.handle)); + printf("cmd_render.ts_frag.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.start.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.offset)); + printf("cmd_render.ts_frag.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.end.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.handle)); + printf("cmd_render.ts_frag.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.end.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.offset)); + printf("cmd_compute.flags %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, flags), + sizeof(((struct drm_asahi_cmd_compute *)0)->flags)); + printf("cmd_compute.sampler_count %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, sampler_count), + sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_count)); + printf("cmd_compute.cdm_ctrl_stream_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_base), + sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_base)); + printf("cmd_compute.cdm_ctrl_stream_end %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_end), + sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_end)); + printf("cmd_compute.sampler_heap %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, sampler_heap), + sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_heap)); + printf("cmd_compute.helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.binary), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.binary)); + printf("cmd_compute.helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.cfg), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.cfg)); + printf("cmd_compute.helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.data), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.data)); + printf("cmd_compute.ts.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.start.handle), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.handle)); + printf("cmd_compute.ts.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.start.offset), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.offset)); + printf("cmd_compute.ts.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.end.handle), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.handle)); + printf("cmd_compute.ts.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.end.offset), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.offset)); + printf("attachment.pointer %zu %zu\n", + offsetof(struct drm_asahi_attachment, pointer), + sizeof(((struct drm_asahi_attachment *)0)->pointer)); + printf("attachment.size %zu %zu\n", + offsetof(struct drm_asahi_attachment, size), + sizeof(((struct drm_asahi_attachment *)0)->size)); + printf("cmd_render.ts_frag_end_copies %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag_end_copies)); + printf("cmd_compute.ts_end_copies %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts_end_copies), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts_end_copies)); +} + +int main(int argc, char **argv) +{ + struct drm_asahi_attachment attachment = {0}; + + attachment.pointer = 0xcdb09376593c1f02ULL; + attachment.size = 0xceb194775a3d2003ULL; + + struct drm_asahi_cmd_render render = {0}; + + render.flags = 0x5b3e2104; + render.isp_zls_pixels = 0x5c3f2205; + render.vdm_ctrl_stream_base = 0xd1b4977a5d402306ULL; + render.vertex_helper.binary = 0x5e412407; + render.vertex_helper.cfg = 0x5f422508; + render.vertex_helper.data = 0xd4b79a7d60432609ULL; + render.fragment_helper.binary = 0x6144270a; + render.fragment_helper.cfg = 0x6245280b; + render.fragment_helper.data = 0xd7ba9d806346290cULL; + render.isp_scissor_base = 0xd8bb9e8164472a0dULL; + render.isp_dbias_base = 0xd9bc9f8265482b0eULL; + render.isp_oclqry_base = 0xdabda08366492c0fULL; + render.depth.base = 0xdbbea184674a2d10ULL; + render.depth.comp_base = 0xdcbfa285684b2e11ULL; + render.depth.stride = 0x694c2f12; + render.depth.comp_stride = 0x6a4d3013; + render.stencil.base = 0xdfc2a5886b4e3114ULL; + render.stencil.comp_base = 0xe0c3a6896c4f3215ULL; + render.stencil.stride = 0x6d503316; + render.stencil.comp_stride = 0x6e513417; + render.zls_ctrl = 0xe3c6a98c6f523518ULL; + render.ppp_multisamplectl = 0xe4c7aa8d70533619ULL; + render.sampler_heap = 0xe5c8ab8e7154371aULL; + render.ppp_ctrl = 0x7255381b; + render.width_px = 0x391c; + render.height_px = 0x3a1d; + render.layers = 0x3b1e; + render.sampler_count = 0x3c1f; + render.utile_width_px = 0x20; + render.utile_height_px = 0x21; + render.samples = 0x22; + render.sample_size_B = 0x23; + render.isp_merge_upper_x = 0x7b5e4124; + render.isp_merge_upper_y = 0x7c5f4225; + render.bg.usc = 0x7d604326; + render.bg.rsrc_spec = 0x7e614427; + render.bg_rsrc_spec_hi = 0x7f624528; + render.eot.usc = 0x80634629; + render.eot.rsrc_spec = 0x8164472a; + render.bg_eot_rsrc_spec_hi = 0x8265482b; + render.partial_bg.usc = 0x8366492c; + render.partial_bg.rsrc_spec = 0x84674a2d; + render.bg_partial_rsrc_spec_hi = 0x85684b2e; + render.partial_eot.usc = 0x86694c2f; + render.partial_eot.rsrc_spec = 0x876a4d30; + render.bg_eot_partial_rsrc_spec_hi = 0x886b4e31; + render.isp_bgobjdepth = 0x896c4f32; + render.isp_bgobjvals = 0x8a6d5033; + render.ts_vtx.start.handle = 0x8b6e5134; + render.ts_vtx.start.offset = 0x8c6f5235; + render.ts_vtx.end.handle = 0x8d705336; + render.ts_vtx.end.offset = 0x8e715437; + render.ts_frag.start.handle = 0x8f725538; + render.ts_frag.start.offset = 0x90735639; + render.ts_frag.end.handle = 0x9174573a; + render.ts_frag.end.offset = 0x9275583b; + + struct drm_asahi_cmd_compute compute = {0}; + + compute.flags = 0x9376593c; + compute.sampler_count = 0x94775a3d; + compute.cdm_ctrl_stream_base = 0xbeccfb295785b3eULL; + compute.cdm_ctrl_stream_end = 0xcedd0b396795c3fULL; + compute.sampler_heap = 0xdeed1b4977a5d40ULL; + compute.helper.binary = 0x987b5e41; + compute.helper.cfg = 0x997c5f42; + compute.helper.data = 0x10f1d4b79a7d6043ULL; + compute.ts.start.handle = 0x9b7e6144; + compute.ts.start.offset = 0x9c7f6245; + compute.ts.end.handle = 0x9d806346; + compute.ts.end.offset = 0x9e816447; + + size_t render_size = + offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies); + size_t compute_size = + offsetof(struct drm_asahi_cmd_compute, ts_end_copies); + + if (argc == 2 && !strcmp(argv[1], "--layout")) { + layout(); + return 0; + } else if (argc == 2 && !strcmp(argv[1], "--full")) { + render_size = sizeof(render); + compute_size = sizeof(compute); + } else if (argc != 1) { + return 1; + } + + return emit(DRM_ASAHI_SET_VERTEX_ATTACHMENTS, &attachment, + sizeof(attachment)) || + emit(DRM_ASAHI_CMD_RENDER, &render, render_size) || + emit(DRM_ASAHI_CMD_COMPUTE, &compute, compute_size); +} diff --git a/tools/asahi/m3-uapi-tests.rs b/tools/asahi/m3-uapi-tests.rs new file mode 100644 index 0000000000000..4dc6012f5213e --- /dev/null +++ b/tools/asahi/m3-uapi-tests.rs @@ -0,0 +1,832 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Offline tests of the production parser. All addresses are synthetic. +#![allow(dead_code)] + +#[path = "../../drivers/gpu/drm/asahi/m3/t8122_admission.rs"] +mod admission; +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +mod m3_profile; +#[path = "../../drivers/gpu/drm/asahi/m3/agx_uapi.rs"] +mod uapi; +use uapi::*; + +fn frame(kind: u16, payload: &[u8]) -> Vec { + let mut bytes = Vec::new(); + for word in [ + kind, + payload.len() as u16, + UAPI_BARRIER_NONE, + UAPI_BARRIER_NONE, + ] { + bytes.extend_from_slice(&word.to_le_bytes()); + } + bytes.extend_from_slice(payload); + bytes +} + +fn parse(bytes: &[u8]) -> Result { + let mut parser = UapiCommandParser::new(bytes); + while parser.next_hardware()?.is_some() {} + parser.finish() +} + +// Each offset below is checked against offsetof of the corresponding C member. +fn layout() -> &'static std::collections::BTreeMap { + static LAYOUT: std::sync::OnceLock> = + std::sync::OnceLock::new(); + LAYOUT.get_or_init(|| { + let path = std::env::var("ASAHI_UAPI_LAYOUT").expect("C layout path"); + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| { + let words: Vec<_> = line.split_whitespace().collect(); + ( + words[0].to_string(), + (words[1].parse().unwrap(), words[2].parse().unwrap()), + ) + }) + .collect() + }) +} + +fn field(raw: &[u8], name: &str, offset: usize, width: usize, actual: u64) { + assert_eq!(layout()[name], (offset, width), "C offset/width for {name}"); + let mut expected = [0u8; 8]; + for (i, byte) in expected[..width].iter_mut().enumerate() { + *byte = raw.get(offset + i).copied().unwrap_or(0); + } + assert_eq!(actual, u64::from_le_bytes(expected), "decoded {name}"); +} + +fn check_render(p: UapiRenderCommand, raw: &[u8]) { + field(raw, "cmd_render.flags", 0, 4, p.flags as u64); + field( + raw, + "cmd_render.isp_zls_pixels", + 4, + 4, + p.depth_dimensions as u64, + ); + field( + raw, + "cmd_render.vdm_ctrl_stream_base", + 8, + 8, + p.vdm_base as u64, + ); + field( + raw, + "cmd_render.vertex_helper.binary", + 16, + 4, + p.vertex_helper.binary as u64, + ); + field( + raw, + "cmd_render.vertex_helper.cfg", + 20, + 4, + p.vertex_helper.config as u64, + ); + field( + raw, + "cmd_render.vertex_helper.data", + 24, + 8, + p.vertex_helper.data as u64, + ); + field( + raw, + "cmd_render.fragment_helper.binary", + 32, + 4, + p.fragment_helper.binary as u64, + ); + field( + raw, + "cmd_render.fragment_helper.cfg", + 36, + 4, + p.fragment_helper.config as u64, + ); + field( + raw, + "cmd_render.fragment_helper.data", + 40, + 8, + p.fragment_helper.data as u64, + ); + field( + raw, + "cmd_render.isp_scissor_base", + 48, + 8, + p.scissor_base as u64, + ); + field( + raw, + "cmd_render.isp_dbias_base", + 56, + 8, + p.depth_bias_base as u64, + ); + field( + raw, + "cmd_render.isp_oclqry_base", + 64, + 8, + p.occlusion_query_base as u64, + ); + field(raw, "cmd_render.depth.base", 72, 8, p.depth.base as u64); + field( + raw, + "cmd_render.depth.comp_base", + 80, + 8, + p.depth.compression_base as u64, + ); + field(raw, "cmd_render.depth.stride", 88, 4, p.depth.stride as u64); + field( + raw, + "cmd_render.depth.comp_stride", + 92, + 4, + p.depth.compression_stride as u64, + ); + field(raw, "cmd_render.stencil.base", 96, 8, p.stencil.base as u64); + field( + raw, + "cmd_render.stencil.comp_base", + 104, + 8, + p.stencil.compression_base as u64, + ); + field( + raw, + "cmd_render.stencil.stride", + 112, + 4, + p.stencil.stride as u64, + ); + field( + raw, + "cmd_render.stencil.comp_stride", + 116, + 4, + p.stencil.compression_stride as u64, + ); + field(raw, "cmd_render.zls_ctrl", 120, 8, p.zls_control as u64); + field( + raw, + "cmd_render.ppp_multisamplectl", + 128, + 8, + p.multisample_control as u64, + ); + field( + raw, + "cmd_render.sampler_heap", + 136, + 8, + p.sampler_heap as u64, + ); + field(raw, "cmd_render.ppp_ctrl", 144, 4, p.ppp_control as u64); + field(raw, "cmd_render.width_px", 148, 2, p.width as u64); + field(raw, "cmd_render.height_px", 150, 2, p.height as u64); + field(raw, "cmd_render.layers", 152, 2, p.layers as u64); + field( + raw, + "cmd_render.sampler_count", + 154, + 2, + p.sampler_count as u64, + ); + field( + raw, + "cmd_render.utile_width_px", + 156, + 1, + p.utile_width as u64, + ); + field( + raw, + "cmd_render.utile_height_px", + 157, + 1, + p.utile_height as u64, + ); + field(raw, "cmd_render.samples", 158, 1, p.samples as u64); + field( + raw, + "cmd_render.sample_size_B", + 159, + 1, + p.sample_size as u64, + ); + field( + raw, + "cmd_render.isp_merge_upper_x", + 160, + 4, + p.merge_upper_x as u64, + ); + field( + raw, + "cmd_render.isp_merge_upper_y", + 164, + 4, + p.merge_upper_y as u64, + ); + field(raw, "cmd_render.bg.usc", 168, 4, p.background.usc as u64); + field( + raw, + "cmd_render.bg.rsrc_spec", + 172, + 4, + (p.background.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_rsrc_spec_hi", + 248, + 4, + (p.background.resource_spec >> 32) as u64, + ); + field(raw, "cmd_render.eot.usc", 176, 4, p.end_of_tile.usc as u64); + field( + raw, + "cmd_render.eot.rsrc_spec", + 180, + 4, + (p.end_of_tile.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_eot_rsrc_spec_hi", + 240, + 4, + (p.end_of_tile.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.partial_bg.usc", + 184, + 4, + p.partial_background.usc as u64, + ); + field( + raw, + "cmd_render.partial_bg.rsrc_spec", + 188, + 4, + (p.partial_background.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_partial_rsrc_spec_hi", + 252, + 4, + (p.partial_background.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.partial_eot.usc", + 192, + 4, + p.partial_end_of_tile.usc as u64, + ); + field( + raw, + "cmd_render.partial_eot.rsrc_spec", + 196, + 4, + (p.partial_end_of_tile.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_eot_partial_rsrc_spec_hi", + 244, + 4, + (p.partial_end_of_tile.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.isp_bgobjdepth", + 200, + 4, + p.depth_clear as u64, + ); + field( + raw, + "cmd_render.isp_bgobjvals", + 204, + 4, + p.stencil_clear as u64, + ); + field( + raw, + "cmd_render.ts_vtx.start.handle", + 208, + 4, + p.vertex_timestamps.start.handle as u64, + ); + field( + raw, + "cmd_render.ts_vtx.start.offset", + 212, + 4, + p.vertex_timestamps.start.offset as u64, + ); + field( + raw, + "cmd_render.ts_vtx.end.handle", + 216, + 4, + p.vertex_timestamps.end.handle as u64, + ); + field( + raw, + "cmd_render.ts_vtx.end.offset", + 220, + 4, + p.vertex_timestamps.end.offset as u64, + ); + field( + raw, + "cmd_render.ts_frag.start.handle", + 224, + 4, + p.fragment_timestamps.start.handle as u64, + ); + field( + raw, + "cmd_render.ts_frag.start.offset", + 228, + 4, + p.fragment_timestamps.start.offset as u64, + ); + field( + raw, + "cmd_render.ts_frag.end.handle", + 232, + 4, + p.fragment_timestamps.end.handle as u64, + ); + field( + raw, + "cmd_render.ts_frag.end.offset", + 236, + 4, + p.fragment_timestamps.end.offset as u64, + ); +} + +fn check_compute(p: UapiComputeCommand, raw: &[u8]) { + field(raw, "cmd_compute.flags", 0, 4, p.flags as u64); + field( + raw, + "cmd_compute.sampler_count", + 4, + 4, + p.sampler_count as u64, + ); + field( + raw, + "cmd_compute.cdm_ctrl_stream_base", + 8, + 8, + p.control_stream_base as u64, + ); + field( + raw, + "cmd_compute.cdm_ctrl_stream_end", + 16, + 8, + p.control_stream_end as u64, + ); + field( + raw, + "cmd_compute.sampler_heap", + 24, + 8, + p.sampler_heap as u64, + ); + field( + raw, + "cmd_compute.helper.binary", + 32, + 4, + p.helper.binary as u64, + ); + field(raw, "cmd_compute.helper.cfg", 36, 4, p.helper.config as u64); + field(raw, "cmd_compute.helper.data", 40, 8, p.helper.data as u64); + field( + raw, + "cmd_compute.ts.start.handle", + 48, + 4, + p.timestamps.start.handle as u64, + ); + field( + raw, + "cmd_compute.ts.start.offset", + 52, + 4, + p.timestamps.start.offset as u64, + ); + field( + raw, + "cmd_compute.ts.end.handle", + 56, + 4, + p.timestamps.end.handle as u64, + ); + field( + raw, + "cmd_compute.ts.end.offset", + 60, + 4, + p.timestamps.end.offset as u64, + ); +} + +fn fixture() -> Vec { + std::fs::read(std::env::var("ASAHI_UAPI_FIXTURE").expect("C fixture path")).unwrap() +} + +#[test] +fn c_header_producer_matches_production_decoder() { + let offsets = layout(); + assert_eq!(offsets["header"].0, UAPI_COMMAND_HEADER_SIZE); + assert_eq!(offsets["attachment"].0, UAPI_ATTACHMENT_SIZE); + assert_eq!(offsets["timestamps"].0, 16); + assert_eq!(offsets["cmd_render.ts_frag_end_copies"].0, UAPI_RENDER_SIZE); + assert_eq!(offsets["cmd_compute.ts_end_copies"].0, UAPI_COMPUTE_SIZE); + let bytes = fixture(); + let render_offset = 2 * UAPI_COMMAND_HEADER_SIZE + UAPI_ATTACHMENT_SIZE; + let render_size = u16::from_le_bytes( + bytes[render_offset - 6..render_offset - 4] + .try_into() + .unwrap(), + ) as usize; + let compute_offset = render_offset + render_size + UAPI_COMMAND_HEADER_SIZE; + let mut parser = UapiCommandParser::new(&bytes); + let Some(ParsedHardwareCommand::Render { + index, + header, + payload, + vertex_attachments, + fragment_attachments, + }) = parser.next_hardware().unwrap() + else { + panic!("render missing") + }; + assert_eq!(index, 2); + assert_eq!( + header, + UapiCommandHeader { + command_type: 0, + size: render_size as u16, + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + } + ); + check_render(payload, &bytes[render_offset..render_offset + render_size]); + assert_eq!(vertex_attachments.count, 1); + let attachment = vertex_attachments.as_slice()[0]; + field( + &bytes[8..32], + "attachment.pointer", + 0, + 8, + attachment.address, + ); + field(&bytes[8..32], "attachment.size", 8, 8, attachment.size); + assert!(fragment_attachments.as_slice().is_empty()); + let Some(ParsedHardwareCommand::Compute { + index, + header, + payload, + attachments, + }) = parser.next_hardware().unwrap() + else { + panic!("compute missing") + }; + assert_eq!(index, 3); + assert_eq!( + header, + UapiCommandHeader { + command_type: 1, + size: (bytes.len() - compute_offset) as u16, + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + } + ); + check_compute(payload, &bytes[compute_offset..]); + assert!(attachments.as_slice().is_empty()); + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(2)); +} + +#[test] +fn short_payloads_zero_extend_but_missing_bytes_are_rejected() { + // Every length includes cuts within each multi-byte field, for both kinds. + for (kind, size) in [(0, 256), (1, 64)] { + let supplied: Vec = (0..size).map(|i| (i % 254 + 1) as u8).collect(); + for length in 0..=size { + let raw = &supplied[..length]; + let bytes = frame(kind, raw); + let mut parser = UapiCommandParser::new(&bytes); + match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { payload, .. } => check_render(payload, raw), + ParsedHardwareCommand::Compute { payload, .. } => check_compute(payload, raw), + } + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + for cut in 1..bytes.len() { + assert!(parse(&bytes[..cut]).is_err()); + } + } + } + assert_eq!(parse(&[]), Err(UapiTranslateError::HardwareCommandCount)); +} + +#[test] +fn named_optional_timestamp_copies_are_refused() { + for (kind, name) in [ + (0, "cmd_render.ts_frag_end_copies"), + (1, "cmd_compute.ts_end_copies"), + ] { + let (offset, size) = layout()[name]; + let mut raw = vec![0; offset + size]; + assert_eq!(parse(&frame(kind, &raw)), Ok(1)); + for i in offset..raw.len() { + raw[i] = 1; + assert_eq!( + parse(&frame(kind, &raw)), + Err(UapiTranslateError::PayloadTail), + "{name} byte {i}" + ); + raw[i] = 0; + } + } +} + +#[test] +fn unsupported_nonzero_extensions_never_disappear() { + for (kind, size) in [(0, UAPI_RENDER_SIZE), (1, UAPI_COMPUTE_SIZE)] { + for extra in [1, 8, 64, 128] { + let mut payload = vec![0; size + extra]; + assert_eq!(parse(&frame(kind, &payload)), Ok(1)); + for offset in size..payload.len() { + payload[offset] = 1; + assert_eq!( + parse(&frame(kind, &payload)), + Err(UapiTranslateError::PayloadTail) + ); + payload[offset] = 0; + } + } + } + assert_eq!( + parse(&frame(0xffff, &[])), + Err(UapiTranslateError::CommandType) + ); +} + +#[test] +fn attachment_shapes_counts_and_reserved_fields_are_checked() { + for kind in 2..=4 { + for size in 1..UAPI_ATTACHMENT_SIZE { + assert_eq!( + parse(&frame(kind, &vec![0; size])), + Err(UapiTranslateError::AttachmentPayloadSize) + ); + } + let mut bytes = frame(kind, &vec![0; UAPI_ATTACHMENT_SIZE * UAPI_MAX_ATTACHMENTS]); + bytes.extend(frame(1, &[])); + assert_eq!(parse(&bytes), Ok(1)); + assert_eq!( + parse(&frame( + kind, + &vec![0; UAPI_ATTACHMENT_SIZE * (UAPI_MAX_ATTACHMENTS + 1)] + )), + Err(UapiTranslateError::AttachmentCount) + ); + for offset in 16..24 { + let mut payload = [0; UAPI_ATTACHMENT_SIZE]; + payload[offset] = 1; + assert_eq!( + parse(&frame(kind, &payload)), + Err(UapiTranslateError::AttachmentFields) + ); + } + let mut bytes = frame(kind, &[]); + bytes[4..6].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(parse(&bytes), Err(UapiTranslateError::AttachmentBarrier)); + } +} + +#[test] +fn attachment_lists_reach_their_commands() { + let entry = |pointer: u64, size: u64| { + let mut raw = [0u8; UAPI_ATTACHMENT_SIZE]; + raw[..8].copy_from_slice(&pointer.to_le_bytes()); + raw[8..16].copy_from_slice(&size.to_le_bytes()); + raw + }; + let vertex = entry(0x1_2345_6000, 0x1_0000_0040).to_vec(); + let fragment = [ + entry(0x2_0000_8000, 0x80), + entry(0x3_0000_c000, 0x2_0000_0100), + ] + .concat(); + let compute = entry(0x4_0000_1000, 0x5_0000_0010).to_vec(); + let check = |list: UapiAttachmentList, raw: &[u8]| { + assert_eq!(list.as_slice().len(), raw.len() / UAPI_ATTACHMENT_SIZE); + for (attachment, raw) in list.as_slice().iter().zip(raw.chunks(UAPI_ATTACHMENT_SIZE)) { + field(raw, "attachment.pointer", 0, 8, attachment.address); + field(raw, "attachment.size", 8, 8, attachment.size); + } + }; + + // Lists persist until replaced; an empty list clears only its own kind. + let mut bytes = frame(2, &vertex); + bytes.extend(frame(3, &fragment)); + bytes.extend(frame(4, &compute)); + bytes.extend(frame(0, &[])); + bytes.extend(frame(1, &[])); + bytes.extend(frame(3, &[])); + bytes.extend(frame(0, &[])); + let mut parser = UapiCommandParser::new(&bytes); + for expected_fragment in [&fragment[..], &[]] { + let Some(ParsedHardwareCommand::Render { + vertex_attachments, + fragment_attachments, + .. + }) = parser.next_hardware().unwrap() + else { + panic!("render missing") + }; + check(vertex_attachments, &vertex); + check(fragment_attachments, expected_fragment); + if !expected_fragment.is_empty() { + let Some(ParsedHardwareCommand::Compute { attachments, .. }) = + parser.next_hardware().unwrap() + else { + panic!("compute missing") + }; + check(attachments, &compute); + } + } + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(3)); +} + +#[test] +fn dependencies_count_hardware_commands_and_bound_the_submit() { + for field in [4, 6] { + let mut bytes = frame(1, &[]); + bytes[field..field + 2].copy_from_slice(&1u16.to_le_bytes()); + assert_eq!(parse(&bytes), Err(UapiTranslateError::BarrierFuture)); + } + let mut bytes = frame(2, &[]); + bytes.extend(frame(0, &[])); + let mut compute = frame(1, &[]); + compute[4..6].copy_from_slice(&1u16.to_le_bytes()); + bytes.extend(compute); + assert_eq!(parse(&bytes), Ok(2)); + let command = frame(1, &[]); + assert_eq!( + parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize)), + Ok(UAPI_MAX_HARDWARE_COMMANDS) + ); + assert_eq!( + parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize + 1)), + Err(UapiTranslateError::HardwareCommandCount) + ); +} + +struct Region { + start: u64, + end: u64, + writable: bool, +} +impl GpuAddressSpace for Region { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool { + let Some(end) = address.checked_add(size) else { + return false; + }; + address >= self.start && end <= self.end && (access == GpuAccess::Read || self.writable) + } +} + +#[test] +fn attachment_validation_rejects_readonly_zero_and_wrapped_ranges() { + let mut list = UapiAttachmentList::EMPTY; + list.count = 1; + let mut region = Region { + start: 0x4000, + end: 0x8000, + writable: false, + }; + list.entries[0] = UapiAttachment { + address: 0x4000, + size: 0x4000, + }; + assert_eq!( + validate_attachments(®ion, &list), + Err(UapiTranslateError::AddressRange) + ); + region.writable = true; + assert_eq!(validate_attachments(®ion, &list), Ok(())); + for (address, size) in [(0x4000, 0), (0x4000, 0x4001), (u64::MAX, 2)] { + list.entries[0] = UapiAttachment { address, size }; + assert_eq!( + validate_attachments(®ion, &list), + Err(UapiTranslateError::AddressRange) + ); + } +} + +#[test] +fn usc_windows_reject_misalignment_overflow_and_unmapped_programs() { + let queue = QueueUscWindow { + base: USC_WINDOW_SIZE, + user_start: 0x4000, + user_end: 2 * USC_WINDOW_SIZE, + }; + assert_eq!(queue.validate(), Ok(())); + assert_eq!( + QueueUscWindow { + base: queue.base + 1, + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscAlignment) + ); + assert_eq!( + QueueUscWindow { + base: u64::MAX & !(USC_WINDOW_SIZE - 1), + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscRange) + ); + assert_eq!( + QueueUscWindow { + user_end: queue.user_end - 1, + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscRange) + ); + let region = Region { + start: queue.base + 8, + end: queue.base + 12, + writable: false, + }; + assert_eq!(queue.full_program(®ion, 9), Ok(queue.base + 8)); + assert_eq!( + queue.full_program(®ion, 0), + Err(UapiTranslateError::ProgramOffset) + ); + assert_eq!( + queue.full_program(®ion, 16), + Err(UapiTranslateError::AddressRange) + ); +} + +#[test] +fn unsupported_firmware_cannot_reach_the_existing_profile() { + let identity = (1, 1, 0x07022000, [0xff, 0, 0]); + assert!(admission::validate_identity( + &[14, 8, 3], + identity.0, + identity.1, + identity.2, + identity.3 + ) + .is_some()); + for version in [&[26, 6, 2][..], &[14, 8][..], &[14, 8, 3, 0][..], &[][..]] { + assert!(admission::validate_identity( + version, identity.0, identity.1, identity.2, identity.3 + ) + .is_none()); + } +} + +#[test] +fn firmware_profile_agrees_with_the_actual_c_header() { + assert_eq!( + layout()["profile.params_required"].0 as u64, + m3_profile::PARAMS_REQUIRED + ); + assert_eq!(layout()["profile.vm_create_size"].0, 24); + assert_eq!( + layout()["profile.hal200"].0 as u32, + m3_profile::Profile::Extended.hal() + ); + assert_eq!(layout()["profile.features"].0, 0x0a); +} diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index c4c6e21383fbf..04a2670f39b6c 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,7 +53,9 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], + 'test-j613-dcp-power.py': [([], PASS)], 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], @@ -171,6 +173,7 @@ def mutations(flag, names, base, expect=KILL): # Production G17 decoder vectors also build directly with rustc --test. ASAHI_RUST.append('g17-telemetry-tests.rs') +ASAHI_RUST.append('m3-firmware-tests.rs') # Python that runs on the host but imports its neighbours, so it keeps the # script directory on sys.path (-E -s instead of -I). diff --git a/tools/asahi/test-j613-dcp-power.py b/tools/asahi/test-j613-dcp-power.py new file mode 100644 index 0000000000000..3bc1a3379c253 --- /dev/null +++ b/tools/asahi/test-j613-dcp-power.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production J613 PMGR predicate and test PMGR fields and unrelated bits.""" +from pathlib import Path +import os +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/j613_dcp_activate.c").read_text() +start = source.index("static bool j613_power_state_active(") +end = source.index("\n}\n", start) + 3 +stub = r""" +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(a,b) (((1U << ((a)-(b)+1))-1) << (b)) +#define FIELD_GET(mask,v) (((v) & (mask)) >> __builtin_ctz(mask)) +""" +main = r""" +int main(void) { + /* Only ACTUAL[7:4], TARGET[3:0] and AUTO_ENABLE[28] affect readiness. */ + const u32 unrelated_mask = ~(GENMASK(7, 0) | BIT(28)); + u32 unrelated[32] = {0, unrelated_mask, + 0xaaaaaaaaU & unrelated_mask, + 0x55555555U & unrelated_mask}; + unsigned int count = 4; + for (unsigned int bit = 8; bit < 32; bit++) + if (bit != 28) unrelated[count++] = BIT(bit); + for (unsigned int i = 0; i < count; i++) + for (u32 actual = 0; actual < 16; actual++) + for (u32 target = 0; target < 16; target++) + for (u32 automatic = 0; automatic < 2; automatic++) { + u32 word = unrelated[i] | (actual << 4) | target | (automatic << 28); + bool expected = actual == 15 || (target == 15 && automatic); + if (j613_power_state_active(word) != expected) { + fprintf(stderr, "PMGR state 0x%08x: expected active=%d\n", word, expected); + return 1; + } + } + if (!j613_power_state_active(0x1404024f)) return 2; + if (j613_power_state_active(0x0404024f)) return 3; + if (j613_power_state_active(0x14040240)) return 4; + puts("PASS: 13824 PMGR combinations and captured-state regression controls"); + return 0; +} +""" +with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "power.c" + path.write_text(stub + source[start:end] + main) + for flags in (["-O0"], ["-O2"]): + exe = str(path.with_suffix("")) + subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", + *flags, str(path), "-o", exe], check=True) + subprocess.run([exe], check=True) diff --git a/tools/asahi/test-m3-mailbox.py b/tools/asahi/test-m3-mailbox.py index 2320d17f10f6f..167b0566b2e5d 100644 --- a/tools/asahi/test-m3-mailbox.py +++ b/tools/asahi/test-m3-mailbox.py @@ -77,6 +77,17 @@ def vec(values): assert!(!ok(&compat,®,&names,&irqs,1)); let mut v=names.clone(); v[0]^=1; assert!(!ok(&compat,®,&v,&irqs,cells)); for v in [b"apple,asc-mailbox-v4\\0".to_vec(), b"apple,t8122-agx-asc-mailbox\\0unknown\\0".to_vec(), b"apple,t6031-agx-asc-mailbox\\0".to_vec()] { assert!(!ok(&v,®,&names,&irqs,cells)); } + // Compatible lists are ordered: reversing either supported pair must fail. + for pair in soc.mailbox_compatibles { + let parts: Vec<_> = pair.split(|byte| *byte == 0).filter(|part| !part.is_empty()).collect(); + if parts.len() == 2 { + let mut reversed = parts[1].to_vec(); + reversed.push(0); + reversed.extend_from_slice(parts[0]); + reversed.push(0); + assert!(!ok(&reversed,®,&names,&irqs,cells)); + } + } for v in soc.mailbox_compatibles { assert!(ok(&v.to_vec(),®,&names,&irqs,cells)); } } fn main() { diff --git a/tools/asahi/test-m3-uapi.py b/tools/asahi/test-m3-uapi.py new file mode 100755 index 0000000000000..a63249e18869d --- /dev/null +++ b/tools/asahi/test-m3-uapi.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile public C producers and run the production Rust parser offline.""" +import argparse +import hashlib +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def run(command, **kwargs): + subprocess.run(command, check=True, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mesa", type=Path, help="matched public Mesa source") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + if sys.byteorder != "little": + raise SystemExit("C fixtures currently require a little-endian host") + cc = os.environ.get("CC", "cc") + rustc = os.environ.get("RUSTC", "rustc") + fixture_source = root / "tools/asahi/m3-uapi-fixture.c" + with tempfile.TemporaryDirectory(prefix="asahi-uapi-") as temporary: + work = Path(temporary) + headers = work / "headers" + run(["make", "-s", "-C", str(root), "O=" + str(work / "headers-build"), + "ARCH=arm64", "headers_install", "INSTALL_HDR_PATH=" + str(headers)]) + sources = [(headers / "include", "drm/asahi_drm.h")] + if args.mesa: + sources.append((args.mesa.resolve() / "include", "drm-uapi/asahi_drm.h")) + fixtures = [] + layouts = [] + prefixes = [] + for index, (includes, header) in enumerate(sources): + producer = work / ("producer-" + str(index)) + run([cc, "-std=c11", "-Wall", "-Wextra", "-Werror", + "-I" + str(includes), '-DASAHI_HEADER="' + header + '"', + str(fixture_source), "-o", str(producer)]) + prefix = work / ("prefix-" + str(index)) + with prefix.open("wb") as output: + run([str(producer)], stdout=output) + prefixes.append(prefix) + layout = work / ("layout-" + str(index)) + with layout.open("wb") as output: + run([str(producer), "--layout"], stdout=output) + layouts.append(layout) + fixture = work / ("fixture-" + str(index)) + with fixture.open("wb") as output: + run([str(producer), "--full"], stdout=output) + fixtures.append(fixture) + reference = prefixes[0].read_bytes() + if any(prefix.read_bytes() != reference for prefix in prefixes[1:]): + raise SystemExit("kernel/Mesa canonical fixture mismatch") + print("canonical fixture sha256=" + hashlib.sha256(reference).hexdigest(), flush=True) + for optimized in (False, True): + binary = work / ("tests-optimized" if optimized else "tests-debug") + command = [rustc, "--edition=2021", "--test", + str(root / "tools/asahi/m3-uapi-tests.rs"), "-o", str(binary)] + if optimized: + command.append("-O") + run(command) + for index, fixture in enumerate(prefixes + fixtures): + env = dict(os.environ, ASAHI_UAPI_FIXTURE=str(fixture), + ASAHI_UAPI_LAYOUT=str(layouts[index % len(layouts)])) + run([str(binary)], env=env) + + +if __name__ == "__main__": + main() From 32ad46810d2a07ffec5f7c1aa675bdb2b772024d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:38:59 -0400 Subject: [PATCH 033/392] drm/asahi: construct and publish extended T8122 initialization data Build extended initialization owners, controller images and channel records with checked allocation spans. Publish the bounded loader table through existing M3 ownership. Keep extended runtime admission closed. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_channel_format.rs | 182 ++++++++ drivers/gpu/drm/asahi/m3/m3_compute_layout.rs | 171 ++++++-- drivers/gpu/drm/asahi/m3/m3_config.rs | 179 +++++++- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 59 +++ drivers/gpu/drm/asahi/m3/m3_init_26.rs | 22 +- drivers/gpu/drm/asahi/m3/m3_mmio_26.rs | 33 +- drivers/gpu/drm/asahi/m3/m3_power_26.rs | 414 ++++++++++++++++++ drivers/gpu/drm/asahi/m3/mod.rs | 5 + tools/asahi/m3-runtime-tests.rs | 12 + tools/asahi/run-host-tests.py | 1 + tools/asahi/test-m3-power.py | 173 ++++++++ 11 files changed, 1215 insertions(+), 36 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_channel_format.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_power_26.rs create mode 100644 tools/asahi/test-m3-power.py diff --git a/drivers/gpu/drm/asahi/m3/m3_channel_format.rs b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs new file mode 100644 index 0000000000000..45eebda112b73 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded firmware channel records. Publication and retirement belong to Config. + +use super::m3_init_layout::Error; + +pub(crate) const CONTROL_BYTES: usize = 64; +pub(crate) const RECEIVE_BYTES: usize = 72; +pub(crate) const SLOTS: u32 = 256; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Action { + Initialize, + Register { + pipe: u32, + engine: u32, + }, + Idle { + enabled: bool, + }, + Release { + group: u8, + index: u8, + weight: u8, + address: u64, + }, +} + +impl Action { + /// The first two actions precede the first generation-bearing publication. + pub(crate) fn generation(self, published: u32, consumed: u32) -> Result { + match self { + Self::Initialize | Self::Register { .. } => { + if published != 0 || consumed != 0 { + return Err(Error::Address); + } + Ok(0) + } + _ => published.max(consumed).checked_add(1).ok_or(Error::Address), + } + } + + pub(crate) fn encode(self, output: &mut [u8]) -> Result<(), Error> { + let mut words = [0u32; 3]; + let mut release = None; + match self { + Self::Initialize => words[0] = 0x16, + Self::Register { pipe, engine } => { + if pipe >= 3 || !(1..=2).contains(&engine) { + return Err(Error::Address); + } + words = [0x1b, pipe, engine]; + } + Self::Idle { enabled } => words = [0x0a, enabled.into(), 0], + Self::Release { + group, + index, + weight, + address, + } => { + if index == 255 + || weight == 255 + || address < 0xffff_fc00_0000_0000 + || address & 7 != 0 + { + return Err(Error::Address); + } + words[0] = 0x14; + words[2] = u32::from_le_bytes([group, index, weight, 4]); + release = Some(address); + } + } + let bytes = output.get_mut(..CONTROL_BYTES).ok_or(Error::Size)?; + bytes.fill(0); + for (index, word) in words.iter().enumerate() { + bytes[index * 4..index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + if let Some(address) = release { + bytes[12..20].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + } +} + +/// An empty ring is required before publishing a synchronous control transaction. +/// Both consumers must retire it before the next control action may be published. +pub(crate) fn idle_cursor(cursors: [u32; 3]) -> Result { + if cursors.iter().any(|value| *value >= SLOTS) { + return Err(Error::Bounds); + } + if cursors[0] != cursors[2] || cursors[1] != cursors[2] { + return Err(Error::Address); + } + Ok(cursors[2]) +} + +pub(crate) fn payload_offset(thread: usize, slot: u32) -> Result { + if thread >= 9 { + return Err(Error::Bounds); + } + Ok((thread * 256 + (slot & 255) as usize) * 0xd8) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn initialization_records_have_only_their_defined_words() { + for action in [Action::Initialize, Action::Register { pipe: 2, engine: 1 }] { + let mut output = [0xa5; CONTROL_BYTES + 8]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[12..CONTROL_BYTES], &[0; CONTROL_BYTES - 12]); + assert_eq!(&output[CONTROL_BYTES..], &[0xa5; 8]); + assert_eq!(action.generation(0, 0), Ok(0)); + assert!(action.generation(1, 0).is_err()); + } + } + + #[test] + fn invalid_actions_preserve_the_entire_output() { + for action in [ + Action::Register { pipe: 3, engine: 1 }, + Action::Register { pipe: 0, engine: 0 }, + Action::Release { + group: 0, + index: 255, + weight: 1, + address: 0xffff_fc21_0000_0000, + }, + Action::Release { + group: 0, + index: 0, + weight: 1, + address: 8, + }, + ] { + let mut output = [0xa5; CONTROL_BYTES]; + assert!(action.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; CONTROL_BYTES]); + } + } + + #[test] + fn generations_follow_consumption_and_never_wrap() { + let action = Action::Idle { enabled: true }; + assert_eq!(action.generation(3, 5), Ok(6)); + assert_eq!(action.generation(5, 3), Ok(6)); + assert!(action.generation(u32::MAX, 0).is_err()); + assert!(action.generation(0, u32::MAX).is_err()); + let mut output = [0; CONTROL_BYTES]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[..8], &[10, 0, 0, 0, 1, 0, 0, 0]); + } + + #[test] + fn both_consumers_and_all_indices_are_checked() { + assert_eq!(idle_cursor([255, 255, 255]), Ok(255)); + for cursors in [[0, 1, 1], [1, 0, 1], [256, 256, 256]] { + assert!(idle_cursor(cursors).is_err()); + } + assert_eq!(payload_offset(8, 0x101), Ok((8 * 256 + 1) * 0xd8)); + assert!(payload_offset(9, 0).is_err()); + } + + #[test] + fn release_uses_the_packed_owned_pointer() { + let action = Action::Release { + group: 1, + index: 2, + weight: 3, + address: 0xffff_fc21_0400_0000, + }; + let mut output = [0; CONTROL_BYTES]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[8..12], &[1, 2, 3, 4]); + assert_eq!( + u64::from_le_bytes(output[12..20].try_into().unwrap()), + 0xffff_fc21_0400_0000 + ); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs index 74bed858b71e2..64c8cb005c4a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT use super::{m3_pool_layout::GpuRegion, m3_queue_layout::{Error, FirmwareVa}}; pub(crate) const SIZE: usize = 0x893; +pub(crate) const EXTENDED_SIZE: usize = 0x8b0; pub(crate) const REGISTERS: usize = 0x20; pub(crate) const REGISTER_COUNT: usize = 22; /// G15G takes the same list without its last two registers (0xa599, 0xd411). @@ -64,12 +65,22 @@ impl Command { /// Validate before any output mutation. Encoding operates on ordinary host /// bytes; the caller copies them into the owned unpublished command buffer. pub(crate) fn encode(&self, out: &mut [u8]) -> Result<(), Error> { - if out.len() < SIZE { return Err(Error::Size); } + self.encode_for(super::m3_profile::Profile::Legacy, 0, 0, out) + } + + /// Firmware envelopes vary independently of the GPU's CDM stream format. + pub(crate) fn encode_for(&self, profile: super::m3_profile::Profile, + asid_generation: u8, event_slot: u8, out: &mut [u8]) -> Result<(), Error> { + let extended = profile == super::m3_profile::Profile::Extended; + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len() < size { return Err(Error::Size); } + if extended && (self.register_set != RegisterSet::G15G || event_slot >= 4 + || (self.context != 0 && asid_generation == 0)) { return Err(Error::Address); } if self.context >= 64 || self.cdm_last.get() < self.cdm.get() { return Err(Error::Address); } if self.preemption.size() < 0x14a0 || self.microsequence_size == 0 || self.microsequence_size > 4096 { return Err(Error::Size); } - self.gpu_alias.add((SIZE - 3) as u64)?; // inclusive final aligned dword + self.gpu_alias.add(((size - 1) & !3) as u64)?; // inclusive final aligned dword let registers = self.gpu_alias.add(REGISTERS as u64)?; let preemption = self.preemption.base(); let values = [ @@ -82,45 +93,143 @@ impl Command { (0x1a0b9, 0), (0x1a0c1, 0), (0x101d1, 0), (0xd479, 0), (0x1a0e9, 8), (0x107a1, 0xff0000), (0xa599, 0x13200400020), (0xd411, 0x200000001), ]; - let bytes = &mut out[..SIZE];bytes.fill(0); + let bytes = &mut out[..size];bytes.fill(0); word(bytes, 0, 3); // compute opcode - pointer(bytes, COUNTER, u64::from(self.counter)); - word(bytes, CONTEXT, self.context); - pointer(bytes, 0x14, self.notifier.get()); - let count = match self.register_set { + if extended { + word(bytes, 0xc, self.context); + pointer(bytes, 0x10, self.notifier.get()); + } else { + pointer(bytes, COUNTER, u64::from(self.counter)); + word(bytes, CONTEXT, self.context); + pointer(bytes, 0x14, self.notifier.get()); + } + let legacy_count = match self.register_set { RegisterSet::G15S => REGISTER_COUNT, RegisterSet::G15G => G15G_REGISTER_COUNT, }; - for (n, (register, value)) in values.into_iter().take(count).enumerate() { - word(bytes, REGISTERS+n*REGISTER_STRIDE, register); - pointer(bytes, REGISTERS+n*REGISTER_STRIDE+4, value); + let mut count = 0; + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + let limit = if extended { REGISTER_COUNT } else { legacy_count }; + for (register, value) in values.into_iter().take(limit) { + if extended && register == 0x12091 { continue; } + emit(register, value); + } + if extended { + emit(0x1a540, 1); + emit(0x014a9, 1); + emit(0x0a351, 1); } pointer(bytes, 0x720, registers.get()); bytes[0x728..0x72a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x72a..0x72c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); pointer(bytes, 0x760, self.microsequence.get()); word(bytes, 0x768, self.microsequence_size); - // ComputeInfo2 mirrors the preemption buffer, inclusive CDM end and - // qualified control word from the register list. - pointer(bytes, 0x76c+0x28, preemption); - pointer(bytes, 0x76c+0x30, self.cdm_last.get()); - pointer(bytes, 0x76c+0x58, QUALIFIED_COMPUTE_CONTROL); - word(bytes, 0x7cc+0x14, u32::MAX); // EncoderParams.unk_mask - pointer(bytes, META, self.stamp.get()); - pointer(bytes, META+8, self.fw_stamp.get()); - word(bytes, STAMP_VALUE, self.stamp_value); - word(bytes, META+0x14, 2); // qualified compute event slot - word(bytes, FLUSH_STAMPS, u32::from(self.flush_stamps)); - word(bytes, META+0x20, 0xc10000); // producer UUID - word(bytes, QUEUE_COUNT, self.counter); - pointer(bytes, TAIL+8, self.timestamps[0].get()); - pointer(bytes, TAIL+16, self.timestamps[1].get()); - pointer(bytes, TAIL+0x2e, self.pool.get()); - bytes[TAIL+0x36] = 1; // SharedCommandTail.unk_846 - pointer(bytes, TAIL+0x37, 0xad0000); - pointer(bytes, TAIL+0x3f, 0x1600000); - pointer(bytes, TAIL+0x47, self.tail_scratch.get()); - bytes[TAIL+0x4f] = 1; // SharedCommandTail.unk_85f + let info = if extended { 0x770 } else { 0x76c }; + pointer(bytes, info + 0x28, preemption); + pointer(bytes, info + 0x30, self.cdm_last.get()); + pointer(bytes, info + 0x58, QUALIFIED_COMPUTE_CONTROL); + word(bytes, if extended { 0x7e8 } else { 0x7e0 }, u32::MAX); + let meta = if extended { 0x800 } else { META }; + pointer(bytes, meta, self.stamp.get()); + pointer(bytes, meta + 8, self.fw_stamp.get()); + word(bytes, meta + 0x10, self.stamp_value); + word(bytes, meta + 0x14, 2); + if extended { word(bytes, meta + 0x18, event_slot.into()); } + word(bytes, meta + 0x1c, u32::from(self.flush_stamps)); + word(bytes, meta + 0x20, 0xc10000); + word(bytes, meta + 0x24, self.counter); + let tail = if extended { 0x82c } else { TAIL }; + pointer(bytes, tail + 8, self.timestamps[0].get()); + pointer(bytes, tail + 16, self.timestamps[1].get()); + pointer(bytes, tail + if extended { 0x3e } else { 0x2e }, self.pool.get()); + bytes[tail + if extended { 0x46 } else { 0x36 }] = 1; + pointer(bytes, tail + if extended { 0x47 } else { 0x37 }, + if extended { 0x1588000 } else { 0xad0000 }); + pointer(bytes, tail + if extended { 0x4f } else { 0x3f }, + if extended { 0xef8000 } else { 0x1600000 }); + pointer(bytes, tail + if extended { 0x57 } else { 0x47 }, self.tail_scratch.get()); + if extended { bytes[0x88b] = asid_generation; } + else { bytes[TAIL + 0x4f] = 1; } Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use super::super::m3_profile::Profile; + + fn fixture() -> Command { + let firmware = |n: u64| FirmwareVa::new(0xffff_fc21_8000_0000 + n * 0x4000).unwrap(); + Command { + context: 3, counter: 7, notifier: firmware(0), + preemption: GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + cdm: GpuVa::new(0x6000_0000).unwrap(), + cdm_last: GpuVa::new(0x6000_0ffc).unwrap(), + gpu_alias: GpuVa::new(0x7000_0000).unwrap(), + microsequence: firmware(1), microsequence_size: 0x300, + stamp: firmware(2), fw_stamp: firmware(3), stamp_value: 0x700, + timestamps: [firmware(4), firmware(5)], pool: firmware(6), + tail_scratch: firmware(7), flush_stamps: true, + register_set: RegisterSet::G15G, + } + } + fn read32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) + } + fn read64(b: &[u8], o: usize) -> u64 { + u64::from_le_bytes(b[o..o + 8].try_into().unwrap()) + } + + #[test] + fn extended_envelope_preserves_stream_and_native_visibility_boundary() { + let c = fixture(); + let mut b = [0xa5; EXTENDED_SIZE + 16]; + c.encode_for(Profile::Extended, 9, 2, &mut b).unwrap(); + assert_eq!(read32(&b, 0xc), c.context); + assert_eq!(read64(&b, 0x10), c.notifier.get()); + assert_eq!(read32(&b, 0x728) & 0xffff, 24); + let count = (read32(&b, 0x728) & 0xffff) as usize; + let cdm = (0..count).find(|n| read32(&b, 0x20 + n * 12) == 0x1a420).unwrap(); + assert_eq!(read64(&b, 0x24 + cdm * 12), c.cdm.get()); + assert_eq!(read64(&b, 0x7a0), c.cdm_last.get()); + assert_eq!(read64(&b, 0x800), c.stamp.get()); + assert_eq!(read64(&b, 0x808), c.fw_stamp.get()); + assert_eq!(read32(&b, 0x818), 2); + assert_eq!(read32(&b, 0x81c), 1); + assert_eq!(read64(&b, 0x834), c.timestamps[0].get()); + assert_eq!(read64(&b, 0x86a), c.pool.get()); + assert_eq!(read64(&b, 0x883), c.tail_scratch.get()); + assert_eq!(b[0x88b], 9); + assert_eq!(&b[EXTENDED_SIZE..], &[0xa5; 16]); + } + + #[test] + fn extended_validation_never_mutates_a_rejected_envelope() { + for (generation, slot) in [(0, 0), (1, 4), (1, 255)] { + let mut b = [0xa5; EXTENDED_SIZE]; + assert!(fixture().encode_for(Profile::Extended, generation, slot, &mut b).is_err()); + assert_eq!(b, [0xa5; EXTENDED_SIZE]); + } + let mut b = [0xa5; EXTENDED_SIZE - 1]; + assert!(fixture().encode_for(Profile::Extended, 1, 0, &mut b).is_err()); + assert_eq!(b, [0xa5; EXTENDED_SIZE - 1]); + } + + #[test] + fn legacy_wrapper_keeps_legacy_geometry_and_register_order() { + let c = fixture(); + let mut b = [0xa5; EXTENDED_SIZE]; + c.encode(&mut b).unwrap(); + assert_eq!(read32(&b, CONTEXT), c.context); + assert_eq!(read64(&b, 0x14), c.notifier.get()); + assert_eq!(read32(&b, 0x728) & 0xffff, 20); + assert_eq!(read32(&b, REGISTERS + 8 * REGISTER_STRIDE), 0x12091); + assert_eq!(read64(&b, META), c.stamp.get()); + assert_eq!(&b[SIZE..], &[0xa5; EXTENDED_SIZE - SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index c4a44c4b4f3b0..67f4f963637f2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -49,6 +49,8 @@ impl Records { pub(crate) struct Config { objects: KVec, + profile: super::m3_profile::Profile, + control_generation: u32, _iomaps: KVec, pub(crate) completed_events: u64, pstates: crate::m3::m3_adt_config::PstateWatch, @@ -140,7 +142,8 @@ impl Config { crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; - Ok(Self { objects, _iomaps: iomaps, completed_events: 0, + Ok(Self { objects, profile: firmware.profile(), control_generation: 0, + _iomaps: iomaps, completed_events: 0, pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, request_check_at:None,request_writes:0,request_observed:0 }) @@ -261,6 +264,9 @@ impl Config { Ok(true) } pub(crate) fn ready(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[RUNTIME_FLAGS].read_u32(0x14)? == 1); + } // NewRegionB+4 is an input initialized to 1, not an acknowledgement. // These are firmware-written transitions present in compute-15's // pre-submit snapshot; both start at zero in the exported schema. @@ -343,10 +349,38 @@ impl Config { agx_memory::publish(); Ok(next) } + /// Publish an extended-format synchronous control. The caller rings the + /// doorbell after this method and waits for both retirement cursors. + pub(crate) fn control_extended(&mut self, action: super::m3_channel_format::Action) -> Result { + use super::m3_channel_format as format; + if self.profile != super::m3_profile::Profile::Extended { return Err(EINVAL); } + let state = storage::DEVICE_CONTROL * 2; + let cursors = [self.objects[state].read_u32(0)?, + self.objects[state].read_u32(0x10)?, self.objects[state].read_u32(0x20)?]; + let cursor = format::idle_cursor(cursors).map_err(|_| EBUSY)?; + let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; + let generation = action.generation(self.control_generation, consumed).map_err(|_| EOVERFLOW)?; + let mut record = [0; format::CONTROL_BYTES]; + action.encode(&mut record).map_err(|_| EINVAL)?; + self.objects[state + 1].write(cursor as usize * format::CONTROL_BYTES, &record)?; + if generation != 0 { + self.objects[GLOBALS].write(0xe41, &generation.to_le_bytes())?; + } + agx_memory::publish(); + let next = (cursor + 1) % format::SLOTS; + self.objects[state].u32(0x20, next)?; + agx_memory::publish(); + self.control_generation = generation; + Ok(next) + } + pub(crate) fn control_done(&mut self, next: u32) -> Result { Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) } pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + return self.drain_extended(dev); + } // Event, trace, stats and six independent firmware log rings. for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), @@ -392,4 +426,147 @@ impl Config { } Ok(()) } + fn drain_extended(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + use super::m3_channel_format as format; + // Snapshot each producer once. New arrivals remain for the next drain; + // a corrupt cursor is rejected before reading its ring allocation. + for (channel, capacity, threads) in [(storage::FIRMWARE_LOG, 256, 9), + (storage::EVENT, 256, 1), (storage::TRACE, 512, 1), (storage::STATS, 256, 1)] { + let state = channel * 2; + for thread in 0..threads { + let at = thread * 0x30; + let mut read = self.objects[state].read_u32(at)?; + let write = self.objects[state].read_u32(at + 0x20)?; + if read >= capacity || write >= capacity { return Err(EIO); } + let before = read; + while read != write { + let offset = (thread * capacity as usize + read as usize) * format::RECEIVE_BYTES; + let mut record = [0; format::RECEIVE_BYTES]; + self.objects[state + 1].read(offset, &mut record)?; + let kind = u32::from_le_bytes(record[..4].try_into().map_err(|_| EIO)?); + if channel == storage::FIRMWARE_LOG { + let slot = u32::from_le_bytes(record[8..12].try_into().map_err(|_| EIO)?); + let payload = format::payload_offset(thread, slot).map_err(|_| EIO)?; + if crate::debug::debug_enabled(crate::debug::DebugFlags::KTraceCh) { + let mut text = [0; 200]; + self.objects[FWLOG_PAYLOAD].read(payload + 0x10, &mut text)?; + let len = text.iter().position(|value| *value == 0).unwrap_or(text.len()); + dev_info!(dev.as_ref(), "M3 firmware log[{}]: {}\n", thread, + core::str::from_utf8(&text[..len]).unwrap_or("")); + } + } else if channel == storage::EVENT { + if kind != 1 { + dev_err!(dev.as_ref(), "M3 firmware event {}: {:02x?}\n", kind, record); + return Err(EIO); + } + let flags = u32::from_le_bytes(record[4..8].try_into().map_err(|_| EIO)?); + self.completed_events = self.completed_events.saturating_add((flags & 7).count_ones().into()); + } else if (channel == storage::TRACE && kind != 5) + || (channel == storage::STATS && kind == 0x16) { + dev_err!(dev.as_ref(), "M3 firmware channel {} record {}: {:02x?}\n", channel, kind, record); + return Err(EIO); + } + read = (read + 1) % capacity; + } + if read != before { self.objects[state].u32(at, read)?; } + } + } + agx_memory::publish(); + Ok(()) + } + +} + +/// All extended initialization owners, retained as one construction transaction. +/// Nothing is published to RTKit by this constructor. A failed allocation drops +/// its mappings before backing storage and leaves no root visible to firmware. +pub(crate) struct ExtendedStorage { + pub(crate) objects: KVec, + pub(crate) iomaps: KVec, + pub(crate) descriptors: KVec, + pub(crate) maximum_khz: u32, +} + +impl ExtendedStorage { + pub(crate) fn new( + dev: &driver::AsahiDrmDevice, + uat: &mmu::Uat, + pwr: &super::hw::PwrConfig, + sram_uv: &[u32], + bases: super::m3_mmio_26::RegisterBases, + ) -> Result { + use super::{m3_init_26 as plan, m3_mmio_26 as io}; + let mut objects = KVec::new(); + for index in 0..plan::COUNT { + let entry = plan::allocation(index).map_err(|_| EINVAL)?; + let alias = match index { + plan::GLOBAL_POOL => Some((uat.kernel_lower_vm(), 0x70_0000_0000)), + plan::GLOBAL_PARAMETERS => Some((uat.kernel_lower_vm(), 0x70_0000_8000)), + _ => None, + }; + let object = if plan::cached(index) { + Buffer::at_prot(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size, + prot::PROT_FW_PRIV_RW, prot::PROT_GPU_SHARED_RW)? + } else { + Buffer::at(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size, entry.gpu_shared)? + }; + objects.push(object, GFP_KERNEL)?; + } + let mut descriptors = KVec::new(); + descriptors.resize(io::SLOTS * io::RECORD_BYTES, 0, GFP_KERNEL)?; + let mut iomaps = KVec::new(); + for slot in 0..io::SLOTS { + let mut region = None; + if let Some(area) = io::area(slot, bases).map_err(|_| EINVAL)? { + let address = 0xffff_fc23_0000_0000 + slot as u64 * 0x10_0000; + let access = if area.writable() { prot::PROT_FW_MMIO_RW } else { prot::PROT_FW_MMIO_RO }; + for (physical, bytes, offset) in area.banks() { + if bytes == 0 { continue; } + let mapping = uat.kernel_vm().map_io_m3(address + offset as u64, + physical.try_into()?, bytes, access)?; + iomaps.push(mapping, GFP_KERNEL)?; + } + region = Some(init::Region::new(address, area.mapped_bytes()).map_err(|_| EINVAL)?); + } + io::encode(slot, bases, region, + &mut descriptors[slot * io::RECORD_BYTES..(slot + 1) * io::RECORD_BYTES]) + .map_err(|_| EINVAL)?; + } + // Runtime+0x18 references Fender SRAM, rather than anonymous DMA RAM. + // Retain its normal shared mapping independently of its MMIO view. + let scratch_physical = bases.gpu.checked_add(0xd6_0000).ok_or(EOVERFLOW)?; + let scratch = uat.kernel_vm().map_io_m3(0xffff_fc22_0000_0000, + scratch_physical.try_into()?, 0x8000, prot::PROT_FW_SHARED_RW)?; + let scratch_region = init::Region::new(scratch.iova(), 0x8000).map_err(|_| EINVAL)?; + iomaps.push(scratch, GFP_KERNEL)?; + let mut regions = [init::Region::new(0xffff_fc21_0000_0000, 1).map_err(|_| EINVAL)?; plan::COUNT]; + for (index, object) in objects.iter().enumerate() { + regions[index] = init::Region::new(object.va(), object.size()).map_err(|_| EINVAL)?; + } + let graph = plan::PointerGraph { owners: ®ions, scratch: scratch_region, + pool_alias: objects[plan::GLOBAL_POOL].gpu_va()?, + parameter_alias: objects[plan::GLOBAL_PARAMETERS].gpu_va()? }; + let mut root = [0; 0xc8]; + let mut runtime = [0; plan::RUNTIME_SIZE]; + let mut control = KVec::new(); + control.resize(plan::allocation(CONTROL_REGION).map_err(|_| EINVAL)?.size, 0, GFP_KERNEL)?; + graph.root(&mut root).map_err(|_| EINVAL)?; + graph.runtime(&mut runtime).map_err(|_| EINVAL)?; + graph.firmware_control(&mut control).map_err(|_| EINVAL)?; + let images = super::m3_power_26::PowerImages::new(pwr, sram_uv, + objects[plan::SENSOR_VALUE].va(), objects[plan::RETENTION_STATE].va(), + agx_memory::TIMESTAMP_RANGE.start, &descriptors)?; + objects[storage::ROOT].write(0, &root)?; + objects[RUNTIME_POINTERS].write(0, &runtime)?; + objects[CONTROL_REGION].write(0, &control)?; + objects[HARDWARE_DATA].write(0, &images.hardware)?; + objects[plan::PERFORMANCE_CONTROL].write(0, &images.performance)?; + objects[GLOBALS_POWER].write(0, &images.power)?; + objects[GLOBALS].write(0, &images.globals)?; + // Fragment scheduler entries start unassigned; these are input sentinels. + objects[UNKNOWN_C1].u32(0xc18, u32::MAX)?; + objects[UNKNOWN_C1].u32(0xc30, u32::MAX)?; + agx_memory::publish(); + Ok(Self { objects, iomaps, descriptors, maximum_khz: images.maximum_khz }) + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index c9aa45cc02864..6bc87a33aaa05 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -255,6 +255,51 @@ impl Firmware { } } +/// Only the admitted extended image has this host-owned boot descriptor slot. +fn loader_span(profile: super::m3_profile::Profile, data_bytes: u64, input_bytes: usize) + -> Option> { + if profile != super::m3_profile::Profile::Extended || input_bytes != 53 * 40 + || data_bytes != T8122_LAYOUT_26.data_size { return None; } + let start: usize = 0x12_28e0; + let end = start.checked_add(input_bytes)?; + if end as u64 > data_bytes { return None; } + Some(start..end) +} + +#[cfg(not(test))] +impl Firmware { + /// Call after RTKit application startup and before exposing the InitData root. + /// The protocol grants the host this slot only during that boot interval. + pub(crate) fn publish_io_table( + &self, + pdev: &kernel::platform::Device>, + descriptors: &[u8], + ) -> kernel::error::Result { + use kernel::{c_str, io::mem::{Mem, MemFlag}, prelude::*}; + let data = self.resources.regions[5]; + let span = loader_span(self.profile(), data.size, descriptors.len()).ok_or(EINVAL)?; + let node = pdev.as_ref().of_node().ok_or(ENODEV)?; + let resource = agx_resources::reserved_resource(&node, c_str!("fw-data"))?; + if resource.start() != data.base || resource.size() != data.size { return Err(EINVAL); } + // SAFETY: This exact admitted no-map reservation remains firmware-owned. + // Only its bounded host boot slot is accessed, before root publication. + let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?; + if span.end > mapping.size() { return Err(EINVAL); } + // Validate the whole slot before changing any byte. Unexpected prior + // contents indicate a protocol/ownership mismatch, not a patchable image. + for offset in span.clone() { + // SAFETY: span is bounded by the retained mapping above. + if unsafe { mapping.ptr().add(offset).read_volatile() } != 0 { return Err(EBUSY); } + } + for (offset, byte) in span.zip(descriptors.iter().copied()) { + // SAFETY: same bounded host slot, exclusive under the boot protocol. + unsafe { mapping.ptr().add(offset).write_volatile(byte) }; + } + crate::m3::agx_memory::publish(); + Ok(()) + } +} + #[cfg(not(test))] pub(crate) fn identify_loaded( pdev: &kernel::platform::Device>, @@ -527,4 +572,18 @@ mod tests { assert!(!newer.accepts_runtime(KNOWN_IMAGES_T8122[0].uuid, &[0; 32])); } + #[test] + fn loader_slot_is_exact_bounded_and_profile_specific() { + use super::super::m3_profile::Profile; + let length = 53 * 40; + assert_eq!(loader_span(Profile::Extended, 0x12c000, length), Some(0x1228e0..0x123128)); + for size in [0, 0x123127, 0x12bfff, 0x12c001, u64::MAX] { + assert!(loader_span(Profile::Extended, size, length).is_none()); + } + for bytes in [0, length - 1, length + 1, usize::MAX] { + assert!(loader_span(Profile::Extended, 0x12c000, bytes).is_none()); + } + assert!(loader_span(Profile::Legacy, 0x12c000, length).is_none()); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_init_26.rs b/drivers/gpu/drm/asahi/m3/m3_init_26.rs index f08f16200b070..a86432b2bbe2b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_init_26.rs +++ b/drivers/gpu/drm/asahi/m3/m3_init_26.rs @@ -62,7 +62,7 @@ pub(crate) fn allocation(index: usize) -> Result { } pub(crate) fn cached(index: usize) -> bool { - matches!(index, 38 | 39 | 42 | 43 | 44 | PERFORMANCE_CONTROL) + matches!(index, 39 | PERFORMANCE_CONTROL) } pub(crate) struct PointerGraph<'a> { @@ -103,7 +103,10 @@ impl PointerGraph<'_> { } let scratch = self.scratch.at(0, 0x8000)?; let scratch_end = scratch.checked_add(0x8000).ok_or(Error::Address)?; - if scratch & 0x3fff != 0 || scratch_end > 0xffff_fc2f_fc00_0000 { + if scratch & 0x3fff != 0 + || scratch < 0xffff_fc20_0000_0000 + || scratch_end > 0xffff_fc2f_fc00_0000 + { return Err(Error::Address); } for (index, address) in addresses.iter().enumerate() { @@ -272,4 +275,19 @@ mod tests { assert!(graph(&owners).runtime(&mut output).is_err()); assert_eq!(output, [0xa5; RUNTIME_SIZE]); } + #[test] + fn only_atomic_controller_owners_are_firmware_cacheable() { + let selected: Vec<_> = (0..COUNT).filter(|index| cached(*index)).collect(); + assert_eq!(selected, vec![39, PERFORMANCE_CONTROL]); + } + + #[test] + fn scratch_requires_its_firmware_host_window() { + let owners = owners(0); + let mut value = graph(&owners); + value.scratch = Region::new(0xffff_fc10_0000_0000, 0x8000).unwrap(); + let mut output = [0xa5; RUNTIME_SIZE]; + assert_eq!(value.runtime(&mut output), Err(Error::Address)); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs index e4008f239c4a0..a17bc0ee399c3 100644 --- a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs +++ b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs @@ -18,6 +18,10 @@ pub(crate) struct Area { } impl Area { + pub(crate) fn writable(self) -> bool { + self.writable + } + pub(crate) fn page_offset(self) -> usize { (self.physical & 0x3fff) as usize } @@ -78,6 +82,16 @@ pub(crate) fn area(slot: usize, bases: RegisterBases) -> Result, Er if physical == 0 || physical.checked_add(bytes as u64).ok_or(Error::Address)? > 1 << 42 { return Err(Error::Address); } + if split + && (physical & 0x3fff != 0 + || physical + .checked_add(0x200_0000) + .and_then(|v| v.checked_add((bytes / 2) as u64)) + .filter(|v| *v <= 1 << 42) + .is_none()) + { + return Err(Error::Address); + } Ok(Some(Area { physical, bytes, @@ -143,8 +157,7 @@ pub(crate) fn encode( mod tests { use super::*; - // Register identities already present in the public T8122 configuration; - // memory-controller bank geometry is shared with the public 25G83 encoder. + // Resource owners exercise subpage mappings and split memory-controller banks. const BASES: RegisterBases = RegisterBases { gpu: 0x2_9000_0000, interrupt: 0x2_d101_4000, @@ -262,4 +275,20 @@ mod tests { assert_eq!(output, [0xa5; RECORD_BYTES]); } } + #[test] + fn split_banks_require_page_alignment_and_both_fit_the_bus() { + for memory in [BASES.memory + 4, (1 << 42) - 0x100_0000] { + let mut bases = BASES; + bases.memory = memory; + let mut output = [0xa5; RECORD_BYTES]; + assert!(encode( + 3, + bases, + Some(Region::new(0xffff_fc21_0000_0000, 0x30000).unwrap()), + &mut output + ) + .is_err()); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_power_26.rs b/drivers/gpu/drm/asahi/m3/m3_power_26.rs new file mode 100644 index 0000000000000..2619c24b4eb6a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_power_26.rs @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Build the extended M3 controller images from admitted device-tree inputs. +//! These are constructor inputs. Firmware owns subsequent controller outputs. + +use super::{hw, initdata::PStateTables}; +use crate::{f32, float::F32}; +use kernel::prelude::*; + +struct Image(KVec); +impl Image { + fn new(size: usize) -> Result { + let mut bytes = KVec::new(); + bytes.resize(size, 0, GFP_KERNEL)?; + Ok(Self(bytes)) + } + fn word(&mut self, offset: usize, value: u32) -> Result { + self.0 + .get_mut(offset..offset.checked_add(4).ok_or(EOVERFLOW)?) + .ok_or(ERANGE)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) + } + fn pointer(&mut self, offset: usize, value: u64) -> Result { + self.0 + .get_mut(offset..offset.checked_add(8).ok_or(EOVERFLOW)?) + .ok_or(ERANGE)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) + } + fn float(&mut self, offset: usize, value: F32) -> Result { + let bits = value.to_bits(); + if bits & 0x7f80_0000 == 0x7f80_0000 { + return Err(EINVAL); + } + self.word(offset, bits) + } + fn reciprocal(&mut self, negative: usize, positive: usize, periods: u32) -> Result { + if periods == 0 { + return Err(EINVAL); + } + let alpha = f32!(1.0) / F32::from(periods); + self.float(negative, f32!(1.0) - alpha)?; + self.float(positive, alpha) + } +} + +pub(crate) struct PowerImages { + pub(crate) hardware: KVec, + pub(crate) performance: KVec, + pub(crate) power: KVec, + pub(crate) globals: KVec, + pub(crate) maximum_khz: u32, +} + +impl PowerImages { + /// `sram_uv` follows the same OPP iteration order as PwrConfig::perf_states. + /// No reference-bin voltage or experimental OPP-power estimate is substituted. + pub(crate) fn new( + pwr: &hw::PwrConfig, + sram_uv: &[u32], + sensor: u64, + retention: u64, + timestamp_base: u64, + io_records: &[u8], + ) -> Result { + let table = PStateTables::new(pwr, true, None, false)?; + let second = table.secondary.ok_or(EINVAL)?; + if table.len != 9 + || table.max != 8 + || sram_uv.len() != pwr.perf_states.len() + || pwr.perf_base_pstate == 0 + || pwr.perf_base_pstate > 8 + || !(1..=1000).contains(&pwr.power_sample_period) + || pwr.pwr_min_duty_cycle > 100 + || pwr.avg_power_min_duty_cycle > 100 + || io_records.len() != super::m3_mmio_26::SLOTS * super::m3_mmio_26::RECORD_BYTES + || [sensor, retention, timestamp_base] + .iter() + .any(|value| *value < 0xffff_fc00_0000_0000 || value & 7 != 0) + { + return Err(EINVAL); + } + let top = &pwr.perf_states[table.primary[8] as usize]; + let base = &pwr.perf_states[table.primary[pwr.perf_base_pstate as usize] as usize]; + if top.freq_hz != 1_338_000_000 || base.freq_hz >= top.freq_hz { + return Err(EINVAL); + } + let period = pwr.power_sample_period; + let ppm_periods = pwr.ppm_filter_time_constant_ms / period; + let average_periods = pwr.avg_power_filter_tc_ms / period; + if [ + ppm_periods, + average_periods, + pwr.pwr_filter_time_constant, + pwr.perf_filter_time_constant, + pwr.perf_filter_time_constant2, + pwr.avg_power_target_filter_tc, + pwr.se_filter_time_constant, + pwr.se_filter_time_constant_1, + ] + .contains(&0) + { + return Err(EINVAL); + } + let clocks = |count: u32| { + count + .checked_mul(period) + .and_then(|v| v.checked_mul(24_000)) + .ok_or(EOVERFLOW) + }; + let dt = F32::from(period) / f32!(1000.0); + let ceiling = 800; + let base_units = pwr.perf_base_pstate * 100; + // The profile controller budget is independent of the loader's experimental + // OPP power model. It is not a demand floor or a replacement voltage table. + let budget = 22_044u32; + let mut main = Image::new(0x2658)?; + let mut performance = Image::new(0x6578)?; + let mut power = Image::new(0x670)?; + let mut globals = Image::new(0xe48)?; + + main.pointer(0x28, timestamp_base)?; + main.0[0x640..0xe88].copy_from_slice(io_records); + main.pointer(0xe88, retention)?; + main.word(0xe90, 0x8122)?; + main.word(0xe94, 2)?; + main.word(0xe9c, 4)?; + main.word(0xea0, 1)?; + main.word(0xea4, 1)?; + main.word(0xeb8, 1)?; + main.word(0xec0, 1)?; + main.word(0xec8, 1)?; + main.word(0xed0, 24_000)?; + main.word(0xed4, 1)?; + main.word(0xed8, period)?; + main.word(0xee0, 1)?; + main.word(0xee4, 1)?; + main.word(0xee8, 1)?; + main.word(0xf04, 31)?; + main.pointer(0xf24, 4)?; + main.word(0xf34, 1)?; + // Firmware completion is polled through its normal bounded status path. + main.word(0xf38, 0)?; + main.word(0xf4c, 49)?; + main.word(0xf6c, 1)?; + main.word(0xf88, 10)?; + main.word(0xf8c, 3)?; + main.word(0xfac, 1)?; + main.word(0xfb8, 22)?; + main.word(0xfbc, 5)?; + main.word(0xfc0, 10)?; + main.word(0xfc4, 8)?; + main.word(0x1cd8, 8)?; + for state in 0..9 { + let primary = table.primary[state] as usize; + let secondary = second[state] as usize; + let item = &pwr.perf_states[primary]; + let auxiliary = &pwr.perf_states[secondary]; + if item.volt_mv.len() != 1 + || auxiliary.volt_mv.len() != 1 + || !(100..=1100).contains(&item.volt_mv[0]) + || !(100..=1100).contains(&auxiliary.volt_mv[0]) + || !(100_000..=1_100_000).contains(&sram_uv[primary]) + || !(100_000..=1_100_000).contains(&sram_uv[secondary]) + || sram_uv[primary].div_ceil(1000) < item.volt_mv[0] + || sram_uv[secondary].div_ceil(1000) < auxiliary.volt_mv[0] + { + return Err(EINVAL); + } + main.word(0xfc8 + state * 4, item.freq_hz / 1_000_000)?; + main.word(0x1808 + state * 4, auxiliary.freq_hz / 1_000_000)?; + main.float(0x1848 + state * 4, f32!(1.02))?; + for lane in 0..16 { + main.word(0x1008 + state * 64 + lane * 4, item.volt_mv[0])?; + main.word( + 0x1408 + state * 64 + lane * 4, + sram_uv[primary].div_ceil(1000), + )?; + } + main.word(0x1cdc + state * 4, auxiliary.freq_hz / 1_000_000)?; + if state != 0 { + main.word(0x1d1c + state * 64, auxiliary.volt_mv[0])?; + } + main.word(0x211c + state * 64, sram_uv[secondary].div_ceil(1000))?; + if state != 0 { + // Profile indices for the admitted eight-state ladder. + let core_profile = if state <= 3 { state } else { 2 * state - 3 }; + let auxiliary_profile = if state <= 4 { state } else { 2 * state - 4 }; + main.word(0x19c8 + state * 4, core_profile as u32)?; + main.word(0x1a08 + state * 4, auxiliary_profile as u32)?; + let boost = item.freq_hz.saturating_sub(base.freq_hz) / 1_000_000; + main.word( + 0x1908 + state * 4, + 100 * boost / ((top.freq_hz - base.freq_hz) / 1_000_000), + )?; + } + } + // The native relative-power calibration of each running firmware state. + // Encode only these semantic percentages, not an object-image overlay. + for (state, percent) in [15, 25, 36, 46, 58, 75, 90, 100].into_iter().enumerate() { + main.word(0x18c8 + (state + 1) * 4, percent)?; + } + main.word(0x2544, 4)?; + main.word(0x2548, 6)?; + main.word(0x254c, 3)?; + main.word(0x2550, 7)?; + main.word(0x2554, 7)?; + main.word(0x2568, 10)?; + main.word(0x2570, 1)?; + main.word(0x258c, 0x160000)?; + main.word(0x2594, 1)?; + main.word(0x2598, 1)?; + main.word(0x259c, 1)?; + main.word(0x25a4, 1)?; + main.word(0x25a8, 1)?; + main.word(0x25ac, 1)?; + main.0[0x25b4..0x25e4].fill(0xff); + main.word(0x2604, 1)?; + main.word(0x2634, 1)?; + + globals.word(0x24, 3000)?; + globals.word(0x78, 1)?; + globals.word(0x84, budget)?; + for offset in [0x88, 0x8c, 0x98] { + globals.word(offset, ceiling)?; + } + globals.word(0x998, 40)?; + globals.word(0x99c, 10)?; + globals.word(0x9a0, 250)?; + globals.word(0x9b8, 2)?; + globals.word(0x9bc, 40)?; + globals.word(0x9c0, 5)?; + globals.word(0x9c8, 40)?; + globals.word(0x9cc, 50)?; + globals.word(0xe28, 3)?; + + power.word(0x10, average_periods)?; + power.float(0x14, pwr.avg_power_ki_only * dt)?; + power.float(0x18, pwr.avg_power_kp)?; + power.word(0x1c, pwr.avg_power_min_duty_cycle)?; + power.word(0x20, pwr.avg_power_target_filter_tc)?; + power.word(0xc8, 11_000)?; + power.word(0xcc, 8_000)?; + power.float(0xd8, pwr.fast_die0_proportional_gain)?; + power.float(0xdc, pwr.fast_die0_integral_gain * dt)?; + power.word(0x12c, budget)?; + power.float(0x130, pwr.ppm_kp)?; + power.float(0x134, pwr.ppm_ki * dt)?; + power.word(0x14c, 1)?; + + performance.word(4, pwr.pwr_sample_period_aic_clks)?; + performance.word(8, pwr.pwr_sample_period_aic_clks)?; + performance.word(0x10, 4)?; + performance.float(0x14, f32!(1.0))?; + performance.word(0x2c, 1)?; + performance.word(0x30, 1)?; + // Leave the observed hardware state at zero until firmware writes it. + for offset in [0x40, 0x80, 0xb78, 0xb80, 0xbc0] { + performance.word(offset, base_units)?; + } + for offset in [0x44, 0x6c, 0x70, 0x84] { + performance.word(offset, 1)?; + } + for offset in [ + 0x48, 0x88, 0x9f4, 0x9f8, 0xa38, 0xaac, 0xaf0, 0xb7c, 0x221c, 0x22b4, 0x22b8, 0x22f8, + 0x2870, 0x29f8, 0x29fc, 0x2a3c, 0x2b88, 0x2c44, 0x2c88, + ] { + performance.word(offset, ceiling)?; + } + performance.word(0x50, 100)?; + performance.word(0x60, 4)?; + performance.float(0x64, f32!(1.0))?; + performance.word(0x90, 100)?; + performance.word(0x9b8, 625)?; + performance.reciprocal(0x9c4, 0x9cc, pwr.pwr_filter_time_constant)?; + performance.float(0x9d4, pwr.pwr_integral_gain)?; + performance.float(0x9dc, F32::from(pwr.pwr_integral_min_clamp))?; + performance.float(0x9e0, F32::from(budget))?; + performance.float(0x9e4, pwr.pwr_proportional_gain)?; + performance.float(0x9ec, -F32::from(ceiling) / F32::from(budget))?; + for offset in [0x9f0, 0xaa8, 0x22b0] { + performance.word(offset, pwr.pwr_min_duty_cycle)?; + } + for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { + performance.word(offset, budget)?; + } + performance.word(0xa70, ppm_periods.checked_mul(4).ok_or(EOVERFLOW)?)?; + performance.reciprocal(0xa7c, 0xa84, ppm_periods)?; + performance.float(0xa8c, pwr.ppm_ki * dt)?; + performance.float(0xa98, f32!(65536.0))?; + performance.float(0xa9c, pwr.ppm_kp)?; + performance.word(0xac8, pwr.ppm_filter_time_constant_ms)?; + performance.word(0xad0, clocks(ppm_periods)?)?; + performance.word(0xb28, pwr.perf_tgt_utilization)?; + performance.word(0xb30, pwr.perf_boost_min_util)?; + performance.word(0xb34, pwr.perf_boost_ce_step)?; + performance.word(0xb38, pwr.perf_reset_iters)?; + performance.word(0xb40, 6)?; + performance.word(0xb44, 1)?; + performance.word(0xb48, pwr.perf_filter_drop_threshold)?; + performance.reciprocal(0xb4c, 0xb54, pwr.perf_filter_time_constant)?; + performance.reciprocal(0xb50, 0xb58, pwr.perf_filter_time_constant2)?; + performance.float(0xb5c, pwr.perf_integral_gain)?; + performance.float(0xb60, pwr.perf_integral_gain2)?; + performance.float(0xb64, F32::from(pwr.perf_integral_min_clamp))?; + performance.float(0xb68, f32!(95.0))?; + performance.float(0xb6c, pwr.perf_proportional_gain)?; + performance.float(0xb70, pwr.perf_proportional_gain2)?; + performance.float(0xb74, F32::from(8 - pwr.perf_base_pstate) / f32!(0.95))?; + performance.word(0xb8c, pwr.perf_tgt_utilization)?; + performance.float(0x21f0, f32!(65536.0))?; + performance.float(0x21f4, F32::from(pwr.pwr_min_duty_cycle))?; + performance.float(0x21f8, F32::from(ceiling))?; + performance.word(0x21fc, 1338)?; + performance.float(0x2218, f32!(100.0))?; + performance.float(0x2224, f32!(0.8))?; + performance.float(0x2228, f32!(0.2))?; + performance.word(0x2268, 0x4248)?; + performance.word(0x2270, 8_000)?; + performance.word(0x2278, 4)?; + performance.float(0x228c, f32!(1.0))?; + performance.float(0x2294, pwr.fast_die0_integral_gain * dt)?; + performance.float(0x22a0, f32!(65536.0))?; + performance.float(0x22a4, pwr.fast_die0_proportional_gain)?; + performance.word(0x22c4, 11_000)?; + performance.word(0x28c0, 1)?; + performance.reciprocal(0x28dc, 0x28e0, pwr.avg_power_target_filter_tc)?; + performance.word( + 0x28e4, + pwr.avg_power_target_filter_tc + .checked_mul(4) + .ok_or(EOVERFLOW)?, + )?; + performance.word( + 0x28e8, + pwr.avg_power_target_filter_tc + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word(0x28ec, clocks(pwr.avg_power_target_filter_tc)?)?; + performance.word(0x29bc, average_periods.checked_mul(4).ok_or(EOVERFLOW)?)?; + performance.reciprocal(0x29c8, 0x29d0, average_periods)?; + performance.float(0x29d8, pwr.avg_power_ki_only * dt)?; + performance.float(0x29e4, f32!(65536.0))?; + performance.float(0x29e8, pwr.avg_power_kp)?; + performance.word(0x29f4, pwr.avg_power_min_duty_cycle)?; + performance.float(0x2a04, F32::from(budget))?; + performance.word(0x2a14, pwr.avg_power_filter_tc_ms)?; + performance.word(0x2a1c, clocks(average_periods)?)?; + performance.word(0x2b4c, 50)?; + performance.word(0x2b50, 1)?; + performance.reciprocal(0x2b58, 0x2b60, pwr.se_filter_time_constant)?; + performance.reciprocal(0x2b5c, 0x2b64, pwr.se_filter_time_constant_1)?; + performance.float(0x2b68, pwr.se_ki * dt)?; + performance.float(0x2b6c, pwr.se_ki_1 * dt)?; + performance.float(0x2b74, f32!(65536.0))?; + performance.float(0x2b78, pwr.se_kp)?; + performance.float(0x2b7c, pwr.se_kp_1)?; + performance.word(0x2b84, 100)?; + performance.word(0x2b8c, 100)?; + performance.word(0x2b98, pwr.se_target)?; + performance.word( + 0x2ba4, + pwr.se_filter_time_constant + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word( + 0x2ba8, + pwr.se_filter_time_constant_1 + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word(0x2bac, clocks(pwr.se_filter_time_constant)?)?; + performance.word(0x2bb4, clocks(pwr.se_filter_time_constant_1)?)?; + performance.float(0x2b94, f32!(8000.0))?; + for offset in [0x2bc0, 0x2bc4, 0x2c30] { + performance.float(offset, f32!(65536.0))?; + } + performance.word(0x2be8, 8000)?; + performance.word(0x2bec, 4)?; + performance.word(0x2bf0, pwr.se_inactive_threshold)?; + performance.word(0x2bf4, pwr.se_engagement_criteria as u32)?; + performance.word(0x2bf8, 2)?; + performance.word(0x2bfc, 4)?; + performance.word(0x2c00, pwr.se_reset_criteria)?; + performance.word(0x2c40, pwr.avg_power_min_duty_cycle)?; + performance.word(0x2e2c, 4)?; + performance.float(0x2e38, f32!(0.96875))?; + performance.float(0x2e40, f32!(0.03125))?; + performance.float(0x2e48, f32!(5.0))?; + performance.float(0x2e54, f32!(65536.0))?; + performance.float(0x2e58, f32!(15.0))?; + performance.word(0x2e64, 700)?; + for offset in [0x2e68, 0x2e6c, 0x2e78] { + performance.word(offset, ceiling)?; + } + performance.word(0x2f08, 0x4248)?; + performance.word(0x2f10, 125)?; + performance.word(0x2f14, 1)?; + performance.pointer(0x3b50, sensor)?; + performance.word(0x3b58, 0x4b48)?; + performance.word(0x3b60, 1)?; + performance.word(0x62a8, 1)?; + Ok(Self { + hardware: main.0, + performance: performance.0, + power: power.0, + globals: globals.0, + maximum_khz: top.freq_hz / 1000, + }) + } +} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 7073ecf294d63..d885da1a8647c 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -202,3 +202,8 @@ mod t8122_hwdata; mod m3_init_26; mod m3_profile; mod m3_mmio_26; + +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_power_26; + +mod m3_channel_format; diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index f1768a242147b..abac06668c0a1 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -24,3 +24,15 @@ mod m3_profile; #[path = "../../drivers/gpu/drm/asahi/m3/m3_mmio_26.rs"] mod m3_mmio_26; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_queue_layout.rs"] +mod m3_queue_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pool_layout.rs"] +mod m3_pool_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_layout.rs"] +mod m3_compute_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_channel_format.rs"] +mod m3_channel_format; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 04a2670f39b6c..5dfda83574ea6 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-power.py': [([], PASS)], 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], 'test-j613-dcp-power.py': [([], PASS)], diff --git a/tools/asahi/test-m3-power.py b/tools/asahi/test-m3-power.py new file mode 100644 index 0000000000000..51bea088cd9db --- /dev/null +++ b/tools/asahi/test-m3-power.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production controller writer and soft-float code without a GPU. + +Kernel allocation/DT adapters are replaced by host adapters. Public production +power types and PStateTables are extracted as complete balanced Rust items, +never restated by hand. Their input hashes are printed with the test evidence. +""" +from pathlib import Path +import hashlib +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +M3 = ROOT / "drivers/gpu/drm/asahi/m3" + + +def item(source, anchor): + start = source.index(anchor) + opening = source.index("{", start) + depth = 1 + end = opening + 1 + while depth: + if source[end] == "{": + depth += 1 + elif source[end] == "}": + depth -= 1 + end += 1 + return source[start:end] + + +shim = r''' +#![allow(dead_code)] +extern crate self as kernel; +pub type Error = i32; +pub type Result = core::result::Result; +pub const EINVAL: Error = 22; +pub const EOVERFLOW: Error = 75; +pub const ERANGE: Error = 34; +pub const GFP_KERNEL: () = (); +pub unsafe trait Zeroable {} +#[derive(Debug)] +pub struct KVec(Vec); +impl Default for KVec { fn default() -> Self { Self(Vec::new()) } } +impl KVec { + pub fn new() -> Self { Self::default() } + pub fn resize(&mut self, n: usize, value: T, _: ()) -> Result where T: Clone { + self.0.resize(n, value); Ok(()) + } +} +impl core::ops::Deref for KVec { type Target = Vec; fn deref(&self) -> &Vec { &self.0 } } +impl core::ops::DerefMut for KVec { fn deref_mut(&mut self) -> &mut Vec { &mut self.0 } } +pub mod prelude { pub use crate::{Error,Result,EINVAL,EOVERFLOW,ERANGE,GFP_KERNEL,KVec,Zeroable}; } +pub mod of { + pub struct Property<'a>(pub &'a [u8]); + pub trait PropertyUnit: Sized { const UNIT_SIZE: usize; fn from_bytes(data: &[u8]) -> crate::Result; } + impl PropertyUnit for u32 { + const UNIT_SIZE: usize = 4; + fn from_bytes(data: &[u8]) -> crate::Result { + Ok(u32::from_be_bytes(data.try_into().map_err(|_| crate::EINVAL)?)) + } + } + impl TryFrom> for u32 { + type Error = crate::Error; + fn try_from(value: Property<'_>) -> crate::Result { Self::from_bytes(value.0) } + } +} +''' +fixture = r''' +#[cfg(test)] mod controller_tests { + use super::*; + use float::F32; + fn input() -> hw::PwrConfig { + let mut p = hw::PwrConfig::default(); + p.perf_states = KVec((0..9).map(|i| hw::PState { + freq_hz: if i == 8 { 1_338_000_000 } else { i * 100_000_000 }, + volt_mv: KVec(vec![if i == 0 { 125 } else { 450 + i * 50 }]), pwr_mw: i * 1000, + }).collect()); + p.perf_base_pstate = 1; + p.power_sample_period = 8; + p.ppm_filter_time_constant_ms = 16; + p.avg_power_filter_tc_ms = 16; + p.pwr_filter_time_constant = 2; + p.perf_filter_time_constant = 2; + p.perf_filter_time_constant2 = 2; + p.avg_power_target_filter_tc = 2; + p.se_filter_time_constant = 2; + p.se_filter_time_constant_1 = 2; + p.pwr_min_duty_cycle = 30; + p.avg_power_min_duty_cycle = 20; + p + } + fn build(p: &hw::PwrConfig, sram: &[u32]) -> Result { + m3_power_26::PowerImages::new(p, sram, 0xffff_fc21_0340_0000, + 0xffff_fc21_0350_0000, 0xffff_fc2f_fc00_0000, &[0; 53 * 40]) + } + fn word(b: &[u8], at: usize) -> u32 { u32::from_le_bytes(b[at..at + 4].try_into().unwrap()) } + #[test] fn native_controller_bounds_are_distinct_from_outputs_and_hardware_state() { + let image = build(&input(), &[950_001; 9]).unwrap(); + assert_eq!(image.hardware.len(), 0x2658); + assert_eq!(image.performance.len(), 0x6578); + assert_eq!(image.power.len(), 0x670); + assert_eq!(image.globals.len(), 0xe48); + assert_eq!(image.maximum_khz, 1_338_000); + assert_eq!(word(&image.performance, 0x2c), 1); + assert_eq!(word(&image.performance, 0x30), 1); + assert_eq!(word(&image.performance, 0x38), 0); + assert_eq!(word(&image.performance, 0x9f0), 30); + assert_eq!(word(&image.performance, 0xa38), 800); + assert_eq!(word(&image.performance, 0xb78), 100); + assert_eq!(word(&image.performance, 0xb7c), 800); + assert_eq!(word(&image.hardware, 0x1408 + 64), 951); + assert_eq!(word(&image.performance, 0x2268), 0x4248); + assert_eq!(word(&image.performance, 0x3b58), 0x4b48); + } + #[test] fn malformed_tunables_and_nonfinite_gains_are_rejected() { + for period in [0, 1001, u32::MAX] { + let mut p = input(); p.power_sample_period = period; + assert!(build(&p, &[950_000; 9]).is_err()); + } + for bits in [0x7f800000, 0xff800000, 0x7fc00001] { + let mut p = input(); p.pwr_integral_gain = F32::from_bits(bits); + assert!(build(&p, &[950_000; 9]).is_err()); + } + assert!(build(&input(), &[950_000; 8]).is_err()); + assert!(build(&input(), &[100_000; 9]).is_err()); + let mut p = input(); p.avg_power_target_filter_tc = u32::MAX; + assert!(build(&p, &[950_000; 9]).is_err()); + } + #[test] fn dt_order_is_not_the_firmware_voltage_order() { + let p = input(); + let expected = build(&p, &[950_000; 9]).unwrap(); + let mut shuffled = input(); + shuffled.perf_states.swap(2, 6); + let actual = build(&shuffled, &[950_000; 9]).unwrap(); + assert_eq!(actual.hardware.0, expected.hardware.0); + assert_eq!(actual.performance.0, expected.performance.0); + } +} +''' + + +def main(): + hw = (M3 / "hw.rs").read_text() + init = (M3 / "initdata.rs").read_text() + code = shim + # Keep the production soft-float prefix verbatim. Its unrelated existing + # print-only TODO test lacks an import and is not part of this harness. + soft_float = (ROOT / "drivers/gpu/drm/asahi/float.rs").read_text() + code += "mod float {\n" + soft_float.split("// TODO: Make this an actual test", 1)[0] + "\n}\n" + code += "mod hw { use crate::{KVec,float::F32};\n" + for name in ["PState", "PowerZone", "CsAfrPwrConfig", "PwrConfig"]: + code += "#[derive(Default)]\n" + item(hw, "pub(crate) struct " + name + " {") + "\n" + code += item(hw, "impl PState {") + "\n" + code += "}\nmod initdata { use crate::{prelude::*,hw}; const PSTATE_TABLE_ENTRIES: usize = 16;\n" + code += item(init, "pub(crate) struct PStateTables {") + "\n" + code += item(init, "impl PStateTables {") + "\n}\n" + code += "mod m3_mmio_26 { pub const SLOTS:usize=53; pub const RECORD_BYTES:usize=40; }\n" + code += f'#[path = "{M3}/m3_power_26.rs"] mod m3_power_26;\n' + code += fixture + for path in [M3 / "hw.rs", M3 / "initdata.rs", M3 / "m3_power_26.rs", ROOT / "drivers/gpu/drm/asahi/float.rs"]: + print(hashlib.sha256(path.read_bytes()).hexdigest(), path.relative_to(ROOT), flush=True) + with tempfile.TemporaryDirectory(prefix="m3-power-") as directory: + source = Path(directory) / "test.rs" + source.write_text(code) + for flags in [[], ["-O"]]: + binary = Path(directory) / "tests" + subprocess.run(["rustc", "--edition=2021", "--test", *flags, str(source), "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() From ee69dab429ed9f6498c03303dd34c9eb4af44ff2 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:41 -0400 Subject: [PATCH 034/392] drm/asahi: prepare extended firmware queues and controller handoff Construct native queue, pool, context and notifier records. Read controller inputs from the admitted OPP handoff, preserve nanosecond durations and reject unsupported optional flags. Prepare native startup without enabling extended admission. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 57 +++++++- drivers/gpu/drm/asahi/m3/m3_compute_layout.rs | 21 +++ .../gpu/drm/asahi/m3/m3_compute_sequence.rs | 110 +++++++++++++++ .../gpu/drm/asahi/m3/m3_compute_storage.rs | 66 +++++++++ drivers/gpu/drm/asahi/m3/m3_config.rs | 79 ++++++++++- drivers/gpu/drm/asahi/m3/m3_device.rs | 6 + drivers/gpu/drm/asahi/m3/m3_drm.rs | 3 +- drivers/gpu/drm/asahi/m3/m3_pool.rs | 31 +++++ drivers/gpu/drm/asahi/m3/m3_pool_layout.rs | 128 ++++++++++++++++++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 20 +++ drivers/gpu/drm/asahi/m3/m3_queue_layout.rs | 41 ++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 57 ++++++++ drivers/gpu/drm/asahi/m3/m3_state_layout.rs | 52 +++++++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 8 +- drivers/gpu/drm/asahi/mmu.rs | 4 +- tools/asahi/m3-runtime-tests.rs | 10 ++ 16 files changed, 678 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 1893c113cf2a7..355cd44b04025 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -260,7 +260,14 @@ impl PstatePolicy { /// The runtime backend's InitData contents: the generated object images and the /// performance-state policy. +pub(crate) struct ExtendedInputs { + pub(crate) power: hw::PwrConfig, + pub(crate) sram_uv: KVec, + pub(crate) bases: super::m3_mmio_26::RegisterBases, +} + pub(crate) struct Contents { + pub(crate) extended: Option, /// Generated images, per owner; None for owners that stay zero. Only the three image /// owners' images are uploaded; the records' are checked. images: KVec>>, @@ -304,6 +311,10 @@ impl Contents { experiment: Option<&Experiment>, ) -> Result { let dev = pdev.as_ref(); + if firmware.profile() == super::m3_profile::Profile::Extended { + if experiment.is_some() { return Err(EINVAL); } + return Self::extended(dev, soc); + } let param = m3_params::initdata_param(); check_version(dev, firmware, experiment)?; let iomaps = match experiment { @@ -370,12 +381,54 @@ impl Contents { ); let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0); let hwdata_object = soc.hwdata_object; - Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) + Ok(Contents { extended: None, images, pstates, iomaps, table, read_only_slots, hwdata_object }) + } + + fn extended(dev: &device::Device, soc: &Soc) -> Result { + let cfg = soc.hwcfg.ok_or(ENODEV)?; + if soc.chip_id != 0x8122 || cfg.chip_id != 0x8122 { return Err(ENODEV); } + let power = hw::PwrConfig::load(dev, cfg)?; + let tables = initdata::PStateTables::new(&power, true, None, false)?; + if tables.len != 9 || tables.max != 8 || tables.max_frequency_khz(&power) != 1_338_000 { + return Err(EINVAL); + } + let node = dev.of_node().ok_or(ENODEV)?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0).ok_or(EINVAL)?; + let mut sram_uv = KVec::new(); + for opp in opps.children() { + let values: KVec = opp.get_property(c_str!("opp-microvolt-sram"))?; + if values.len() != 1 { return Err(EINVAL); } + sram_uv.push(values[0], GFP_KERNEL)?; + } + if sram_uv.len() != power.perf_states.len() { return Err(EINVAL); } + let block = |index: usize| -> Result { + Ok(cfg.io_mappings.get(index).and_then(|value| value.as_ref()).ok_or(EINVAL)?.base as u64) + }; + let bases = super::m3_mmio_26::RegisterBases { + gpu: soc.windows.sgx, interrupt: block(2)?, + memory: block(11)?.checked_add(0x1c4000).ok_or(EOVERFLOW)?, + telemetry_write: block(18)?, telemetry_read: block(19)?, + neural: block(25)?, metrology: block(26)?, + }; + let mut freqs = [0; 16]; + for (index, value) in freqs.iter_mut().enumerate().take(tables.len) { + *value = tables.primary_state(&power, index).freq_hz / 1_000_000; + } + // This profile enables the native firmware's thermal/PID controllers. + // The legacy host-only thermal governor must not replace those inputs. + let pstates = PstatePolicy { table_max: 8, table_max_mhz: 1338, + max: 8, safe: 8, boot: 1, max_mhz: 1338, safe_mhz: 1338, + boot_mhz: freqs[1], freqs, thermal: ThermalMode::Off }; + let mut images = KVec::new(); + for _ in 0..super::m3_init_26::COUNT { images.push(None, GFP_KERNEL)?; } + Ok(Self { extended: Some(ExtendedInputs { power, sram_uv, bases }), + images, pstates, iomaps: &[], table: (8, 1338), read_only_slots: 0, hwdata_object: None }) } /// Check the contents before they are uploaded. pub(crate) fn check_for_upload(&self) -> Result { - if self.images.len() == storage::COUNT { + let count = if self.extended.is_some() { super::m3_init_26::COUNT } else { storage::COUNT }; + if self.images.len() == count { Ok(()) } else { Err(EINVAL) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs index 64c8cb005c4a1..ace58e38f38a6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs @@ -71,7 +71,14 @@ impl Command { /// Firmware envelopes vary independently of the GPU's CDM stream format. pub(crate) fn encode_for(&self, profile: super::m3_profile::Profile, asid_generation: u8, event_slot: u8, out: &mut [u8]) -> Result<(), Error> { + self.encode_with_usc(profile, asid_generation, event_slot, 0, out) + } + + pub(crate) fn encode_with_usc(&self, profile: super::m3_profile::Profile, + asid_generation: u8, event_slot: u8, usc_base: u64, out: &mut [u8]) -> Result<(), Error> { let extended = profile == super::m3_profile::Profile::Extended; + if usc_base & 0xffff_ffff != 0 || usc_base >= 1 << 42 + || (!extended && usc_base != 0) { return Err(Error::Address); } let size = if extended { EXTENDED_SIZE } else { SIZE }; if out.len() < size { return Err(Error::Size); } if extended && (self.register_set != RegisterSet::G15G || event_slot >= 4 @@ -122,6 +129,7 @@ impl Command { emit(0x1a540, 1); emit(0x014a9, 1); emit(0x0a351, 1); + if usc_base != 0 { emit(0x10071, usc_base); } } pointer(bytes, 0x720, registers.get()); bytes[0x728..0x72a].copy_from_slice(&(count as u16).to_le_bytes()); @@ -232,4 +240,17 @@ mod tests { assert_eq!(read64(&b, META), c.stamp.get()); assert_eq!(&b[SIZE..], &[0xa5; EXTENDED_SIZE - SIZE]); } + #[test] + fn optional_usc_window_is_a_dispatch_register_without_stream_rewriting() { + let c = fixture(); + let mut output = [0xa5; EXTENDED_SIZE]; + c.encode_with_usc(Profile::Extended, 1, 0, 1 << 32, &mut output).unwrap(); + assert_eq!(read32(&output, 0x728) & 0xffff, 25); + assert_eq!(read32(&output, REGISTERS + 24 * REGISTER_STRIDE), 0x10071); + assert_eq!(read64(&output, REGISTERS + 24 * REGISTER_STRIDE + 4), 1 << 32); + output.fill(0xa5); + assert!(c.encode_with_usc(Profile::Extended, 1, 0, 1, &mut output).is_err()); + assert_eq!(output, [0xa5; EXTENDED_SIZE]); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs index a10c3040d570f..bd78c03823aa0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs @@ -82,3 +82,113 @@ impl Sequence { Ok(()) } } + +impl Sequence { + pub(crate) const EXTENDED_LENGTH: usize = 0x300; + pub(crate) const EXTENDED_ATTACHMENTS: usize = 0x58; + pub(crate) const EXTENDED_HAS_ATTACHMENTS: usize = 0x2c0; + + /// The extended firmware operates on a native G15 CDM stream. Its firmware + /// microsequence has wider timestamp operations and different engine state. + pub(crate) fn encode_extended(&self, auxiliary: command::GpuVa, + context_store: super::m3_pool_layout::GpuRegion, queue_sequence: u64, + output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.generation != 0 { return Err(Error::Address); } + if context_store.size() < 0x20000 { return Err(Error::Size); } + let registers = self.command.at(command::REGISTERS, 24 * command::REGISTER_STRIDE)?; + let engine = self.command.at(0x76c, 0x60)?; + let clock = self.command.at(0x82c, 8)?; + let pairs = self.command.at(0x834, 16)?; + let end_pair = self.command.at(0x83c, 8)?; + let pool_pointer = self.command.at(0x86a, 8)?; + let status = self.command.at(0x8ac, 4)?; + let time_aux = self.command.at(0x89c, 8)?; + let elapsed = self.command.at(0x7dc, 8)?; + let tail = self.command.at(0x88c, 8)?; + let activity = self.stats.at(0, 8)?; + let bytes = output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes, 0, 0x0b); + pointer(bytes, 0x14, registers); + pointer(bytes, 0x1c, activity); + pointer(bytes, 0x24, self.queue.get()); + word(bytes, 0x2c, self.context); + word(bytes, 0x30, 1); + pointer(bytes, 0x38, queue_sequence); + pointer(bytes, 0x44, engine); + word(bytes, 0x50, 1); + pointer(bytes, 0x160, self.pool.get()); + pointer(bytes, 0x178, auxiliary.get()); + for (offset, delta) in [0, 0xf400, 0x1e800, 0x1f000].into_iter().enumerate() { + pointer(bytes, 0x188 + offset * 8, context_store.base() + delta); + } + pointer(bytes, 0x1a8, status); + word(bytes, 0x1b8, 2); + // Timestamp records are 76 bytes; WFI sits between the two records. + for (offset, opcode, selected) in [(0x1bc, 0x8000_0003, pairs), (0x20c, 3, end_pair)] { + word(bytes, offset, opcode); + pointer(bytes, offset + 4, clock); + pointer(bytes, offset + 0xc, pairs); + pointer(bytes, offset + 0x14, selected); + pointer(bytes, offset + 0x1c, self.queue.get()); + pointer(bytes, offset + 0x2c, time_aux); + pointer(bytes, offset + 0x34, elapsed); + pointer(bytes, offset + 0x44, 0xc10000); + } + word(bytes, 0x208, 1); + let finish = 0x258; + word(bytes, finish, 0x0c); + pointer(bytes, finish + 4, activity); + pointer(bytes, finish + 0xc, self.queue.get()); + word(bytes, finish + 0x14, self.context); + pointer(bytes, finish + 0x18, engine); + pointer(bytes, finish + 0x24, 0xc10000); + pointer(bytes, finish + 0x2c, self.fw_stamp.get()); + word(bytes, finish + 0x34, self.stamp_value); + pointer(bytes, finish + 0x5c, pool_pointer); + word(bytes, finish + 0x64, (-(finish as i32)) as u32); + pointer(bytes, finish + 0x69, status); + pointer(bytes, finish + 0x71, tail); + word(bytes, 0x2d4, 0x4000_0002); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Sequence { + let fw = |i: u64| FirmwareVa::new(0xffff_fc21_a000_0000 + i * 0x4000).unwrap(); + Sequence { command: Region::new(fw(0).get(), command::EXTENDED_SIZE).unwrap(), + stats: Region::new(fw(1).get(), 0xe10).unwrap(), + notifier: Region::new(fw(2).get(), 0x200).unwrap(), queue: fw(3), + context: 3, generation: 0, pool: fw(4), scratch: [fw(5); 5], + fw_stamp: fw(6), stamp_value: 0x700 } + } + fn ptr(bytes: &[u8], at: usize) -> u64 { u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) } + #[test] fn extended_timestamps_and_retirement_refer_to_their_live_owners() { + let s = fixture(); + let mut output = [0xa5; STORAGE + 8]; + s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(), + super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 7, &mut output).unwrap(); + assert_eq!(ptr(&output, 0x1c), s.stats.at(0, 8).unwrap()); + assert_eq!(ptr(&output, 0x38), 7); + assert_eq!(ptr(&output, 0x188), 0x6000_0000); + assert_eq!(ptr(&output, 0x1bc + 0x14), s.command.at(0x834, 8).unwrap()); + assert_eq!(ptr(&output, 0x20c + 0x14), s.command.at(0x83c, 8).unwrap()); + assert_eq!(ptr(&output, 0x258 + 0x2c), s.fw_stamp.get()); + assert_eq!(ptr(&output, 0x258 + 0x5c), s.command.at(0x86a, 8).unwrap()); + assert_eq!(&output[STORAGE..], &[0xa5; 8]); + } + #[test] fn extended_sequence_rejects_legacy_command_extents_and_event_generations() { + for generation in [0, 1] { + let mut s = fixture(); + s.generation = generation; + if generation == 0 { s.command = Region::new(s.command.at(0, 1).unwrap(), command::SIZE).unwrap(); } + let mut output = [0xa5; STORAGE]; + assert!(s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(), + super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 0, &mut output).is_err()); + assert_eq!(output, [0xa5; STORAGE]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs index 6bd3aecda387d..a897a45f0922d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs @@ -87,3 +87,69 @@ pub(crate) fn allocation(index:usize)->Result { pub(crate) fn coalesce_stamp_flush(wide:bool,timestamps:[u64;2],last:bool)->bool { wide && timestamps==[0,0] && !last } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) + -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + match index { + // The initial allocator has eight 4-MiB payload owners. Retain the + // historical indices without allocating unused 4-MiB payloads. + 9..=22 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; } + POOL_PAGES => owner.size = 0x20000, + POOL_TABLE => { owner.size = 0x4000; owner.mapping = Mapping::Client(POOL_TABLE_VA & !0x3fff); } + POOL => owner.size = 0xa0, + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x200, + EVENT => owner.size = 8, + QUEUE => owner.size = 0x24c0, + PREEMPTION => owner.size = 0x4000, + COMMAND => { owner.size = 0x8b0; owner.mapping = Mapping::CommandAlias(0x73_0000_0000); } + TAIL_SCRATCH => { owner.size = 0x4000; owner.mapping = Mapping::Client(0x72_0100_0000); } + 46 => { owner.size = 0x40000; owner.mapping = Mapping::Client(0x72_0000_0000); } + 45 | 47..=49 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; } + _ => {} + } + Ok(owner) +} + +pub(crate) fn slot_offset_for(profile: super::m3_profile::Profile, index: usize, slot: usize) + -> Result { + if slot >= SLOTS { return Err(InvalidOwner); } + let size = allocation_for(profile, index)?.size; + Ok(if RESET.contains(&index) { ((size + 0x3fff) & !0x3fff) * slot } else { 0 }) +} + +pub(crate) fn batch_allocation_for(profile: super::m3_profile::Profile, index: usize) + -> Result { + let mut owner = allocation_for(profile, index)?; + owner.size += slot_offset_for(profile, index, SLOTS - 1)?; + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::m3_profile::Profile; + #[test] fn extended_commands_do_not_straddle_another_slots_page() { + let command = allocation_for(Profile::Extended, COMMAND).unwrap(); + let Mapping::CommandAlias(base) = command.mapping else { panic!("missing command alias") }; + assert_eq!(base & 0x3fff, 0); + for slot in 0..SLOTS - 1 { + let start = slot_offset_for(Profile::Extended, COMMAND, slot).unwrap(); + let next = slot_offset_for(Profile::Extended, COMMAND, slot + 1).unwrap(); + assert!(start + command.size <= next); + } + assert_eq!(batch_allocation_for(Profile::Extended, QUEUE).unwrap().size, 0x24c0); + assert!(slot_offset_for(Profile::Extended, COMMAND, SLOTS).is_err()); + } + #[test] fn legacy_plan_is_byte_for_byte_the_same_for_every_owner_and_slot() { + for index in 0..COUNT { + assert_eq!(allocation_for(Profile::Legacy, index), allocation(index)); + for slot in 0..SLOTS { + assert_eq!(slot_offset_for(Profile::Legacy, index, slot), slot_offset(index, slot)); + } + assert_eq!(batch_allocation_for(Profile::Legacy, index), batch_allocation(index)); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index 67f4f963637f2..bfb408b07a530 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -51,6 +51,7 @@ pub(crate) struct Config { objects: KVec, profile: super::m3_profile::Profile, control_generation: u32, + io_descriptors: Option>, _iomaps: KVec, pub(crate) completed_events: u64, pstates: crate::m3::m3_adt_config::PstateWatch, @@ -67,6 +68,17 @@ impl Config { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware, contents: &crate::m3::m3_adt_config::Contents) -> Result { contents.check_for_upload()?; + if let Some(inputs) = &contents.extended { + if firmware.profile() != super::m3_profile::Profile::Extended { return Err(EINVAL); } + let owners = ExtendedStorage::new(dev, uat, &inputs.power, &inputs.sram_uv, inputs.bases)?; + let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; + return Ok(Self { objects: owners.objects, _iomaps: owners.iomaps, + profile: firmware.profile(), control_generation: 0, completed_events: 0, + io_descriptors: Some(owners.descriptors), + pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + request_top: 8, request_value: 0, request_checked: false, request_check_at: None, + request_writes: 0, request_observed: 0 }); + } let mut objects = KVec::new(); let mut iomaps = KVec::new(); for index in 0..storage::COUNT { @@ -142,7 +154,7 @@ impl Config { crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; - Ok(Self { objects, profile: firmware.profile(), control_generation: 0, + Ok(Self { objects, profile: firmware.profile(), control_generation: 0, io_descriptors: None, _iomaps: iomaps, completed_events: 0, pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, @@ -156,6 +168,15 @@ impl Config { objects[RUNTIME_POINTERS].write(0, &records.runtime)?; objects[CONTROL_REGION].write(0, &records.control) } + pub(crate) fn publish_io_table(&self, + pdev: &kernel::platform::Device>, + firmware: &crate::m3::m3_firmware::Firmware) -> Result { + if let Some(descriptors) = &self.io_descriptors { + firmware.publish_io_table(pdev, descriptors)?; + } + Ok(()) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; @@ -164,6 +185,8 @@ impl Config { Ok(()) } pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { + // Native controller inputs supersede the legacy fixed-request interface. + if self.profile == super::m3_profile::Profile::Extended { return Ok(()); } let Some(value)=crate::m3::m3_consistent_perf::request(chip, crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; if value==0 && self.request_value==0 {return Ok(());} @@ -195,6 +218,14 @@ impl Config { /// state the firmware reports against the ceiling and the runtime cap. Called after boot and /// after every job, by the job thread. pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let state = device.pstate_register()? & 15; + if state > self.request_top { + dev_err!(dev.as_ref(), "M3: native performance state {} exceeds {} ({})\n", state, self.request_top, what); + return Err(ERANGE); + } + return Ok(()); + } let now = kernel::time::Instant::::now(); if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) { crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; @@ -206,13 +237,25 @@ impl Config { Ok(()) } pub(crate) fn stats_region(&self) -> Result { - let owner = &self.objects[HARDWARE_DATA]; + let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { + UNKNOWN_C3 + } else { HARDWARE_DATA }]; init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) } pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, sequence: u32) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let state = device.pstate_register()? & 15; + let performance = &mut self.objects[super::m3_init_26::PERFORMANCE_CONTROL]; + let actual = performance.read_u32(0x2c)?; + let target = performance.read_u32(0x30)?; + let hardware = performance.read_u32(0x38)?; + dev_info!(dev.as_ref(), "M3_NATIVE_POWER sample={} state={} actual={} target={} hardware={}\n", + sequence, state, actual, target, hardware); + return Ok(()); + } let cap = self.thermal.allowed(kernel::time::Instant::::now()); self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], device.pstate_register()?, cap, sequence)?; @@ -232,6 +275,7 @@ impl Config { } pub(crate) fn render_pb(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?) } pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { @@ -274,6 +318,13 @@ impl Config { && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) } pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let ready = self.objects[RUNTIME_FLAGS].read_u32(0x14)?; + let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; + dev_info!(dev.as_ref(), "M3: extended firmware ready={} control-generation={}/{}\n", + ready, consumed, self.control_generation); + return Ok(()); + } let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; @@ -284,12 +335,14 @@ impl Config { } /// The firmware's halt counter and its halted and resume flags. pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } let control=&mut self.objects[CONTROL_REGION]; Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?)) } /// Let a halted firmware continue: clear halted, then set resume. These /// words keep the halt/resume layout of earlier Apple GPU firmware. pub(crate) fn resume_halted(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } let control=&mut self.objects[CONTROL_REGION]; control.u32(0x4590,0)?; agx_memory::publish(); @@ -303,6 +356,18 @@ impl Config { /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. /// Nothing is acknowledged, cleared or resumed here. pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + for (owner, start, length) in [(super::m3_init_26::RECOVERY_ACTIVITY, 0, 0x60), + (CONTROL_REGION, 0x4ebc, 0x100)] { + for offset in (start..start + length).step_by(32) { + let mut bytes = [0; 32]; + self.objects[owner].read(offset, &mut bytes)?; + dev_err!(dev.as_ref(), "M3: firmware recovery owner={} offset={:#x} bytes={:02x?}\n", + owner, offset, bytes); + } + } + return Ok(()); + } let control=&mut self.objects[CONTROL_REGION]; let halt_count=control.read_u64(0x4580)?; let halted=control.read_u32(0x4590)?; @@ -318,7 +383,7 @@ impl Config { /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the /// completion count is not changed. pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result { - const SIZE: usize = 0x38; + let size = if self.profile == super::m3_profile::Profile::Extended { 0x48 } else { 0x38 }; const COUNT: usize = 256; let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize; @@ -327,8 +392,8 @@ impl Config { let mut index = r; for _ in 0..16 { if index == w { break; } - let mut raw = [0u8; SIZE]; - self.objects[storage::EVENT*2+1].read(index * SIZE, &mut raw)?; + let mut raw = [0u8; 0x48]; + self.objects[storage::EVENT*2+1].read(index * size, &mut raw[..size])?; dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n", index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw); index = (index + 1) % COUNT; @@ -357,7 +422,9 @@ impl Config { let state = storage::DEVICE_CONTROL * 2; let cursors = [self.objects[state].read_u32(0)?, self.objects[state].read_u32(0x10)?, self.objects[state].read_u32(0x20)?]; - let cursor = format::idle_cursor(cursors).map_err(|_| EBUSY)?; + let cursor = format::idle_cursor(cursors).map_err(|error| { + if error == init::Error::Bounds { EIO } else { EBUSY } + })?; let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; let generation = action.generation(self.control_generation, consumed).map_err(|_| EOVERFLOW)?; let mut record = [0; format::CONTROL_BYTES]; diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index c43017a8933c0..05a3961369a6d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -176,6 +176,12 @@ impl Device { /// Read-only snapshot. Do not select or acknowledge fault banks here; /// the firmware owns those registers until recovery is ported. pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { + if self.firmware.profile() == super::m3_profile::Profile::Extended { + // Adaptive clock gating can leave individual engine banks inaccessible + // even while the GPU performance-state register says powered up. + dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); + return Ok(()); + } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; let sgx = sgx_guard.as_view(); let pstate = sgx.try_read32(0xe01000)? & 0xf; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 796abc7793f90..e1db1adf9e6a9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -258,7 +258,8 @@ impl gpu::Gpu for Backend { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64) -> Result> { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, - self.soc.features.fragment_dependency)?,GFP_KERNEL)?) + self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64 != 0, + self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64 != 0)?,GFP_KERNEL)?) } fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs index ab50ab7eca8a1..4eb284e1291bd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -39,3 +39,34 @@ impl Owners { objects[self.manager].write(0, &manager) } } + +impl Owners { + pub(crate) fn initialize_extended(&self, objects: &mut [Buffer], slot: u8, + compute_only: bool) -> Result { + let mut backings = KVec::new(); + for index in self.backing.clone().take(8) { + let buffer = objects.get(index).ok_or(EINVAL)?; + backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?; + } + let backing = wire::ModernBacking(&backings); + let populated = backing.pages().map_err(|_| EINVAL)?; + let pages = objects.get(self.pages).ok_or(EINVAL)?; + let entries = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?; + let table = objects.get(self.table).ok_or(EINVAL)?; + let groups = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?; + if groups.size() < populated / 512 * 64 { return Err(EINVAL); } + let descriptor = wire::ModernPool { entries, groups, + submitted: FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?, + identity: FirmwareVa::new(objects.get(self.manager).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?, + slot, capacity: (entries.size() / 8).try_into()?, populated: populated.try_into()?, compute_only }; + let mut bytes = [0; wire::ModernPool::SIZE]; + descriptor.encode(&mut bytes).map_err(|_| EINVAL)?; + for index in 0..populated { + objects[self.pages].u64(index * 8, backing.page(index).map_err(|_| EINVAL)?)?; + } + for index in 0..populated / 512 { + objects[self.table].write(index * 64, &backing.group(index).map_err(|_| EINVAL)?)?; + } + objects[self.manager].write(0, &bytes) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs index a8474d262bc26..ac7e6771f7880 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs @@ -69,3 +69,131 @@ impl<'a> Pool<'a> { Ok(()) } } + +/// Extended firmware's private-memory pool descriptor. The owners and initial +/// population are supplied separately from the descriptor's firmware state. +pub(crate) struct ModernPool { + pub(crate) identity: FirmwareVa, + pub(crate) slot: u8, + pub(crate) entries: GpuRegion, + pub(crate) groups: GpuRegion, + pub(crate) submitted: FirmwareVa, + pub(crate) capacity: u32, + pub(crate) populated: u32, + pub(crate) compute_only: bool, +} + +impl ModernPool { + pub(crate) const SIZE: usize = 0xa0; + + pub(crate) fn encode(&self, output: &mut [u8]) -> Result<(), Error> { + let entry_bytes = (self.capacity as usize).checked_mul(8).ok_or(Error::Size)?; + let group_bytes = (self.populated as usize / 512).checked_mul(8).ok_or(Error::Size)?; + if !(1..=64).contains(&self.slot) || self.capacity == 0 || self.populated >= self.capacity || self.populated % 512 != 0 + || self.entries.base() & 0x3fff != 0 || self.groups.base() & 0x3fff != 0 + || self.entries.size() < entry_bytes || self.groups.size() < group_bytes + || self.submitted.get() & 7 != 0 || self.identity.get() & 7 != 0 { return Err(Error::Address); } + let group_count: u32 = group_bytes.try_into().map_err(|_| Error::Size)?; + let bytes = output.get_mut(..Self::SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + bytes[..8].copy_from_slice(&self.identity.get().to_le_bytes()); + word(bytes, 8, self.slot.into()); + word(bytes, 12, (self.populated == 0).into()); + word(bytes, 16, 2); + bytes[0x14..0x1c].copy_from_slice(&self.entries.base().to_le_bytes()); + word(bytes, 0x1c, self.capacity); + word(bytes, 0x24, self.populated); + word(bytes, 0x28, (self.populated == 0).into()); + word(bytes, 0x2c, self.populated); + bytes[0x30..0x38].copy_from_slice(&self.groups.base().to_le_bytes()); + word(bytes, 0x40, 4); + word(bytes, 0x48, group_count); + bytes[0x4c..0x54].copy_from_slice(&self.submitted.get().to_le_bytes()); + word(bytes, 0x5c, (self.compute_only && self.populated != 0).into()); + word(bytes, 0x84, (self.populated != 0).into()); + word(bytes, 0x94, 2); + Ok(()) + } +} + +#[cfg(test)] +mod modern_tests { + use super::*; + fn pool(compute_only: bool) -> ModernPool { + ModernPool { identity: FirmwareVa::new(0xffff_fc21_9100_0000).unwrap(), slot: 1, + entries: GpuRegion::new(0x5000_0000, 0x20000).unwrap(), + groups: GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + submitted: FirmwareVa::new(0xffff_fc21_9000_0000).unwrap(), + capacity: 0x4000, populated: 512, compute_only } + } + fn word(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) + } + #[test] fn shared_render_pool_never_gets_compute_exclusion() { + for compute in [false, true] { + let mut output = [0xa5; ModernPool::SIZE + 8]; + pool(compute).encode(&mut output).unwrap(); + assert_eq!(word(&output, 0x5c), compute.into()); + assert_eq!(word(&output, 0x48), 8); + assert_eq!(word(&output, 0x84), 1); + assert_eq!(&output[ModernPool::SIZE..], &[0xa5; 8]); + } + } + #[test] fn bad_population_and_insufficient_owners_reject_before_mutation() { + for pages in [1, 511, 513, 0x4000] { + let mut value = pool(true); value.populated = pages; + let mut output = [0xa5; ModernPool::SIZE]; + assert!(value.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; ModernPool::SIZE]); + } + let mut value = pool(true); value.capacity = 0x4001; + let mut output = [0xa5; ModernPool::SIZE]; + assert!(value.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; ModernPool::SIZE]); + } +} + +/// Populate the extended freelist from independently retained 4-MiB owners. +/// Hardware group records describe 2-MiB subregions, not entire owners. +pub(crate) struct ModernBacking<'a>(pub(crate) &'a [GpuRegion]); +impl ModernBacking<'_> { + pub(crate) fn pages(&self) -> Result { + if self.0.is_empty() || self.0.len() > 8 + || self.0.iter().any(|owner| owner.size() != BACKING_SIZE || owner.base() & 0x3fff != 0) { + return Err(Error::Size); + } + Ok(self.0.len() * BACKING_SIZE / PAGE_SIZE) + } + pub(crate) fn page(&self, index: usize) -> Result { + if index >= self.pages()? { return Err(Error::Index); } + let per_owner = BACKING_SIZE / PAGE_SIZE; + Ok(self.0[index / per_owner].base() + (index % per_owner * PAGE_SIZE) as u64) + } + pub(crate) fn group(&self, index: usize) -> Result<[u8; 64], Error> { + if index >= self.pages()? / 512 { return Err(Error::Index); } + let base = self.page(index * 512)?; + let mut record = [0; 64]; + record[..8].copy_from_slice(&(base | (1 << 61)).to_le_bytes()); + Ok(record) + } +} + +#[cfg(test)] +mod backing_tests { + use super::*; + #[test] fn group_records_never_cross_a_noncontiguous_backing_owner() { + let owners = [GpuRegion::new(0x5000_0000, BACKING_SIZE).unwrap(), + GpuRegion::new(0x6000_0000, BACKING_SIZE).unwrap()]; + let backing = ModernBacking(&owners); + assert_eq!(backing.pages(), Ok(2048)); + assert_eq!(backing.page(1024), Ok(0x6000_0000)); + let second = backing.group(2).unwrap(); + assert_eq!(u64::from_le_bytes(second[..8].try_into().unwrap()), (1 << 61) | 0x6000_0000); + assert_eq!(&second[8..], &[0; 56]); + assert!(backing.group(4).is_err()); + assert!(backing.page(2048).is_err()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index c70a88029072b..004ac2c7d1eef 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,13 @@ impl Profile { } } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { + match self { + Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), + Self::Extended => Some(elapsed), + } + } + pub(crate) const fn vm_acknowledged(self, params_bytes: u64, flags: u32) -> bool { // Both profiles consume G15 records; the existing VM padding stays zero. params_bytes >= PARAMS_REQUIRED && flags == 0 @@ -66,4 +73,17 @@ mod tests { assert_eq!(Profile::Extended.user_end(), 0x3ff_ffff_8000); assert_eq!(Profile::Legacy.user_end(), 0x2ff_ffff_8000); } + #[test] + fn native_nanoseconds_are_never_rescaled_as_legacy_ticks() { + assert_eq!( + Profile::Extended.gpu_duration_ns(24_000_000), + Some(24_000_000) + ); + assert_eq!( + Profile::Legacy.gpu_duration_ns(24_000_000), + Some(1_000_000_000) + ); + assert_eq!(Profile::Extended.gpu_duration_ns(u64::MAX), Some(u64::MAX)); + assert_eq!(Profile::Legacy.gpu_duration_ns(u64::MAX), None); + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs index 3daba7acba2d0..8fd5ffdf1d30d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs @@ -113,3 +113,44 @@ pub(crate) fn ring_entry(index: u32, capacity: u32, seeds: &[FirmwareVa]) -> Res if seeds.is_empty() || seeds.len() > capacity as usize || capacity > 65536 || index >= capacity { return Err(Error::Index); } Ok(if (index as usize) < seeds.len() { seeds[index as usize].0.to_le_bytes() } else { [0xcc; 8] }) } + +impl QueueInfo { + pub(crate) const EXTENDED_SIZE: usize = 0x24c0; + + /// Scheduler storage belongs to this same retained allocation; it is not + /// interchangeable with the legacy GPU scratch object. + pub(crate) fn encode_extended(&self, owner: FirmwareVa, output: &mut [u8]) -> Result<(), Error> { + let end = owner.get().checked_add(Self::EXTENDED_SIZE as u64).ok_or(Error::Address)?; + if end > 0xffff_fc30_0000_0000 { return Err(Error::Address); } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + for (offset, value) in [(0, self.state.get()), (8, self.ring.get()), + (16, self.job_list.get()), (24, owner.get() + 0xb0), (0xa4, self.context.get())] { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for offset in [0x2c, 0x4c] { bytes[offset..offset + 4].fill(0xff); } + for offset in [0x30, 0x34, 0x48] { + bytes[offset..offset + 4].copy_from_slice(&2u32.to_le_bytes()); + } + bytes[0x38..0x40].copy_from_slice(&0xffff_0000_0000_0000u64.to_le_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn scheduler_pointer_relocates_with_its_retained_queue_owner() { + let fw = |i: u64| FirmwareVa::new(0xffff_fc21_4000_0000 + i * 0x4000).unwrap(); + let info = QueueInfo { state: fw(1), ring: fw(2), job_list: fw(3), + gpu_scratch: fw(4), context: fw(5), uuid: 9 }; + let mut output = vec![0xa5; QueueInfo::EXTENDED_SIZE + 8]; + info.encode_extended(fw(0), &mut output).unwrap(); + assert_eq!(u64::from_le_bytes(output[24..32].try_into().unwrap()), fw(0).get() + 0xb0); + assert_eq!(&output[0xb0..QueueInfo::EXTENDED_SIZE], &vec![0; QueueInfo::EXTENDED_SIZE - 0xb0]); + assert_eq!(&output[QueueInfo::EXTENDED_SIZE..], &[0xa5; 8]); + let mut short = [0xa5; INFO_SIZE]; + assert!(info.encode_extended(fw(0), &mut short).is_err()); + assert_eq!(short, [0xa5; INFO_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 1a366d7f4b340..40399a80a01f3 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1214,6 +1214,9 @@ impl Runtime { } fn boot_inner(&mut self, pdev: &platform::Device>) -> Result { use crate::m3::t8122_start::BootStep; + if self.inner.device.firmware().profile() == super::m3_profile::Profile::Extended { + return self.boot_extended(pdev); + } let root = self.inner.config.root(); dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); self.inner.boot_step = BootStep::Publish; @@ -1245,6 +1248,60 @@ impl Runtime { dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n"); Ok(()) } + fn boot_extended(&mut self, pdev: &platform::Device>) -> Result { + use super::m3_channel_format::Action; + use crate::m3::t8122_start::BootStep; + // The first control is pending when the root becomes visible. Its + // retirement comes from InitData consumption, without a work kick. + let first = self.inner.config.control_extended(Action::Initialize)?; + self.inner.config.publish_io_table(pdev, self.inner.device.firmware())?; + let root = self.inner.config.root(); + Pin::new(&mut self.inner.transport).send_message(0x20, + 0x0081_0000_0000_0000 | (root & ((1u64 << 44) - 1)))?; + self.inner.boot_step = BootStep::AwaitReady; + let deadline = Instant::::now(); + while !self.inner.config.ready()? { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); } + fsleep(Delta::from_millis(1)); + } + self.retire_extended_control(first)?; + self.inner.boot_step = BootStep::DeviceControl; + // Each data master registers its two firmware engine classes. + for ordinal in 0..6 { + let action = Action::Register { pipe: ordinal / 2, engine: ordinal % 2 + 1 }; + self.send_extended_control(action)?; + } + self.inner.boot_step = BootStep::PostReady; + self.send_extended_control(Action::Idle { enabled: true })?; + let inner: &mut Inner = &mut *self.inner; + inner.config.check_pstate(&inner.drm, &inner.device, "after native boot")?; + inner.config.log_ready(&inner.drm)?; + inner.state.health.set_gpu_pending(false); + dev_info!(pdev.as_ref(), "M3: extended firmware ready; native controllers and idle policy admitted\n"); + Ok(()) + } + + fn send_extended_control(&mut self, action: super::m3_channel_format::Action) -> Result { + let cursor = self.inner.config.control_extended(action)?; + Pin::new(&mut self.inner.transport).send_message(0x21, 0x0084_0000_0000_0011)?; + self.retire_extended_control(cursor) + } + + fn retire_extended_control(&mut self, expected: u32) -> Result { + let deadline = Instant::::now(); + loop { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if inner.config.control_done(expected)? { return Ok(()); } + if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); } + fsleep(Delta::from_millis(1)); + } + } + fn send_control(&mut self, opcode: u32) -> Result { let next = match self.inner.config.control(opcode) { Ok(next) => next, diff --git a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs index 199b7082b26c2..6eeafea666e74 100644 --- a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs @@ -66,3 +66,55 @@ pub(crate) fn manager_misc(out: &mut [u8]) -> Result<(), Error> { word(bytes, 0x20, 1); // cpu_flag / reset Ok(()) } + +pub(crate) const EXTENDED_CONTEXT_SIZE: usize = 0x38; +pub(crate) const EXTENDED_NOTIFIER_SIZE: usize = 0x200; + +pub(crate) fn context_extended(output: &mut [u8]) -> Result<(), Error> { + let bytes = output.get_mut(..EXTENDED_CONTEXT_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..2].fill(0xff); + bytes[5] = 1; + bytes[0x26] = 2; + bytes[0x33] = 0xff; + Ok(()) +} + +impl Notifier { + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 { return Err(Error::Address); } + let bytes = output.get_mut(..EXTENDED_NOTIFIER_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&self.threshold.get().to_le_bytes()); + word(bytes, 0x10, 0x50); + word(bytes, NOTIFIER_CONTEXT, self.context); + bytes[0xa8..0xb0].fill(0xff); + word(bytes, 0x114, 1); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn registration_state_is_left_to_firmware() { + let mut bytes = [0xa5; EXTENDED_CONTEXT_SIZE + 8]; + context_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[..6], &[255, 255, 0, 0, 0, 1]); + assert_eq!(bytes[0x26], 2); + assert_eq!(bytes[0x33], 255); + assert_eq!(&bytes[EXTENDED_CONTEXT_SIZE..], &[0xa5; 8]); + } + #[test] fn notifier_has_zero_generation_and_distinct_firmware_storage() { + let notifier = Notifier { threshold: FirmwareVa::new(0xffff_fc21_7000_0000).unwrap(), context: 3 }; + let mut bytes = [0xa5; EXTENDED_NOTIFIER_SIZE + 8]; + notifier.encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[8..12], &[0; 4]); + assert_eq!(&bytes[0xa8..0xb0], &[255; 8]); + assert_eq!(&bytes[0x114..0x118], &[1, 0, 0, 0]); + assert_eq!(&bytes[EXTENDED_NOTIFIER_SIZE..], &[0xa5; 8]); + let mut short = [0xa5; NOTIFIER_SIZE]; + assert!(notifier.encode_extended(&mut short).is_err()); + assert_eq!(short, [0xa5; NOTIFIER_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ffc57c77afcb2..2aac0dfad36ff 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -176,11 +176,12 @@ pub(crate) struct Queue { shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts, /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`). early_tiling:bool, + wide_visibility:bool, urgency:u8, } impl Queue { pub(crate) fn new(shared:Shared,scheduler:Arc>,vm:mmu::Vm,priority:u32,usc:u64, - early_tiling:bool)->Result { + early_tiling:bool, wide_visibility:bool)->Result { agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?; // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME. let urgency=3u8.saturating_sub(priority.min(3) as u8); @@ -188,7 +189,7 @@ impl Queue { 1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)}; Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler, shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?, - early_tiling,urgency}) + early_tiling,wide_visibility,urgency}) } } impl queue::Queue for Queue { @@ -216,7 +217,8 @@ impl queue::Queue for Queue { let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);}; // Other engines and helpers need their own M3 firmware translation. let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32; - if payload.flags & !wide !=0 || payload.sampler_count!=0 || payload.sampler_heap!=0 { + if payload.flags & !wide !=0 || (!self.wide_visibility && payload.flags & wide != 0) + || payload.sampler_count!=0 || payload.sampler_heap!=0 { pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle); return Err(ENOTSUPP); } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..2651d0aca3198 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1155,8 +1155,7 @@ impl VmBind { self.1 } - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] + /// Stable firmware wire incarnation for this active root lease; zero is unbound. pub(crate) fn generation(&self) -> u8 { self.0.binding.lock().generation } @@ -2703,7 +2702,6 @@ impl Vm { } /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn mark_binding_published(&self) -> Result { let uat = self.gpuvm.lock_inner().uat_inner.clone(); let binding = self.binding.lock(); diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index abac06668c0a1..458603366c954 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -36,3 +36,13 @@ mod m3_compute_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_channel_format.rs"] mod m3_channel_format; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs"] +mod m3_compute_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_state_layout.rs"] +mod m3_state_layout; + + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] +mod m3_compute_storage; From c5e1b35de06e10f3587132f93ea51bac621c6510 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:41 -0400 Subject: [PATCH 035/392] drm/asahi: dispatch G15 compute through extended T8122 firmware Dispatch extended firmware compute records through the existing M3 runtime while preserving G15 CDM streams. Retain owned pool backing and leave firmware-managed register setup to the extended interface. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 162 +++++++++++++++++-------- drivers/gpu/drm/asahi/m3/m3_config.rs | 10 ++ drivers/gpu/drm/asahi/m3/m3_device.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 18 ++- 4 files changed, 132 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 34aa0c0d2cd3f..577510b7e9ecf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -40,25 +40,28 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + profile: super::m3_profile::Profile, /// Last: drops after the command aliases. _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, stats: Region, control: crate::m3::agx_compute::Control, - registers: command::RegisterSet) -> Result { + registers: command::RegisterSet, profile: super::m3_profile::Profile) -> Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; let mut command_bytes = KVec::new(); - command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + command_bytes.resize(if profile == super::m3_profile::Profile::Extended { + command::EXTENDED_SIZE + } else { command::SIZE }, 0u8, GFP_KERNEL)?; let mut sequence_bytes = KVec::new(); sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; // A private compute aperture, disjoint from the retained render graph. // Address-bearing graph fields are constructed from their live owners. for index in 0..storage::COUNT { - let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let a=storage::batch_allocation_for(profile,index).map_err(|_|EINVAL)?; let mut b=match a.mapping { Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, @@ -68,10 +71,13 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; - Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; - UMA_POOL.initialize(&mut objects)?; - Self::initialize_state(&mut objects, binding.slot())?; + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + registers, profile, binding.generation())?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; + if profile == super::m3_profile::Profile::Extended { + UMA_POOL.initialize_extended(&mut objects, 1, true)?; + } else { UMA_POOL.initialize(&mut objects)?; } + Self::initialize_state(&mut objects, binding.slot(), profile)?; // Queue references are firmware VAs from live allocations. In contrast // to render's unpublished seed entries, compute dispatches entry zero // immediately, so its initial producer cursor remains one. @@ -80,8 +86,13 @@ impl Compute { job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; let command = address(storage::COMMAND)?; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes = KVec::new(); + bytes.resize(if profile == super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else { queue::INFO_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + info.encode_extended(address(storage::QUEUE)?, &mut bytes).map_err(|_| EINVAL)?; + } else { info.encode(&mut bytes).map_err(|_| EINVAL)?; } objects[storage::QUEUE].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } @@ -107,54 +118,74 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, - register_set: command::RegisterSet) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); let preemption = &objects[PREEMPTION]; let value = command::Command { context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, - preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation_for(profile,PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, - microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: + if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } + else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, stamp_value: timeline::stamp(0xc1000000,sequence), timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, }; - value.encode(bytes).map_err(|_| EINVAL)?; + value.encode_with_usc(profile, generation, 0, + if profile == super::m3_profile::Profile::Extended { control.usc_base } else { 0 }, bytes).map_err(|_| EINVAL)?; objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) } fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, - context: u32, stamp_value: u32, slot: usize) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + context: u32, stamp_value: u32, slot: usize, profile: super::m3_profile::Profile, + queue_sequence: u64) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); - let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation_for(profile,owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); let value = microsequence::Sequence { command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, - queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + queue: fw(storage::QUEUE)?, context, + generation: if profile == super::m3_profile::Profile::Extended { 0 } else { 1 }, pool: fw(storage::POOL)?, scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], fw_stamp: fw(storage::FW_STAMP)?, stamp_value, }; - value.encode(bytes).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + let auxiliary = command::GpuVa::new(objects[TAIL_SCRATCH].gpu_va()? + offset(TAIL_SCRATCH)? as u64).map_err(|_| EINVAL)?; + let context_store = GpuRegion::new(objects[MICROSEQUENCE_SCRATCH[1]].gpu_va()? + + offset(MICROSEQUENCE_SCRATCH[1])? as u64, 0x40000).map_err(|_| EINVAL)?; + value.encode_extended(auxiliary, context_store, queue_sequence, bytes).map_err(|_| EINVAL)?; + } else { value.encode(bytes).map_err(|_| EINVAL)?; } objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) } - fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + fn initialize_state(objects: &mut [Buffer], context_slot: u32, profile: super::m3_profile::Profile) -> Result { let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_| EINVAL)?; + } else { state::context(&mut context).map_err(|_| EINVAL)?; } + let context_size = if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_CONTEXT_SIZE + } else { state::CONTEXT_SIZE }; + objects[GPU_CONTEXT].write(0, &context[..context_size])?; + let mut notifier = KVec::new(); + notifier.resize(if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_NOTIFIER_SIZE + } else { state::NOTIFIER_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + notification.encode_extended(&mut notifier).map_err(|_| EINVAL)?; + } else { notification.encode(&mut notifier).map_err(|_| EINVAL)?; } objects[NOTIFIER].write(0, ¬ifier)?; let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; @@ -182,7 +213,7 @@ impl Compute { let binding = uat.bind(vm)?; let mut views = KVec::new(); for i in 0..storage::COUNT { - if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + if matches!(storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } @@ -221,18 +252,17 @@ impl Compute { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { - let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; - let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + let offset=storage::slot_offset_for(self.profile,i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.size; self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); - Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; - let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; + self.write_counters()?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; self.batch_count+=1; @@ -245,39 +275,55 @@ impl Compute { pub(crate) fn abort_batch(&mut self)->Result { (self.sequence,self.head,self.batch_count)=self.checkpoint; self.slot=self.batch_count.saturating_sub(1); - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.write_counters()?; self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; agx_memory::publish();Ok(()) } + fn write_counters(&mut self) -> Result { + for index in [storage::EVENT, storage::COUNTER] { + if self.profile == super::m3_profile::Profile::Extended { + self.objects[index].u64(0, self.sequence)?; + } else { self.objects[index].u32(0, self.sequence as u32)?; } + } + Ok(()) + } + fn offset(&self,owner:usize)->Result { - storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + storage::slot_offset_for(self.profile,owner,self.slot).map_err(|_|EINVAL) } /// Same cache-generation contract as M4: StartCompute and the persistent /// notifier must use the same tag before the firmware flushes user stamps. /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + if coalesce && self.profile == super::m3_profile::Profile::Extended { return Err(EINVAL); } let sequence=self.offset(storage::MICROSEQUENCE)?; let command=self.offset(storage::COMMAND)?; self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; - self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + self.objects[storage::COMMAND].u32(command+if self.profile == super::m3_profile::Profile::Extended { 0x81c } else { command::FLUSH_STAMPS },u32::from(!coalesce))?; agx_memory::publish();Ok(()) } pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { - let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + let offset=self.offset(storage::COMMAND)?+if self.profile == super::m3_profile::Profile::Extended { + 0x844 + } else { command::USER_TIMESTAMPS }; self.objects[storage::COMMAND].u64(offset,addresses[0])?; self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; let sequence=self.offset(storage::MICROSEQUENCE)?; - for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + for start in if self.profile == super::m3_profile::Profile::Extended { [0x1bc, 0x20c] } + else { [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] } { self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; } agx_memory::publish();Ok(()) } pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_ATTACHMENTS + } else { microsequence::ATTACHMENTS },&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_HAS_ATTACHMENTS + } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } @@ -319,26 +365,31 @@ impl Compute { agx_memory::publish();crate::mem::sync();Ok(()) } pub(crate) fn gpu_ns(&mut self)->Result { - let mut total=0; - for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + let mut total: u64 = 0; + for slot in 0..self.batch_count { + total = total.checked_add(self.slot_gpu_ns(slot)?).ok_or(EOVERFLOW)?; + } Ok(total) } pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { if slot>=self.batch_count {return Err(EINVAL);} - let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; - let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + if start == 0 || end < start { return Ok(0); } + self.profile.gpu_duration_ns(end - start).ok_or(EOVERFLOW) } /// A batch span includes gaps between its compute commands, unlike gpu_ns. /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} - pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn first(&self) -> bool { + self.profile == super::m3_profile::Profile::Legacy && self.sequence == self.batch_count as u64 + } pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. @@ -361,10 +412,15 @@ impl Compute { pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} pub(crate) fn complete(&mut self)->Result { - Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) - && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + let stamped = self.objects[storage::STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)? == u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)? == u32::from(self.head); + if !stamped { return Ok(false); } + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[storage::POOL].read_u64(0x54)? == self.sequence); + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index bfb408b07a530..ffd1bb963b905 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -177,6 +177,16 @@ impl Config { Ok(()) } + pub(crate) fn global_table_views(&mut self, vm: &mmu::Vm) -> Result> { + let mut mappings = KVec::new(); + if self.profile == super::m3_profile::Profile::Extended { + for index in [super::m3_init_26::GLOBAL_POOL, super::m3_init_26::GLOBAL_PARAMETERS] { + mappings.push(self.objects[index].map_gpu_view(vm)?, GFP_KERNEL)?; + } + } + Ok(mappings) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 05a3961369a6d..474a8b3f561ec 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -129,6 +129,7 @@ impl Device { } e.sgx_setup() } + None if device.firmware.profile() == super::m3_profile::Profile::Extended => None, None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { @@ -179,7 +180,6 @@ impl Device { if self.firmware.profile() == super::m3_profile::Profile::Extended { // Adaptive clock gating can leave individual engine banks inaccessible // even while the GPU performance-state register says powered up. - dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); return Ok(()); } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 40399a80a01f3..058c6610399ae 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -389,7 +389,7 @@ impl Inner { let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -469,6 +469,9 @@ impl Inner { crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; + if self.device.firmware().profile() == super::m3_profile::Profile::Extended { + packet.vm.mark_binding_published().map_err(early)?; + } let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; self.state.health.set_gpu_pending(true); self.gpu_pending = true; @@ -535,7 +538,7 @@ impl Inner { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, - self.config.stats_region()?,control,self.device.soc().registers)?; + self.config.stats_region()?,control,self.device.soc().registers,self.device.firmware().profile())?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; @@ -1059,7 +1062,7 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources,device.firmware().profile())?); let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. @@ -1103,7 +1106,8 @@ impl Runtime { // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; - vm.install_driver_mappings(KVec::new(),reserved)?; + let mappings = self.inner.config.global_table_views(&vm)?; + vm.install_driver_mappings(mappings,reserved)?; Ok(vm) } pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { @@ -1369,13 +1373,15 @@ pub(crate) fn physical_counter() -> u64 { /// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the /// G15 backend does it on 7.2 (`g15::uat`). fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, - resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + resources: &crate::m3::agx_resources::Resources, profile: super::m3_profile::Profile) + -> Result<(mmu::Uat, crate::g15::uat::Window)> { let cfg = soc.hwcfg.ok_or(ENODEV)?; let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; - let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let uat = mmu::Uat::new_adopted_with_user_upper(drm, geometry, ®ions, + profile == super::m3_profile::Profile::Legacy)?; let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; window.bootstrap(drm.as_ref().as_ref(), &uat)?; // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be From ec30b227fefa70ee9bb6b962bceb2bf03495d791 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:42 -0400 Subject: [PATCH 036/392] drm/asahi: enable J613 extended firmware render and retirement Encode extended tiler and fragment work, scene records, independent notifications and attachment visibility through the existing runtime. Retire native pools and enable only the exact J613 26.6.2 profile. Keep the public UAPI and physical G15 stream formats unchanged. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_board.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_config.rs | 12 + drivers/gpu/drm/asahi/m3/m3_drm.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 10 +- .../gpu/drm/asahi/m3/m3_fragment_command.rs | 138 ++++++++++-- .../gpu/drm/asahi/m3/m3_parameter_layout.rs | 74 +++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_pass_layout.rs | 36 +++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 21 ++ drivers/gpu/drm/asahi/m3/m3_render.rs | 206 +++++++++++++----- .../gpu/drm/asahi/m3/m3_render_sequence.rs | 168 ++++++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_scene_layout.rs | 40 ++++ drivers/gpu/drm/asahi/m3/m3_shared_layout.rs | 26 +++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 7 +- drivers/gpu/drm/asahi/m3/m3_sync_layout.rs | 55 +++++ drivers/gpu/drm/asahi/m3/m3_tiler_command.rs | 131 +++++++++-- drivers/gpu/drm/asahi/m3/t8122_admission.rs | 40 +++- tools/asahi/m3-runtime-tests.rs | 60 +++++ 20 files changed, 953 insertions(+), 101 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index c44b2a42d99d5..5d4ea4277b618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -13,7 +13,7 @@ use crate::m3::m3_resources::{ use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. -fn board_is(compatible: &[u8]) -> bool { +pub(crate) fn board_is(compatible: &[u8]) -> bool { let Some(root) = kernel::of::root() else { return false; }; diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 577510b7e9ecf..b6e9444ed5449 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -350,10 +350,12 @@ impl Compute { let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; - crate::m3::m3_sync_layout::Barrier { + let barrier=crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, - }.encode(&mut bytes).map_err(|_|EINVAL)?; + }; + if self.profile==super::m3_profile::Profile::Extended {barrier.encode_extended(&mut bytes)} + else {barrier.encode(&mut bytes)}.map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; @@ -382,8 +384,8 @@ impl Compute { /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.profile.profiler_ticks(self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?), + self.profile.profiler_ticks(self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?)]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index ffd1bb963b905..02b847e210dc9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -246,6 +246,18 @@ impl Config { self.thermal.busy(now); Ok(()) } + pub(crate) fn render_stats(&self) -> Result<[init::Region;2]> { + if self.profile==super::m3_profile::Profile::Legacy { + let value=self.stats_region()?;return Ok([value;2]); + } + let region=|index:usize,offset:usize| { + let object=&self.objects[index]; + init::Region::new(object.va().checked_add(offset as u64).ok_or(EOVERFLOW)?, + object.size().checked_sub(offset).ok_or(EINVAL)?).map_err(|_|EINVAL) + }; + Ok([region(UNKNOWN_C0,4)?,region(UNKNOWN_C1,8)?]) + } + pub(crate) fn stats_region(&self) -> Result { let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { UNKNOWN_C3 diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index e1db1adf9e6a9..8aedd920f31dc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -113,10 +113,12 @@ impl Backend { crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 - // needs a complete table, which it does not have: it is refused here. - let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() { + // Extended T8122 uses its dedicated initialization graph. Keep the + // legacy guarded-start and completeness checks for older interfaces. + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + let experiment = if extended {None} else {crate::m3::t8122_start::arm(pdev.as_ref(), soc)?}; + if experiment.is_none() && !extended { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 6bc87a33aaa05..94a2d3384d08a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,7 +62,7 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; -/// Image record for the extended format. The SoC's runtime selection does not use it yet. +/// Exact normalized image identity for the extended firmware interface. pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { name: "T8122 C0 firmware-compat 26.6.2", uuid: None, @@ -164,7 +164,7 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; -/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +/// T8122 C0, firmware-compat 26.6.2 segment and loader-mutable bounds. pub(crate) static T8122_LAYOUT_26: Layout = Layout { text_size: 393216, data_size: 1228800, @@ -313,7 +313,10 @@ pub(crate) fn identify_loaded( prelude::*, }; - let layout = soc.firmware.ok_or(ENODEV)?; + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + if extended && experiment.is_some() {return Err(EINVAL);} + let layout = if extended {&T8122_LAYOUT_26} else {soc.firmware.ok_or(ENODEV)?}; if !Firmware::matching_layout(layout, &resources) { dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); return Err(ENODEV); @@ -337,6 +340,7 @@ pub(crate) fn identify_loaded( // The T8122 start experiment identifies the same image, with its own InitData version. let images = match experiment { Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None if extended => &T8122_IMAGES_26[..], None => soc.images, }; let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs index 44671ee3cd0ab..e0aaf2584284a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs @@ -3,6 +3,8 @@ use super::{agx_render::Geometry, agx_render_state::{Program,DepthStencil}, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::fragment}; pub(crate) const SIZE:usize=0xc73; +pub(crate) const EXTENDED_SIZE:usize=0xca0; +pub(crate) const EXTENDED_FLUSH:usize=0xbdc; pub(crate) const REGISTERS:usize=fragment::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;4]=[0x160a8,0x160b8,0xa5a9,0xd429]; @@ -74,16 +76,30 @@ impl Command { /// Check references before touching output. The caller owns validation of /// client resource mappings; this layer serializes their semantic state. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len()Result<(),Error> { + self.encode_format(out,Some(dispatch)) + } + fn encode_format(&self,out:&mut[u8],dispatch:Option)->Result<(),Error> { + let extended=dispatch.is_some(); + let size=if extended {EXTENDED_SIZE} else {SIZE}; + if out.len()=1<<42 || self.register_set!=RegisterSet::G15G {return Err(Error::Address);} + } if self.context>=64 {return Err(Error::Address);} - let seq=self.sequence.at(0,fragment::LENGTH)?; + let sequence_size=if extended {0x370} else {fragment::LENGTH}; + let seq=self.sequence.at(0,sequence_size)?; let alias=self.gpu_alias.get(); - GpuVa::new(alias.checked_add((SIZE-3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; + GpuVa::new(alias.checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; let registers=alias+REGISTERS as u64; let s=&self.state;let g=&s.geometry; let layer_metadata=self.heap.get().checked_add(if g.layers>1 {0x200}else{0}).ok_or(Error::Address)?; GpuVa::new(layer_metadata).map_err(|_|Error::Address)?; - let user=super::agx_render_state::compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?) + let user=super::agx_render_state::compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?) .map_err(|_|Error::Address)? & !15; let width=(g.pixels&0xffff)+1;let height=(g.pixels>>16)+1; let tx=width.div_ceil(32);let ty=height.div_ceil(32); @@ -116,43 +132,85 @@ impl Command { (0x101c1,0),(0xd469,0),(0x1a0f9,8),(0x1c838,0),(0x1ca28,user), (0xa5a9,0x6000400020),(0xd429,0x200000001), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); + let bytes=&mut out[..size];bytes.fill(0); word(bytes,0,1); // fragment opcode; work counter remains zero word(bytes,0xc,self.context); for (offset,value) in [(0x14,seq),(0x20,self.notifier.get()),(0x28,self.manager.get()), (0x30,self.scene.get()),(0x38,self.empty.get()),(0x40,self.tilemap.get()),(0x780,registers)] {pointer(bytes,offset,value);} - word(bytes,SEQUENCE_LENGTH,fragment::LENGTH as u32); - // The qualified patch path updates register MSAA but retains these - // producer header values. Representation cleanup must not change that. - pointer(bytes,0x48,0x88);word(bytes,0x50,1); - bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); - bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); - word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); - pointer(bytes,0x78,u64::from(g.tiles)); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); + if extended { + pointer(bytes,4,dispatch.as_ref().map_or(0,|native|native.serial)); + word(bytes,0x50,1); + word(bytes,0x54,g.isp_blocks); + word(bytes,0x60,g.x_blocks); + word(bytes,0x64,g.y_blocks); + word(bytes,0x68,g.blocks); + word(bytes,0x78,g.tiles); + } else { + pointer(bytes,0x48,0x88);word(bytes,0x50,1); + bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); + bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); + word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); + pointer(bytes,0x78,u64::from(g.tiles)); + } let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register);pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit=|register,value| { + word(bytes,REGISTERS+count*REGISTER_STRIDE,register); + pointer(bytes,REGISTERS+count*REGISTER_STRIDE+4,value); + count+=1; + }; + for (register,mut value) in values { + if extended { + if matches!(register,0x160b8|0x120a1|0x1c838|0x1ca28) {continue;} + if register==0x10791 {emit(0xa301,0);} + if register==0x16068 {value=(value&!0x3ffff)|0x3717f;} + if matches!(register,0x15109|0x15101) {value&=!3;} + } else if g15g && G15G_OMIT.contains(®ister) {continue;} + emit(register,value); + } + if let Some(native)=&dispatch { + for register in [0x160e0,0x1499,0xa341] {emit(register,fragment::UUID.into());} + emit(0x1c838,0); + emit(0x1ca28,user); + emit(0x10069,native.usc_window); } bytes[0x788..0x78a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x78a..0x78c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); // Start3DJobParameters3: mirror records use their actual packed widths. for (offset,value) in [(0,s.depth_bias),(0x10,s.scissor),(0x20,s.query), - (0x168,s.partial_background.resources),(0x170,s.partial_background.address), + (0x168,if extended {s.background.resources}else{s.partial_background.resources}), + (0x170,if extended {s.background.address}else{s.partial_background.address}), (0x198,s.partial_background.resources),(0x1a0,s.partial_background.address), (0x1a8,s.zls_control),(0x1b8,s.depth.base),(0x1c0,s.depth.stride), (0x1c8,s.depth_compression.stride),(0x1e0,s.depth_compression.base), (0x1d0,s.depth.base),(0x1d8,s.depth.base), (0x1e8,s.stencil.base),(0x1f0,s.stencil.stride),(0x200,s.stencil.base),(0x208,s.stencil.base), (0x1f8,s.stencil_compression.stride),(0x210,s.stencil_compression.base), - (0x26c,s.partial_eot.address),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { - pointer(bytes,PARAMETERS+offset,value); + (0x26c,if extended {s.eot.address}else{s.partial_eot.address}),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { + pointer(bytes,PARAMETERS+offset+if extended && offset<=0x20 {0x10}else{0},value); } for (offset,value) in [(0x1b0,0x04040404),(0x228,s.mirror_tilebuffer_control), (0x230,s.isp_control),(0x238,width),(0x23c,height),(0x240,0x100000), - (0x248,s.mirror_tile_mode),(0x264,s.partial_eot.resources as u32),(0x284,s.partial_eot.resources as u32), + (0x248,s.mirror_tile_mode),(0x264,if extended {s.eot.resources as u32}else{s.partial_eot.resources as u32}),(0x284,s.partial_eot.resources as u32), (0x298,s.depth_clear),(0x29c,s.stencil_clear),(0x2a0,s.sample_size)] {word(bytes,PARAMETERS+offset,value);} + if let Some(native)=dispatch { + pointer(bytes,0xbc0,self.stamp.get()); + pointer(bytes,0xbc8,self.fw_stamp.get()); + word(bytes,0xbd0,self.stamp_value); + word(bytes,0xbd4,1); + word(bytes,EXTENDED_FLUSH,1); + word(bytes,0xbe0,fragment::UUID); + pointer(bytes,0xc10,self.timestamps[0].get()); + pointer(bytes,0xc18,self.timestamps[1].get()); + pointer(bytes,0xc20,self.user_timestamps[0].get()); + pointer(bytes,0xc28,self.user_timestamps[1].get()); + pointer(bytes,0xc46,self.pool.get()); + bytes[0xc4e]=1; + pointer(bytes,0xc57,native.firmware_aux.get()); + bytes[0xc5f]=native.incarnation; + bytes[0xc84]=1; + } else { word(bytes,0xb88,0x67); // EncoderParams.encoder_id word(bytes,0xb90,u32::MAX); // EncoderParams.unk_mask word(bytes,0xb94,u32::from(s.process_empty_tiles)); @@ -167,6 +225,46 @@ impl Command { pointer(bytes,fragment::COMMAND_TAIL+offset,value); } bytes[fragment::COMMAND_TAIL+0x36]=1;bytes[fragment::COMMAND_TAIL+0x4f]=1; + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] + fn native_fragment_uses_distinct_program_mirrors_and_completion_tail() { + let fw=|n:u64|FirmwareVa::new(0xffff_fc25_3000_0000+n*0x4000).unwrap(); + let gpu=|n:u64|GpuVa::new(0x10_0300_0000+n*0x4000).unwrap(); + let program=|n:u64|Program{address:0x1_0000_0000+n*0x1000,resources:n}; + let surface=DepthStencil{base:0,stride:0}; + let command=Command {context:3,register_set:RegisterSet::G15G, + sequence:Region::new(fw(0).get(),0x1000).unwrap(),notifier:fw(1),manager:fw(2),scene:fw(3),empty:fw(4), + gpu_alias:GpuVa::new(0x72_1000_4000).unwrap(),tilemap:gpu(0),heap:gpu(1),auxiliary:gpu(2),scene_user:gpu(3), + scratch:gpu(4),pool:fw(5),stamp:fw(6),fw_stamp:fw(7),stamp_value:0x3d000100, + timestamps:[fw(8),fw(9)],user_timestamps:[fw(10),fw(11)], + state:State {geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),multisample:0,merge_upper:[0;2], + eot:program(1),background:program(2),partial_eot:program(3),partial_background:program(4), + scissor:0,depth_bias:0,query:0,isp_control:0xc000,tilebuffer_control:1,mirror_tilebuffer_control:1, + tile_mode:0x280,mirror_tile_mode:0x280,depth:surface,stencil:surface,depth_compression:surface, + stencil_compression:surface,zls_control:0,depth_dimensions:0,depth_clear:0,stencil_clear:0, + sample_size:4,process_empty_tiles:false}}; + let dispatch=||super::super::m3_tiler_command::Dispatch26 {incarnation:9,serial:1,peer_stamp:0, + usc_window:1<<32,firmware_aux:fw(12),scene_scratch:gpu(4)}; + let mut bytes=[0xa5;EXTENDED_SIZE+8]; + command.encode_extended(dispatch(),&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + let pointer=|at|u64::from_le_bytes(bytes[at..at+8].try_into().unwrap()); + assert_eq!(word(0xbd0),command.stamp_value); + assert_eq!(word(EXTENDED_FLUSH),1); + assert_eq!(pointer(0x900),command.state.background.address); + assert_eq!(pointer(0x930),command.state.partial_background.address); + assert_eq!(bytes[0xc5f],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let mut short=[0xa5;EXTENDED_SIZE-1]; + assert!(command.encode_extended(dispatch(),&mut short).is_err()); + assert!(short.iter().all(|b|*b==0xa5)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs index 684505615adf3..97c1367b29455 100644 --- a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs @@ -61,3 +61,77 @@ impl Manager { Ok(()) } } + +impl Manager { + pub(crate) const EXTENDED_SIZE: usize = 0xc0; + + /// J613's extended manager has both firmware and GPU page-list aliases. + /// The block ring is read through its canonical firmware mapping. + pub(crate) fn encode_extended(&self, pages_fw: FirmwareVa, blocks_fw: FirmwareVa, + hardware_id: u32, output: &mut [u8]) -> Result<(), Error> { + let capacity = self.block_count.checked_add(1).and_then(u32::checked_next_power_of_two) + .ok_or(Error::Size)?; + if hardware_id >= 32 || capacity as usize > self.blocks.size() / 8 { + return Err(Error::Size); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + let address = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + word(bytes, 8, hardware_id); + word(bytes, 12, hardware_id); + address(bytes, 0x1c, pages_fw.get()); + address(bytes, 0x24, self.pages.base()); + word(bytes, 0x2c, self.pages.size() as u32); + word(bytes, 0x30, self.page_count); + word(bytes, 0x34, capacity); + word(bytes, 0x38, self.block_count); + address(bytes, 0x40, blocks_fw.get()); + address(bytes, 0x48, self.control.get()); + word(bytes, 0x50, self.page_count - 1); + word(bytes, 0x54, BLOCK_SIZE as u32); + address(bytes, 0x60, self.counter.get()); + word(bytes, 0x78, self.page_count); + word(bytes, 0x7c, self.page_count); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Manager { + Manager::new(GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), + FirmwareVa::new(0xffff_fc24_0020_0000).unwrap(), 32).unwrap() + } + #[test] fn native_manager_keeps_distinct_aliases_and_hardware_identity() { + let mut bytes = [0xa5; Manager::EXTENDED_SIZE + 8]; + let pages = FirmwareVa::new(0xffff_fc24_0030_0000).unwrap(); + let blocks = FirmwareVa::new(0xffff_fc24_0040_0000).unwrap(); + fixture().encode_extended(pages, blocks, 7, &mut bytes).unwrap(); + let word = |at| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()); + let pointer = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(word(8), 7); assert_eq!(word(12), 7); + assert_eq!(pointer(0x1c), pages.get()); + assert_eq!(pointer(0x24), 0x5000_0000); + assert_eq!(pointer(0x40), blocks.get()); + assert_eq!(word(0x34), 64); assert_eq!(word(0x38), 32); + assert_eq!(word(0x50), 127); + assert_eq!(pointer(0x80), 0); + assert_eq!(&bytes[Manager::EXTENDED_SIZE..], &[0xa5; 8]); + } + #[test] fn native_manager_rejects_short_storage_and_invalid_identity_before_mutation() { + let fw = FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(); + let mut bytes = [0xa5; Manager::EXTENDED_SIZE]; + assert!(fixture().encode_extended(fw, fw, 32, &mut bytes).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + assert!(fixture().encode_extended(fw, fw, 0, &mut bytes[..MANAGER_SIZE]).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 94c2172e33279..9c0eb070039cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -8,10 +8,10 @@ use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, - slot:usize,clusters:u32,aliases:&mut KVec)->Result { + slot:usize,clusters:u32,profile:super::m3_profile::Profile,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { - let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let a=layout::allocation_for(profile,slot,field,clusters).map_err(|_|EINVAL)?; let mut b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, diff --git a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs index 5ace477a44fff..f88a2a2c9b4a5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs @@ -219,3 +219,39 @@ mod tests { } } } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, slot:usize, field:Field, clusters:u32) + -> Result { + let mut owner = board_allocation(slot,field,clusters)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc25_0000_0000 + (slot * COUNT + field as usize) as u64 * 0x10_0000; + } + match field { + Field::Scene => owner.size = 0x400, + Field::TilerCommand | Field::FragmentCommand => { + let fragment = field == Field::FragmentCommand; + owner.size = if fragment {0xca0} else {0x940}; + owner.gpu_alias = Some(0x72_1000_0000 + slot as u64 * 0x4_0000 + + u64::from(fragment) * 0x4000); + } + Field::TilerToFragment | Field::TilerDependency | Field::FragmentDependency => owner.size = 0x40, + _ => {} + } + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn extended_command_aliases_keep_whole_disjoint_pages() { + for slot in 0..SLOTS { + for field in [Field::TilerCommand,Field::FragmentCommand] { + let owner = allocation_for(super::super::m3_profile::Profile::Extended,slot,field,1).unwrap(); + assert_eq!(owner.address & 0x3fff,0); + assert_eq!(owner.gpu_alias.unwrap() & 0x3fff,0); + assert!(owner.size <= 0x4000); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index 004ac2c7d1eef..04b872ed9ea80 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,15 @@ impl Profile { } } + /// Keep the diagnostic AP/GPU timeline in its established 24 MHz units. + /// User timestamps and duration calculations retain native firmware units. + pub(crate) const fn profiler_ticks(self, timestamp: u64) -> u64 { + match self { + Self::Legacy => timestamp, + Self::Extended => (timestamp / 1000) * 24 + (timestamp % 1000) * 24 / 1000, + } + } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { match self { Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), @@ -49,6 +58,18 @@ impl Profile { mod tests { use super::*; + #[test] + fn profiling_uses_one_clock_domain_without_changing_native_durations() { + for value in [0, 1, 999, 1000, 24_000_000, u64::MAX] { + assert_eq!(Profile::Legacy.profiler_ticks(value), value); + assert_eq!( + Profile::Extended.profiler_ticks(value), + ((value as u128) * 24 / 1000) as u64 + ); + } + assert_eq!(Profile::Extended.profiler_ticks(1_000_000_000), 24_000_000); + } + #[test] fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { for profile in [Profile::Legacy, Profile::Extended] { diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index e7a02982e96f1..9a9f63d15b784 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -26,7 +26,7 @@ pub(crate) struct Render { passes:KVec, init_bm:Buffer, // All active mappings above must drop before their ASID binding. _binding:mmu::VmBind, - stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, @@ -67,37 +67,45 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:[Region;2],clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet,profile:super::m3_profile::Profile)->Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { - let a=s::allocation(index).map_err(|_|EINVAL)?; + let a=s::allocation_for(profile,index).map_err(|_|EINVAL)?; let protection=prot::Prot::from_pte(a.access.pte()); let b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, - Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), a.size,protection,prot::PROT_GPU_SHARED_RW)?, }; + let mut b=b; + if a.gpu_alias.is_some() { aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?; } objects.push(b,GFP_KERNEL)?; } // InitBM is shared, not part of a pass. Preserve its qualified mapping. - let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(if profile==super::m3_profile::Profile::Extended {0xffff_fc24_4000_0000} + else {0xfffffc200062bfe0}), sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; - for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,profile,&mut aliases)?,GFP_KERNEL)?;} let mut sequence_bytes=KVec::new(); sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; let mut tiler_bytes=KVec::new(); - tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); - fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; - for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + for (index,pool) in UMA_POOLS.iter().enumerate() { + if profile==super::m3_profile::Profile::Extended { + pool.initialize_extended(&mut job.objects,33+index as u8,false)?; + } else {pool.initialize(&mut job.objects)?;} + } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); @@ -113,8 +121,14 @@ impl Render { let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; - let mut bytes = [0; parameter::MANAGER_SIZE]; - manager.encode(&mut bytes).map_err(|_| EINVAL)?; + let extended=self.profile==super::m3_profile::Profile::Extended; + let mut bytes = [0; parameter::Manager::EXTENDED_SIZE]; + let length=if extended { + manager.encode_extended(queue::FirmwareVa::new(self.objects[PARAMETER_PAGE_LIST].va()).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(self.objects[PARAMETER_BLOCK_LIST].va()).map_err(|_|EINVAL)?,0,&mut bytes) + .map_err(|_|EINVAL)?; + parameter::Manager::EXTENDED_SIZE + } else {manager.encode(&mut bytes).map_err(|_|EINVAL)?;parameter::MANAGER_SIZE}; for group in 0..PB_GROUPS { let buffer = &self.objects[PB_FIRST + group]; let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) @@ -128,18 +142,30 @@ impl Render { backing.page(page).map_err(|_| EINVAL)?)?; } } - self.objects[BUFFER_MANAGER].write(0, &bytes) + self.objects[BUFFER_MANAGER].write(0, &bytes[..length]) } fn initialize_state(&mut self) -> Result { let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - self.objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; - self.objects[NOTIFIER].write(0, ¬ifier)?; + if self.profile==super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_|EINVAL)?; + self.objects[GPU_CONTEXT].write(0,&context[..state::EXTENDED_CONTEXT_SIZE])?; + for engine in 0..2 { + let value=state::Notifier {threshold:queue::FirmwareVa::new(self.objects[s::EVENT_COUNT].va()+engine as u64*8).map_err(|_|EINVAL)?, + context:self._binding.slot()}; + let mut notifier=[0;state::EXTENDED_NOTIFIER_SIZE]; + value.encode_extended(&mut notifier).map_err(|_|EINVAL)?; + self.objects[NOTIFIER].write(engine*state::EXTENDED_NOTIFIER_SIZE,¬ifier)?; + } + } else { + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + } let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; self.objects[JOB_LIST].write(0, &list)?; @@ -157,6 +183,10 @@ impl Render { state::counter(bytes, value).map_err(|_| EINVAL)?; self.objects[owner].write(0, bytes)?; } + if self.profile==super::m3_profile::Profile::Extended { + self.objects[s::EVENT_COUNT].fill(0)?; + self.objects[s::BM_COUNTER].fill(0)?; + } Ok(()) } fn initialize_queues(&mut self) -> Result { @@ -172,8 +202,13 @@ impl Render { }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes=KVec::new(); + bytes.resize(if self.profile==super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else {queue::INFO_SIZE},0u8,GFP_KERNEL)?; + if self.profile==super::m3_profile::Profile::Extended { + info.encode_extended(address(owner.info)?,&mut bytes).map_err(|_|EINVAL)?; + } else {info.encode(&mut bytes).map_err(|_|EINVAL)?;} self.objects[owner.info].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } @@ -187,9 +222,12 @@ impl Render { Ok(()) } fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { - let mut bytes = [0; sync::BARRIER_SIZE]; - command.encode(&mut bytes).map_err(|_| EINVAL)?; - self.passes[self.slot].get_mut(owner).write(0, &bytes) + let mut bytes=[0;sync::Barrier::EXTENDED_SIZE]; + let length=if self.profile==super::m3_profile::Profile::Extended { + command.encode_extended(&mut bytes).map_err(|_|EINVAL)?; + sync::Barrier::EXTENDED_SIZE + } else {command.encode(&mut bytes).map_err(|_|EINVAL)?;sync::BARRIER_SIZE}; + self.passes[self.slot].get_mut(owner).write(0,&bytes[..length]) } // Construct synchronization from the actual owners after resetting a pass, // before publishing its ring entries. InitBM is shared and dispatched only @@ -201,9 +239,11 @@ impl Render { let ta_stamp = address(s::TA_FW_STAMP)?; let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; let mut init = [0; sync::INIT_BM_SIZE]; - sync::InitBufferManager { context: self._binding.slot(), slot: 0, - block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } - .encode(&mut init).map_err(|_| EINVAL)?; + let value=sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) }; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(&mut init).map_err(|_|EINVAL)?; + } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); @@ -225,7 +265,7 @@ impl Render { } // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. - fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); @@ -243,7 +283,7 @@ impl Render { let value=tiler::Command { context:self._binding.slot(),register_set:self.registers, notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, - scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], @@ -253,7 +293,12 @@ impl Render { vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; - value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, + peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) } fn prepare_sequences(&mut self,draw:u64)->Result { @@ -266,14 +311,21 @@ impl Render { let notifier=&self.objects[NOTIFIER]; let owners=sequence::Owners { command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, - stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + stats:self.stats[usize::from(fs)],notifier:Region::new(notifier.va()+ + if fs && self.profile==super::m3_profile::Profile::Extended {0x200} else {0}, + if self.profile==super::m3_profile::Profile::Extended {0x200} else {notifier.size()}).map_err(|_|EINVAL)?, queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), }; let timed=self.stage_timestamps[usize::from(fs)]; - let length=if fs { + let length=if self.profile==super::m3_profile::Profile::Extended { + let pool=&self.objects[UMA_POOLS[usize::from(fs)].manager]; + let auxiliary=super::m3_compute_layout::GpuVa::new(pool.gpu_va()?+if fs{0x200}else{0x100}).map_err(|_|EINVAL)?; + let serial=draw.checked_mul(2).and_then(|v|v.checked_sub(u64::from(fs))).ok_or(EOVERFLOW)?; + owners.encode_extended(fs,auxiliary,serial,&mut self.sequence_bytes).map_err(|_|EINVAL)? + } else if fs { let b=pass.get(P::FragmentScratch); let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? @@ -290,6 +342,9 @@ impl Render { fn patch_context(&mut self)->Result { let context=self._binding.slot(); self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + if self.profile==super::m3_profile::Profile::Extended { + self.objects[NOTIFIER].u32(0x200+state::NOTIFIER_CONTEXT,context)?; + } Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. @@ -320,7 +375,8 @@ impl Render { let value=fragment::Command { context:self._binding.slot(),register_set:self.registers, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, - notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + notifier:queue::FirmwareVa::new(self.objects[NOTIFIER].va()+ + if self.profile==super::m3_profile::Profile::Extended {0x200} else {0}).map_err(|_|EINVAL)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?}, gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, @@ -341,15 +397,34 @@ impl Render { stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, }, }; - value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).and_then(|v|v.checked_sub(1)).ok_or(EOVERFLOW)?, + peer_stamp:0,usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[1].manager].va()+0x200).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.fragment_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) } + pub(crate) fn set_attachments(&mut self,attachments:&[crate::m3::agx_attachments::Attachments;2])->Result { + // Preserve the legacy renderer's existing packet format. The native + // interface consumes separate attachment apertures for both stages. + if self.profile!=super::m3_profile::Profile::Extended {return Ok(());} + for (fragment,list) in attachments.iter().enumerate() { + let field=if fragment==0 {P::TilerSequence}else{P::FragmentSequence}; + let buffer=self.passes[self.slot].get_mut(field); + sequence::patch_extended_attachments(fragment!=0,list,|offset,data|buffer.write(offset,data))?; + } + crate::m3::agx_memory::publish();Ok(()) + } pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} for (stage,(command,sequence,offset,start,end)) in [ (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) ].into_iter().enumerate() { + let (offset,start,end)=if self.profile==super::m3_profile::Profile::Extended { + if stage==0 {(0x8d8,0x1cc,0x21c)} else {(0xc20,0x1e4,0x234)} + } else {(offset,start,end)}; let pair=addresses[stage]; if !self.stage_timestamps[stage] {continue;} self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; @@ -365,19 +440,19 @@ impl Render { pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + self.profile.gpu_duration_ns(if start!=0 && end>=start {end-start} else {0}).ok_or(EOVERFLOW) } /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { - Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + Ok((self.profile.profiler_ticks(self.passes[base].get_mut(P::TilerStart).read_u64(0)?), + self.profile.profiler_ticks(self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?))) } /// Read retained timestamps only after exact batch retirement. The first /// TA and final fragment delimit the batch in the shared 24 MHz counter. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + let (start,end)=self.gpu_ticks(self.base,self.batch_count)?; + Ok([start,end]) } pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { let mut total=[0;3]; @@ -390,7 +465,7 @@ impl Render { pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { if index>=count {return Err(EINVAL);} let slot=Self::at(base,index); - let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let duration=|a:u64,b:u64| self.profile.gpu_duration_ns(if a!=0 && b>=a {b-a} else {0}).unwrap_or(0); let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; @@ -415,7 +490,9 @@ impl Render { } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} - pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + pub(crate) fn first(&self)->bool { + self.profile==super::m3_profile::Profile::Legacy && self.draw==self.batch_count as u64 + } /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). @@ -447,13 +524,15 @@ impl Render { let binding=uat.bind(vm)?; let mut views=KVec::new();let mut aliases=KVec::new(); for i in 0..s::COUNT { - if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + let a=s::allocation_for(self.profile,i).map_err(|_|EINVAL)?; + if a.gpu_alias.is_some() {aliases.push(self.objects[i].map_gpu_view(vm)?,GFP_KERNEL)?;} + if a.space==Space::ClientGpu { views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } for (slot,pass) in self.passes.iter_mut().enumerate() { for field in pass_layout::FIELDS { - let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + let a=pass_layout::allocation_for(self.profile,slot,field,self.clusters).map_err(|_|EINVAL)?; if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} } @@ -498,12 +577,22 @@ impl Render { self.base=base; Ok(()) } + fn publish_counts(&mut self, draw:u64) -> Result { + if self.profile==super::m3_profile::Profile::Extended { + for offset in [0,8] {self.objects[s::EVENT_COUNT].u64(offset,draw)?;} + for counter in [TA_POOL_COUNTER,FRAGMENT_POOL_COUNTER] {self.objects[counter].u64(0,draw)?;} + } else { + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + } + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. pub(crate) fn abort_batch(&mut self)->Result { (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; - self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + self.publish_counts(self.draw)?; for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } @@ -547,8 +636,20 @@ impl Render { self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; self.batch_count+=1; let draw=self.draw; - self.objects[s::BM_COUNTER].u32(0,draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + self.publish_counts(draw)?; + if self.profile==super::m3_profile::Profile::Extended { + let pass=&self.passes[self.slot]; + let scratch=pass.get(P::FragmentScratch); + let scratch_gpu=GpuRegion::new(scratch.gpu_va()?,scratch.size()).map_err(|_|EINVAL)?; + let scratch_fw=queue::FirmwareVa::new(scratch.va()).map_err(|_|EINVAL)?; + let list=pass.get(P::SceneUser); + let list=GpuRegion::new(list.gpu_va()?,list.size()).map_err(|_|EINVAL)?; + let metadata=pass.get(P::Scene); + let owner=Region::new(metadata.va(),metadata.size()).map_err(|_|EINVAL)?; + let mut bytes=[0;scene::EXTENDED_SIZE]; + scene::encode_extended(&mut bytes,owner,scratch_gpu,scratch_fw,list).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0,&bytes)?; + } else { let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; let scene_list = &mut self.objects[s::BM_SCENES]; if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } @@ -560,7 +661,8 @@ impl Render { let mut bytes = [0; scene::SIZE]; scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; - self.prepare_tiler(r,draw)?; + } + self.prepare_tiler(r,usc,draw)?; self.patch_context()?; self.prepare_fragment(r,usc,draw)?; self.prepare_sequences(draw)?; @@ -655,7 +757,13 @@ impl Render { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} - } Ok(true) + } + if self.profile==super::m3_profile::Profile::Extended && self.draw!=0 { + for pool in UMA_POOLS { + if self.objects[pool.manager].read_u64(0x54)?!=self.draw {return Ok(false);} + } + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter diff --git a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs index f9a0483e77b21..3d421e9906692 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs @@ -184,3 +184,171 @@ impl Fragment { Ok(if timed {LENGTH} else {without_timestamps(bytes,TIMESTAMP_START,FINALIZE,LENGTH,0xa4)}) } } + +impl Owners { + pub(crate) fn encode_extended(&self, fragment:bool, + auxiliary:super::m3_compute_layout::GpuVa, serial:u64, output:&mut[u8]) -> Result { + if self.context>=64 || serial==0 || self.stamp==0 {return Err(Error::Address);} + let command=self.command.at(0,if fragment {0xca0}else{0x940})?; + let sequence=self.sequence.at(0,STORAGE)?; + let activity=self.stats.at(0,8)?; + let scene_tail=self.scene.get().checked_add(0x54).ok_or(Error::Address)?; + let (clock,pair,aux_time,elapsed)=if fragment {(0xc08,0xc10,0xc70,0xb84)}else{(0x8c0,0x8c8,0x928,0x83c)}; + let first=if fragment {0x1e4}else{0x1cc}; + let finalize=if fragment {0x280}else{0x268}; + let pool=if fragment {0x1b0}else{0x190}; + let length=if fragment {0x370}else{0x300}; + let uuid=if fragment {fragment::UUID}else{ta::UUID}; + let bytes=output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes,0,if fragment {7}else{5}); + pointer(bytes,0x14,command+if fragment {0x80}else{0x40}); + pointer(bytes,pool,self.pool.get()); + pointer(bytes,pool+0x18,auxiliary.get()); + pointer(bytes,pool+0x28,serial); + word(bytes,pool+0x30,u32::from(fragment)); + if fragment { + pointer(bytes,0x1c,self.scene.get()); + pointer(bytes,0x24,activity); + pointer(bytes,0x2c,command+0xb70); + pointer(bytes,0x34,scene_tail); + pointer(bytes,0x3c,command+0xbb0); + pointer(bytes,0x44,command+0xbb4); + pointer(bytes,0x4c,self.queue.get()); + pointer(bytes,0x54,command); + word(bytes,0x5c,self.context); + word(bytes,0x60,1); + pointer(bytes,0x7c,command+0xa58); + pointer(bytes,0x84,command+0xbf0); + word(bytes,0xa0,uuid); + } else { + pointer(bytes,0x1c,self.manager.get()); + pointer(bytes,0x24,self.scene.get()); + pointer(bytes,0x2c,activity); + pointer(bytes,0x34,self.queue.get()); + pointer(bytes,0x3c,command+0x85c); + word(bytes,0x44,self.context); + word(bytes,0x48,1); + pointer(bytes,0x64,command+0x784); + pointer(bytes,0x6c,command+0x8a8); + word(bytes,0x7c,uuid); + } + for (at,opcode,selected) in [(first,0x8000_0003,pair),(first+0x50,3,pair+8)] { + word(bytes,at,opcode); + pointer(bytes,at+4,command+clock); + pointer(bytes,at+0xc,command+pair); + pointer(bytes,at+0x14,command+selected); + pointer(bytes,at+0x1c,self.queue.get()); + pointer(bytes,at+0x2c,command+aux_time); + pointer(bytes,at+0x34,command+elapsed); + pointer(bytes,at+0x44,uuid.into()); + } + word(bytes,first+0x4c,1); + word(bytes,finalize,if fragment {8}else{6}); + if fragment { + word(bytes,finalize+4,uuid); + pointer(bytes,finalize+0xc,self.fw_stamp.get()); + word(bytes,finalize+0x14,self.stamp); + pointer(bytes,finalize+0x1c,self.scene.get()); + pointer(bytes,finalize+0x24,self.manager.get()); + word(bytes,finalize+0x2c,1); + pointer(bytes,finalize+0x30,activity); + pointer(bytes,finalize+0x38,command+0xbb0); + pointer(bytes,finalize+0x40,command+0xbb4); + pointer(bytes,finalize+0x48,command+0xb70); + pointer(bytes,finalize+0x50,self.queue.get()); + pointer(bytes,finalize+0x58,command); + word(bytes,finalize+0x60,self.context); + pointer(bytes,finalize+0x64,command+0xa58); + pointer(bytes,finalize+0x98,sequence+pool as u64); + word(bytes,finalize+0xa0,(-(finalize as i32))as u32); + pointer(bytes,finalize+0xa5,command+0xc60); + word(bytes,finalize+0xad,1); + bytes[finalize+0xb1]=1; + // J613 terminates this record after the 16-bit event field. + bytes[finalize+0xb2..finalize+0xb4].copy_from_slice(&1u16.to_le_bytes()); + word(bytes,finalize+0xb4,0x4000_0002); + } else { + pointer(bytes,finalize+4,self.scene.get()); + pointer(bytes,finalize+0xc,self.manager.get()); + pointer(bytes,finalize+0x14,activity); + pointer(bytes,finalize+0x1c,self.queue.get()); + pointer(bytes,finalize+0x24,command+0x85c); + word(bytes,finalize+0x2c,self.context); + pointer(bytes,finalize+0x34,command+0x784); + word(bytes,finalize+0x40,uuid); + pointer(bytes,finalize+0x48,self.fw_stamp.get()); + word(bytes,finalize+0x50,self.stamp); + pointer(bytes,finalize+0x78,sequence+pool as u64); + word(bytes,finalize+0x80,(-(finalize as i32))as u32); + pointer(bytes,finalize+0x85,command+0x918); + bytes[finalize+0x8d]=1; + word(bytes,finalize+0x94,0x4000_0002); + } + Ok(length) + } +} + +#[cfg(test)] +mod native_tests { + use super::*; + #[test] fn j613_fragment_ends_before_the_m4_tail_and_keeps_relative_restart() { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_2000_0000+index*0x4000).unwrap(); + let o=Owners {command:Region::new(fw(0).get(),0xca0).unwrap(),sequence:Region::new(fw(1).get(),STORAGE).unwrap(), + stats:Region::new(fw(2).get(),0x1000).unwrap(),notifier:Region::new(fw(3).get(),0x200).unwrap(), + queue:fw(4),scene:fw(5),manager:fw(6),pool:fw(7),fw_stamp:fw(8),context:3,stamp:0x3d000100}; + let mut output=[0xa5;STORAGE+8]; + let length=o.encode_extended(true,super::super::m3_compute_layout::GpuVa::new(0x72_3000_0200).unwrap(),1,&mut output).unwrap(); + assert_eq!(length,0x370); + let word=|at|u32::from_le_bytes(output[at..at+4].try_into().unwrap()); + assert_eq!(word(0x1e4),0x8000_0003); + assert_eq!(word(0x230),1); + assert_eq!(word(0x234),3); + assert_eq!(word(0x280),8); + assert_eq!(word(0x320),(-0x280i32)as u32); + assert_eq!(word(0x334),0x4000_0002); + assert_eq!(&output[STORAGE..],&[0xa5;8]); + } +} + +/// Native start records reserve one packed attachment list per engine. The +/// matching finalizer flag enables completion visibility for that same list. +pub(crate) const fn extended_attachment_offsets(fragment:bool)->(usize,usize) { + if fragment {(0xa8,0x324)} else {(0x88,0x2ec)} +} + +/// Send both pieces through the same bounded writer used for firmware storage. +#[cfg(any(test, CONFIG_DRM_ASAHI_M3_T8122))] +pub(crate) fn patch_extended_attachments(fragment:bool, + attachments:&super::agx_attachments::Attachments, + mut write:impl FnMut(usize,&[u8])->Result<(),E>)->Result<(),E> { + let (start,flag)=extended_attachment_offsets(fragment); + write(start,&attachments.0)?; + write(flag,&[u8::from(attachments.count()!=0)]) +} + +#[cfg(test)] +mod attachment_tests { + use super::*; + use super::super::agx_attachments::{Attachments,SIZE}; + #[test] fn each_engine_keeps_its_list_and_can_clear_a_previous_list() { + for (fragment,start,flag,pool) in [(false,0x88,0x2ec,0x190),(true,0xa8,0x324,0x1b0)] { + let mut bytes=[0xa5;STORAGE]; + let list=Attachments::new(&[(0x100_0040,0x180),(0x200_0000,0x4000)]).unwrap(); + let mut apply=|list:&Attachments|patch_extended_attachments(fragment,list,|at,data|{ + bytes.get_mut(at..at+data.len()).ok_or(Error::Size)?.copy_from_slice(data);Ok::<(),Error>(()) + }); + apply(&list).unwrap(); + assert_eq!(&bytes[start..start+SIZE],&list.0); + assert_eq!(bytes[flag],1); + assert_eq!(&bytes[pool..pool+8],&[0xa5;8]); + patch_extended_attachments(fragment,&Attachments::EMPTY,|at,data|{ + bytes[at..at+data.len()].copy_from_slice(data);Ok::<(),Error>(()) + }).unwrap(); + assert!(bytes[start..start+SIZE].iter().all(|b|*b==0)); + assert_eq!(bytes[flag],0); + assert_eq!(bytes[start-1],0xa5); + assert_eq!(bytes[start+SIZE],0xa5); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 058c6610399ae..3b1f3d6d08535 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -390,7 +390,7 @@ impl Inner { if found.is_none() { let job=match control { crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), - crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.render_stats().map_err(early)?,self.device.soc().clusters,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { @@ -439,6 +439,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); j.append(command,usc,coalesce,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*next])?; coalesced_stamps+=u64::from(coalesce); } Ok(()) @@ -563,6 +564,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); j.append_after(command,usc,false,predecessor,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*i])?; dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, diff --git a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs index 880cf4c2b9a75..ea3c2064ef39e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs @@ -39,3 +39,43 @@ impl Scene { Ok(()) } } + +pub(crate) const EXTENDED_SIZE: usize = 0x400; + +/// Encode a native scene header from its retained aliases. The local +/// statistics and parameter scratch remain part of this same owned record. +pub(crate) fn encode_extended(output: &mut [u8], owner: super::m3_init_layout::Region, + scratch_gpu: super::m3_pool_layout::GpuRegion, scratch_fw: FirmwareVa, + scene_list: super::m3_pool_layout::GpuRegion) -> Result<(), Error> { + let statistics = owner.at(0x200, 0x40).map_err(|_| Error::Size)?; + owner.at(0, EXTENDED_SIZE).map_err(|_| Error::Size)?; + if scratch_gpu.size() < 0x4000 || scene_list.size() < 0x4000 { return Err(Error::Size); } + let compact = super::agx_render_state::compact(scene_list.base()).map_err(|_| Error::Address)?; + let bytes = output.get_mut(..EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&scratch_gpu.base().to_le_bytes()); + bytes[8..16].copy_from_slice(&scratch_fw.get().to_le_bytes()); + bytes[0x28..0x30].copy_from_slice(&compact.to_le_bytes()); + bytes[0x40..0x48].copy_from_slice(&statistics.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::{m3_init_layout::Region, m3_pool_layout::GpuRegion}; + #[test] fn native_scene_keeps_scratch_aliases_and_local_statistics_distinct() { + let owner = Region::new(0xffff_fc25_0010_0000, EXTENDED_SIZE).unwrap(); + let scratch = GpuRegion::new(0x10_0200_0000, 0x4000).unwrap(); + let scratch_fw = FirmwareVa::new(0xffff_fc25_0020_0000).unwrap(); + let list = GpuRegion::new(0x10_0300_0000, 0x4000).unwrap(); + let mut bytes = [0xa5; EXTENDED_SIZE + 8]; + encode_extended(&mut bytes, owner, scratch, scratch_fw, list).unwrap(); + let read = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(read(0), scratch.base()); + assert_eq!(read(8), scratch_fw.get()); + assert_eq!(read(0x28), 0x300_0000); + assert_eq!(read(0x40), owner.at(0x200, 0x40).unwrap()); + assert_eq!(&bytes[EXTENDED_SIZE..], &[0xa5; 8]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs index 9b8d503836d0a..0a2262858556d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs @@ -100,3 +100,29 @@ pub(crate) fn fault_buffers()->impl Iterator { .chain((PB_FIRST..PB_FIRST+PB_GROUPS).map(|i|(i,256))) .chain([(TILEMAP_LABEL,1280),(HEAP_METADATA_LABEL,256)]) } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc24_0000_0000 + index as u64 * 0x10_0000; + } + match index { + // Extended firmware reads the uncached kernel aliases supplied to + // encode_extended; these lower client mappings are GPU-only. + PARAMETER_PAGE_LIST | PARAMETER_BLOCK_LIST => owner.access = Access::GpuCachedRw, + 8 | 20 => owner.size = 0x20000, + 9 | 21 => { owner.address &= !0x3fff; owner.size = 0x4000; } + 10 | 22 => { + owner.size = 0x4000; + owner.gpu_alias = Some(0x72_3000_0000 + u64::from(index == 22) * 0x4000); + } + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x400, + EVENT_COUNT => owner.size = 16, + BUFFER_MANAGER => owner.size = 0xc0, + FRAGMENT_QUEUE | TA_QUEUE => owner.size = 0x24c0, + _ => {} + } + Ok(owner) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 2aac0dfad36ff..29d2bc7de5f68 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -41,6 +41,7 @@ pub(crate) struct Packet { timestamps:KVec<[Option;4]>, pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, + pub(crate) render_attachments:KVec<[crate::m3::agx_attachments::Attachments;2]>, completion:UserFence, finish_claimed: AtomicBool, /// Set before a batch with any of this packet's commands can change firmware-visible state, @@ -196,7 +197,7 @@ impl queue::Queue for Queue { fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { let vm_job=self.vm.retain_job()?; - let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new();let mut render_attachments=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { @@ -211,6 +212,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + render_attachments.push([encode_attachments(&self.vm,&vertex_attachments)?,encode_attachments(&self.vm,&fragment_attachments)?],GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -226,6 +228,7 @@ impl queue::Queue for Queue { // The common translator validates the original command/attachment // addresses; this M3 scheduling hint changes none of them. payload.flags &= !wide; + render_attachments.push([crate::m3::agx_attachments::Attachments::EMPTY;2],GFP_KERNEL)?; flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; let c=agx_uapi::translate_compute_command(payload,attachments, agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, @@ -239,7 +242,7 @@ impl queue::Queue for Queue { commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; - let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility,render_attachments, completion:self.fences.new_fence(0,Completion)?.into(), finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs index 92d533efa8234..2cf5e18914043 100644 --- a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs @@ -91,3 +91,58 @@ impl Barrier { Ok(()) } } + +impl InitBufferManager { + /// The extended rebuild path requires a zero staged-block count. Appending + /// already-staged blocks would leave a previous hardware descriptor live. + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.slot >= 32 { return Err(Error::Address); } + Self { context: self.context, slot: self.slot, block_count: 0, + manager: self.manager, stamp: self.stamp }.encode(output) + } +} + +impl Barrier { + pub(crate) const EXTENDED_SIZE: usize = 0x40; + + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.event >= 128 || self.wait_value == 0 || self.self_value == 0 { + return Err(Error::Address); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + u32_at(bytes, 0, 4); + address_at(bytes, 4, self.stamp); + address_at(bytes, 12, self.stamp); + u32_at(bytes, 0x14, self.wait_value); + u32_at(bytes, 0x20, self.event); + u32_at(bytes, 0x24, self.self_value); + if matches!(self.dependency, Dependency::PreviousPass) { u32_at(bytes, 0x3c, 1); } + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn full_rebuild_never_uses_the_legacy_staged_count() { + let mut bytes = [0xa5; INIT_BM_SIZE]; + InitBufferManager { context: 3, slot: 7, block_count: 2048, + manager: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), stamp: 0x100 } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[12..20], &[0; 8]); + } + #[test] fn extended_dependencies_use_their_native_tail_and_preserve_trailing_storage() { + for previous in [false, true] { + let mut bytes = [0xa5; Barrier::EXTENDED_SIZE + 8]; + Barrier { stamp: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), wait_value: 0x100, + event: 2, self_value: 0x200, uuid: 9, + dependency: if previous { Dependency::PreviousPass } else { Dependency::TilerToFragment } } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[0x18..0x20], &[0; 8]); + assert_eq!(&bytes[0x20..0x24], &[2, 0, 0, 0]); + assert_eq!(u32::from_le_bytes(bytes[0x3c..0x40].try_into().unwrap()), previous.into()); + assert_eq!(&bytes[Barrier::EXTENDED_SIZE..], &[0xa5; 8]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs index eeda857a3ed88..caa61621e0b04 100644 --- a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs @@ -3,6 +3,17 @@ use super::{agx_render::Geometry, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::ta}; pub(crate) const SIZE:usize=0x92b; +pub(crate) const EXTENDED_SIZE:usize=0x940; +pub(crate) const EXTENDED_FLUSH:usize=0x894; +pub(crate) struct Dispatch26 { + pub(crate) incarnation:u8, + pub(crate) serial:u64, + pub(crate) peer_stamp:u32, + pub(crate) usc_window:u64, + pub(crate) firmware_aux:FirmwareVa, + pub(crate) scene_scratch:GpuVa, +} + pub(crate) const REGISTERS:usize=ta::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;14]=[0x1c041,0x1c9d0,0x1c948,0x1c888,0x1c890,0x1c918,0x1c089,0x1c9e0,0x16c41,0x1ca40,0x1c9a8,0x1c920,0xa5a1,0xd419]; @@ -57,17 +68,32 @@ impl Command { /// All address checks precede mutation. Zero all reserved fields, then /// emit semantic registers and explicit packed command/tail fields. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len() Result<(),Error> { + self.encode_format(out, Some(dispatch)) + } + fn encode_format(&self, out: &mut [u8], dispatch: Option) -> Result<(),Error> { + let extended = dispatch.is_some(); + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len()= 1 << 42 || self.register_set != RegisterSet::G15G { + return Err(Error::Address); + } + } if self.context>=64 || self.page_count==0 || self.tpc_bytes==0 {return Err(Error::Address);} let registers=self.gpu_alias.get().checked_add(REGISTERS as u64).ok_or(Error::Address)?; - GpuVa::new(self.gpu_alias.get().checked_add((SIZE-3) as u64).ok_or(Error::Address)?) + GpuVa::new(self.gpu_alias.get().checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?) .map_err(|_|Error::Address)?; - let seq=self.sequence.at(0,ta::LENGTH)?; + let sequence_size = if extended { 0x300 } else { ta::LENGTH }; + let seq=self.sequence.at(0,sequence_size)?; let tilemap=compact(self.tilemap.get())?; let tpc=compact(self.tpc.get())?; let heap=compact(self.heap.get())?; let preemption=[compact(self.preemption[0].get())?,compact(self.preemption[1].get())?,compact(self.preemption[2].get())?]; - let user=compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)?; + let user=compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)?; let g=&self.geometry; let layers=g.layers; let layer_metadata=compact(self.heap.get().checked_add(if layers>1 {0x200}else{0}).ok_or(Error::Address)?)?; @@ -99,23 +125,67 @@ impl Command { (0x1c830,0),(0x1ca30,user),(0x16c39,user), (0x1c910,0xa0_00000000|(self.initial_scene_entry.get()&0xf_ffffffff)|1), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); // TA opcode is zero - pointer(bytes,4,1); // qualified work-command counter remains one + let bytes=&mut out[..size];bytes.fill(0); // TA opcode is zero + pointer(bytes,4,dispatch.as_ref().map_or(1, |native| native.serial)); word(bytes,CONTEXT,self.context); - for (offset,value) in [(0x14,self.notifier.get()),(0x24,self.manager.get()), - (0x2c,self.scene.get()),(0x34,self.empty.get()),(0x740,registers), + for (offset,value) in [(if extended {0x10} else {0x14},self.notifier.get()), + (if extended {0x20} else {0x24},self.manager.get()), + (if extended {0x28} else {0x2c},self.scene.get()), + (if extended {0x30} else {0x34},self.empty.get()),(0x740,registers), (0x760,self.tpc.get()),(0x768,self.tpc_bytes),(0x770,seq)] {pointer(bytes,offset,value);} let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register); - pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + for (register, mut value) in values { + if g15g && G15G_OMIT.contains(®ister) { continue; } + if extended && matches!(register, 0x12099 | 0x1c898) { continue; } + if let Some(native) = &dispatch { + if register == 0x101e1 { + emit(0xa309, 0); + emit(0x1c8e0, u64::MAX); + emit(0x1c8e8, u64::MAX); + emit(0x1c898, 0); + } + if register == 0x1c910 { + let address = native.scene_scratch.get(); + let escaped = (address & (1 << 42)) >> 3; + let bias = if escaped == 0 { 0x70_0000_0000 } else { 0 }; + value = ((address + bias) & 0x7f_ffff_fffe) | escaped | 1; + } + } + emit(register, value); + } + if let Some(native) = &dispatch { + emit(0xa5a1, 0x60_0040_0020); + emit(0xd419, 0x2_0000_0001); + for register in [0x1ca10, 0x14a1, 0xa349] { emit(register, ta::UUID.into()); } + emit(0x10061, native.usc_window); } bytes[0x748..0x74a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x74a..0x74c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); - word(bytes,SEQUENCE_LENGTH,ta::LENGTH as u32); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); word(bytes,0x77c,1); // fragment event slot + if let Some(native) = dispatch { + word(bytes, STAMP_VALUE, native.peer_stamp); + pointer(bytes, 0x7a8, preemption[0]); + pointer(bytes, 0x878, self.stamp.get()); + pointer(bytes, 0x880, self.fw_stamp.get()); + word(bytes, 0x888, self.stamp_value); + word(bytes, EXTENDED_FLUSH, 1); + word(bytes, 0x898, ta::UUID); + pointer(bytes, 0x8c8, self.timestamps[0].get()); + pointer(bytes, 0x8d0, self.timestamps[1].get()); + pointer(bytes, 0x8d8, self.user_timestamps[0].get()); + pointer(bytes, 0x8e0, self.user_timestamps[1].get()); + pointer(bytes, 0x8fe, self.pool.get()); + bytes[0x906] = 1; + pointer(bytes, 0x90f, native.firmware_aux.get()); + bytes[0x917] = native.incarnation; + } else { word(bytes,STAMP_VALUE,self.stamp_value); pointer(bytes,ta::STRUCT3+0x24,preemption[0]); word(bytes,ta::STRUCT3+0xbc,0x67); // retained producer encoder_id @@ -132,6 +202,41 @@ impl Command { (0x47,self.scratch.get())] {pointer(bytes,ta::COMMAND_TAIL+offset,value);} bytes[ta::COMMAND_TAIL+0x36]=1; // unk_846 bytes[ta::COMMAND_TAIL+0x4f]=1; // unk_85f + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + fn command() -> Command { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_1000_0000+index*0x4000).unwrap(); + let gpu=|index:u64|GpuVa::new(0x10_0100_0000+index*0x4000).unwrap(); + Command {context:3,register_set:RegisterSet::G15G,notifier:fw(0),manager:fw(1),scene:fw(2),empty:fw(3), + gpu_alias:GpuVa::new(0x72_1000_0000).unwrap(),sequence:Region::new(fw(4).get(),0x1000).unwrap(), + pool:fw(5),scratch:fw(6),stamp:fw(7),fw_stamp:fw(8),stamp_value:0x7a000100, + timestamps:[fw(9),fw(10)],user_timestamps:[fw(11),fw(12)],preemption:[gpu(0),gpu(1),gpu(2)], + tilemap:gpu(3),tpc:gpu(4),tpc_bytes:0x4000,heap:gpu(5),scene_user:gpu(6), + initial_scene_entry:fw(13),geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),page_count:128, + vdm:0x100000,multisample:0,ppp:0,process_empty_tiles:false} + } + #[test] fn native_tiler_keeps_peer_completion_and_usc_window_explicit() { + let value=command();let mut bytes=[0xa5;EXTENDED_SIZE+8]; + value.encode_extended(Dispatch26 {incarnation:9,serial:2,peer_stamp:0x3d000100, + usc_window:1<<32,firmware_aux:value.scratch, + scene_scratch:GpuVa::new(0x10_0200_0000).unwrap()},&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + assert_eq!(word(0x780),0x3d000100); + assert_eq!(word(0x888),value.stamp_value); + assert_eq!(word(0x894),1); + assert_eq!(word(0x748)&0xffff,66); + assert_eq!(bytes[0x917],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let registers=&bytes[REGISTERS..REGISTERS+66*REGISTER_STRIDE]; + assert!(!registers.chunks_exact(12).any(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x12099)); + let usc=registers.chunks_exact(12).find(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x10061).unwrap(); + assert_eq!(u64::from_le_bytes(usc[4..].try_into().unwrap()),1<<32); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index fdc41024899f1..fb280ebcde520 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Read-only T8122 identity admission. Firmware startup remains disabled. +//! Read-only T8122 identity and firmware-interface admission. use super::m3_profile::Profile; @@ -53,6 +53,28 @@ pub(crate) fn validate_identity_for( }) } +/// Firmware-interface selection is exact and does not change physical GPU encodings. +pub(crate) fn profile_for(compat:&[u32],j613:bool)->Option { + match compat { + [14,8,3]=>Some(Profile::Legacy), + [26,6,2] if j613=>Some(Profile::Extended), + _=>None, + } +} + +#[cfg(not(test))] +pub(crate) fn loaded_profile(pdev:&kernel::platform::Device>) + ->kernel::error::Result { + use kernel::{c_str,prelude::*}; + let node=pdev.as_ref().of_node().ok_or(ENODEV)?; + let compat:KVec=node.get_property(c_str!("apple,firmware-compat"))?; + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let j613 = super::m3_board::board_is(b"apple,j613"); + #[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] + let j613 = false; + profile_for(&compat, j613).ok_or(ENODEV) +} + /// Shape of the voltage-sorted running-state tables: one entry per distinct millivolt, /// and the highest primary frequency in MHz. Voltages round up, matching `PwrConfig`. /// The first point is the off state. Both primary and secondary frequencies must increase @@ -108,7 +130,7 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt pub(crate) fn read_identity( pdev: &kernel::platform::Device>, ) -> kernel::error::Result { - read_identity_for(pdev, Profile::Legacy) + read_identity_for(pdev, loaded_profile(pdev)?) } #[cfg(not(test))] @@ -300,3 +322,17 @@ mod tests { assert!(validate_identity(&[14, 8, 3], 1, 1, G15G_C0, [0x7f, 0, 0]).is_none()); } } + +#[cfg(test)] +mod selection_tests { + use super::*; + #[test] fn extended_admission_requires_exact_firmware_and_j613() { + for board in [false,true] { + assert_eq!(profile_for(&[14,8,3],board),Some(Profile::Legacy)); + assert_eq!(profile_for(&[26,6,2],board),if board{Some(Profile::Extended)}else{None}); + for wrong in [&[][..],&[26,6][..],&[26,6,1][..],&[26,6,2,0][..],&[14,8,4][..]] { + assert_eq!(profile_for(wrong,board),None); + } + } + } +} diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 458603366c954..91cbefd298fe2 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -46,3 +46,63 @@ mod m3_state_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] mod m3_compute_storage; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs"] +mod m3_parameter_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_sync_layout.rs"] +mod m3_sync_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render.rs"] +mod agx_render; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render_state.rs"] +mod agx_render_state; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_scene_layout.rs"] +mod m3_scene_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_command_layout.rs"] +mod m3_command_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_render_sequence.rs"] +mod m3_render_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_tiler_command.rs"] +mod m3_tiler_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pass_layout.rs"] +mod m3_pass_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_shared_layout.rs"] +mod m3_shared_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_fragment_command.rs"] +mod m3_fragment_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_attachments.rs"] +mod agx_attachments; + +#[test] +fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { + use m3_pass_layout::{Access, Space}; + use m3_profile::Profile; + use m3_shared_layout as shared; + for index in 0..shared::COUNT { + let original = shared::allocation(index).unwrap(); + let legacy = shared::allocation_for(Profile::Legacy, index).unwrap(); + assert_eq!(legacy, original); + let extended = shared::allocation_for(Profile::Extended, index).unwrap(); + if [shared::PARAMETER_PAGE_LIST, shared::PARAMETER_BLOCK_LIST].contains(&index) { + assert_eq!(extended.space, Space::ClientGpu); + assert_eq!(extended.address, original.address); + assert_eq!(extended.size, original.size); + assert_eq!(extended.access, Access::GpuCachedRw); + assert_eq!(extended.access.pte(), 0xc0000000000080); + assert_eq!(original.access, Access::GpuFirmwareCachedRw); + } + if extended.space == Space::ClientGpu { + assert_ne!(extended.access, Access::GpuFirmwareCachedRw); + } + } +} From efcd402f4ccd5ecfb91745a944c328486af3745e Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 16:28:32 -0500 Subject: [PATCH 037/392] drm/asahi: admit authorized HIGH queues on Extended T8122 Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/file.rs | 10 ++++- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +++ tools/asahi/queue-priority-tests.py | 69 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/queue-priority-tests.py diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..a09fa33ed02a4 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1350,9 +1350,15 @@ impl File { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. + // Keep support tied to a backend and admitted profile. G13/G14 and + // unqualified M3 profiles retain their previous rejection. + let high_supported = reg_data.gpu.as_any().is::(); + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let high_supported = high_supported || reg_data.gpu.as_any() + .downcast_ref::() + .is_some_and(|gpu| gpu.supports_high_priority()); if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() + && !high_supported { cls_pr_debug!( Errors, diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 8aedd920f31dc..555edf7074a6b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -96,6 +96,11 @@ pub(crate) struct Backend { } impl Backend { + /// HIGH promises host publication ordering, not firmware preemption. + pub(crate) fn supports_high_priority(&self) -> bool { + self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended + } + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the /// caller registers afterwards. pub(crate) fn start( diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py new file mode 100644 index 0000000000000..5ecce0620588d --- /dev/null +++ b/tools/asahi/queue-priority-tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production queue admission gate with controlled credentials/backends.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[2] +s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() +a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) +b = s.index(' let queues_xa', a) +gate = s[a:b] +predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +unit = r''' +#![allow(dead_code, non_upper_case_globals)] +use std::any::Any; +const EINVAL:i32=22; const EACCES:i32=13; +macro_rules! cls_pr_debug { ($($arg:tt)*) => {} } +mod uapi { + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3; + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; +} +mod bindings { pub const CAP_SYS_NICE:u32=23; + pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} +} +mod g17 {pub struct Gpu;} +mod m3 {pub mod m3_drm { + #[derive(PartialEq)] pub enum Profile {Legacy, Extended} + pub struct Soc {pub chip_id:u32} + pub struct Backend {pub soc:Soc,pub profile:Profile} + impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }} +}} +struct Gpu(Box); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}} +struct Reg {gpu:Gpu} +struct Data {flags:u32,priority:u32} +struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} +fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +fn main() { + use m3::m3_drm::*; + let mut n=0; + for backend in 0..5 { for cap in [false,true] { for master in [false,true] { + let gpu:Box=match backend { + 0=>Box::new(g17::Gpu), + 1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}), + 2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}), + 3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}), + _=>Box::new(())}; + let reg=Reg{gpu:Gpu(gpu)}; + bindings::CAP.with(|c|c.set(cap)); + for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] { + let expected=if flags!=0 || priority>=3 {Err(EINVAL)} + else if priority!=2 {Ok(())} + else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)} + else if cap||master {Ok(())} else {Err(EACCES)}; + assert_eq!(admit(®,&Data{flags,priority},&File(master)),expected, + "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; + }} + }}} + println!("{n} actual admission cases passed"); +} +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: + p = Path(d) / 'gate.rs' + p.write_text(unit) + for enabled in [False, True]: + subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True) + # Without M3 compiled only the G17 backend is admitted. + subprocess.run([str(p.with_suffix(''))], check=True) +print('M3 enabled/disabled gate compilation passed') From c1c0e23c6e4f50e03038a22a6cab7f8d72e39a54 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:25:00 -0500 Subject: [PATCH 038/392] drm/asahi: test the M3 host priority mapping alongside admission Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index 5ecce0620588d..cfbbe8f36d4e6 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -10,6 +10,10 @@ b = s.index(' let queues_xa', a) gate = s[a:b] predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() +ordering = submit.split(' let urgency=', 1)[1].split(' Ok(Self', 1)[0] +ordering = 'let urgency=' + ordering +assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s unit = r''' #![allow(dead_code, non_upper_case_globals)] use std::any::Any; @@ -35,6 +39,13 @@ struct Data {flags:u32,priority:u32} struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +mod sched { + #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel} +} +fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> { + ORDERING + Ok((urgency,priority)) +} fn main() { use m3::m3_drm::*; let mut n=0; @@ -56,9 +67,13 @@ "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; }} }}} - println!("{n} actual admission cases passed"); + for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] { + assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected))); + } + assert_eq!(ordering(4),Err(EINVAL)); + println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed"); } -'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering) with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: p = Path(d) / 'gate.rs' p.write_text(unit) From ba872b99a0039f15a82fa0f91ed7b8ab4e5905c1 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 20:36:32 -0500 Subject: [PATCH 039/392] ci: register the M3 queue priority admission tests Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5dfda83574ea6..4ebce94ccb47d 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'queue-priority-tests.py': [([], PASS)], 'test-m3-power.py': [([], PASS)], 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], From bd92c01562acb283b2138ba90843606a4b924164 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:39:31 -0500 Subject: [PATCH 040/392] drm/asahi: back off interrupt-assisted M3 completion waits Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_completion.rs | 26 +++---- .../gpu/drm/asahi/m3/m3_completion_policy.rs | 77 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 14 ++-- drivers/gpu/drm/asahi/m3/mod.rs | 2 + tools/asahi/m3-fence-latency.py | 63 +++++++++++++++ tools/asahi/m3-runtime-tests.rs | 3 + 7 files changed, 166 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion_policy.rs create mode 100644 tools/asahi/m3-fence-latency.py diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047e..387aaeca1b6c6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 0000000000000..9c0da8c3246e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 555edf7074a6b..04759396c7757 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d92..ba8ad365ce20a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647c..eafd5732bceac 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 0000000000000..c8428d65dbf0d --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 91cbefd298fe2..0d6d4e5841194 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { } } } + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy; From 9f3b31006dd1ac17b751b73d9feceba550675f48 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:59:21 -0500 Subject: [PATCH 041/392] drm/asahi: omit no-op M3 module pins without unload support Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 3b1f3d6d08535..8e25e2c3c602f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1079,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1348,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; return; } From 2d76bcead7cc87a1d487551ca4c25bd65f1a8055 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:54 -0500 Subject: [PATCH 042/392] drm/asahi: adapt the carried T8122 foundation to Aurora 7.3 Retain the newer G16 frontend and binding lifetime, and compose the public T8122 runtime with Aurora. Original dependency patches and their complete metadata are retained in the preceding carried commits. Signed-off-by: Satya Benson --- arch/arm64/configs/asahi.config | 35 +- drivers/gpu/drm/asahi/Kconfig | 39 +- drivers/gpu/drm/asahi/debug.rs | 2 +- drivers/gpu/drm/asahi/driver.rs | 259 ++++-- drivers/gpu/drm/asahi/file.rs | 273 +++++- drivers/gpu/drm/asahi/float.rs | 9 +- drivers/gpu/drm/asahi/g15/mod.rs | 71 +- drivers/gpu/drm/asahi/g15/uat.rs | 8 +- drivers/gpu/drm/asahi/gpu.rs | 123 ++- drivers/gpu/drm/asahi/mmu.rs | 1348 ++++++++++++++++++++++++------ rust/kernel/soc/apple/rtkit.rs | 90 +- 11 files changed, 1787 insertions(+), 470 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index ffecb85c3beb6..2f9a58855b04b 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,9 +8,16 @@ CONFIG_ARCH_APPLE=y # CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +# 16K pages make a page block 32 MiB, and khugepaged then sets min_free_kbytes +# to 352 MiB on an 8 GB Mac. Keep page blocks at 2 MiB as on 4K kernels. +CONFIG_PAGE_BLOCK_MAX_ORDER=7 CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y +# CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE is not set CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m @@ -20,7 +27,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m @@ -31,7 +38,7 @@ CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=m +CONFIG_SPMI_APPLE=y CONFIG_PINCTRL_APPLE_GPIO=m CONFIG_GPIO_MACSMC=m CONFIG_POWER_RESET_MACSMC=m @@ -41,21 +48,30 @@ CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y CONFIG_APPLE_WATCHDOG=y +# m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). +CONFIG_MTD=y +CONFIG_MTD_PHRAM=m +# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m +CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m +CONFIG_VIDEO_APPLE_PRORES=m +CONFIG_VIDEO_APPLE_JPEG=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ASAHI_M3_T8122=y CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m +CONFIG_DRM_APPLE_DCP_T8132=y CONFIG_DRM_APPLE_AUDIO=y CONFIG_DRM_ACCEL=y CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m @@ -77,14 +93,14 @@ CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=m +CONFIG_RTC_DRV_MACSMC=y CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m -CONFIG_APPLE_SIO_T6030=y -CONFIG_MFD_MACSMC=m +CONFIG_MFD_MACSMC=y CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y +CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y @@ -104,16 +120,20 @@ CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m CONFIG_APPLE_PMP_REPORT=y +CONFIG_APPLE_PMP_THERMAL=m +CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m +CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=m +CONFIG_NVMEM_APPLE_SPMI=y CONFIG_MUX_APPLE_DPXBAR=m CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y @@ -125,3 +145,4 @@ CONFIG_RDMA_SIW=m CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y +CONFIG_DETECT_HUNG_TASK=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 45e907b5af86d..109e1985ce559 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Asahi (DRM support for Apple AGX GPUs)" + tristate "Aurora (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -21,8 +21,27 @@ config DRM_ASAHI select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM select RUST_APPLE_RTKIT + select CRYPTO_LIB_SHA256 + help + DRM driver for Apple AGX GPUs, from the M1 family onwards. + + To compile this driver as a module, choose M here: the module will + be called aurora. It keeps "asahi" as a module alias. + +config DRM_ASAHI_G16 + bool "Apple M4 (G16G) support (experimental)" + depends on DRM_ASAHI && ARM64 + depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI + depends on APPLE_CLPC=y || APPLE_CLPC=DRM_ASAHI || APPLE_CLPC=n + depends on THERMAL_OF help - DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + Rust backend for the qualified J713/T8132 GPU firmware. + Load aurora explicitly after the native display and GFX TVM supplier + are ready. Firmware-visible storage remains retained until a verified + retirement or reboot. Keep the working kernel available during bring-up. + A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. + An aborted receipt disables that command class until reboot. + Suspend, hibernation and kexec are blocked while native graphics owns DMA. config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" @@ -36,21 +55,11 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. -config DRM_ASAHI_UAT_ADOPTION - bool - help - The firmware UAT handoff adoption shared by the G15 backend and the - T8122 runtime (g15/uat.rs, g15/regs.rs). - config DRM_ASAHI_M3_T8122 bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" depends on DRM_ASAHI - # The runtime adopts the firmware's UAT as the G15 backend does, without - # needing that backend: M3 Pro/Max keep their own setting. - # Its PMP votes call apple-pmp directly (m3_device.rs). - depends on APPLE_PMP=y || (APPLE_PMP=m && DRM_ASAHI=m) - select DRM_ASAHI_UAT_ADOPTION - select APPLE_T8122_GPU_PMP_LINK + # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. + depends on DRM_ASAHI_G15 select CRYPTO_LIB_SHA256 default n help @@ -68,8 +77,6 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 - select RUST_FW_LOADER_ABSTRACTIONS - select DRM_ASAHI_UAT_ADOPTION default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index e8087442d5320..6161d8d8dd748 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -91,7 +91,7 @@ pub(crate) enum DebugFlags { /// Update the cached global debug flags from the module parameter pub(crate) fn update_debug_flags() { - let flags = *module_parameters::debug_flags.value(); + let flags = module_parameters::debug_flags.value(); DEBUG_FLAGS.store(flags, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d..6227bb4bf815b 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,11 +45,13 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) g16: Option>, m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -83,12 +85,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. @@ -103,7 +111,6 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -143,68 +150,82 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] +// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on +// its entries, so the entries that the native G15 backend owns when it is +// built are passed in by configuration. +macro_rules! asahi_of_device_table { + ($($g15_owned:tt)*) => { + kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + $($g15_owned)* + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] + ); + }; +} + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +asahi_of_device_table!( + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), ); +#[cfg(CONFIG_DRM_ASAHI_G15)] +asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -212,19 +233,16 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { - // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } + this.close_binding(); } /// Device probe function. @@ -233,6 +251,7 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); + crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -244,6 +263,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -260,6 +280,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -291,12 +314,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -304,14 +335,17 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && *module_parameters::stats_export.value() != 0, + stats.is_some() && module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev, raw_stats, stats, @@ -331,6 +365,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -340,10 +377,13 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -362,6 +402,7 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -376,6 +417,9 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { + binding, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -395,6 +439,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -410,20 +457,75 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); + } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev, + raw_stats, + stats, + }) + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl AsahiPlatformDriver { + pub(crate) fn probe_g16<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static crate::g16::Config, + ) -> Result> { + // SAFETY: the qualified UAT output width constrains this device's DMA. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G16Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: data_gpu, + resources, + }); + // SAFETY: registration is stored in bound platform data and never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; + Ok(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: None, + g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -431,9 +533,16 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + self.binding.close(); + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a..24e8db30e580e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + gpu, + memacct, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -122,10 +127,23 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -146,6 +164,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -168,12 +188,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -188,28 +206,52 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], + anchor - in_count as usize, + )?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -218,7 +260,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -233,6 +275,10 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// GPU memory this client created and holds. + mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, @@ -255,12 +301,35 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -270,25 +339,41 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + usage: Arc, + ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + pin_init::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + pin_init::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + pin_init::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; + addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) @@ -341,7 +426,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +445,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -381,13 +469,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; file.inner() @@ -402,15 +490,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +512,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +583,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +674,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -580,7 +688,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - reg_data: &driver::AsahiDrmRegistrationData<'_>, + _reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -589,7 +697,6 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); Ok(0) } } @@ -632,11 +739,48 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); + let gem = gem::new_object( + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, + )?; if reg_data.gpu.prepare_new_objects() { - gem::prepare_new_object(&gem)?; + #[cfg(CONFIG_DRM_ASAHI_G15)] + if reg_data.gpu.as_any().is::() { + gem::prepare_new_object_g15(&gem)?; + } else { + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let registered = device.registration_guard().ok_or(ENODEV)?; + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + { + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } } + let handle = gem.create_handle(file)?; data.handle = handle; @@ -713,11 +857,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +878,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +896,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1344,33 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); + return Err(EINVAL); + } + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } + && !file.is_current_master() + { + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1219,6 +1391,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1260,6 +1433,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1468,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1304,8 +1480,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, @@ -1369,7 +1550,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index a740ad3bb945d..51cac02624d3d 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,6 +44,11 @@ struct F32U { } impl F32 { + /// Return the IEEE-754 binary32 representation. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) @@ -140,12 +145,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() + F32U::from_i64(i64::from(v)).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() + F32U::from_u64(u64::from(v)).pack() } } }; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 324bc08194e53..ae0512de0b156 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -143,22 +144,23 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &hw::t6030::CONFIG, + &&hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &hw::t6031::CONFIG, + &&hw::t6031::CONFIG, ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - MODULE_OF_TABLE_G15, - (), - [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] + @none, + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -177,6 +179,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -211,7 +219,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -222,6 +229,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -372,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -437,7 +446,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -450,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -535,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, @@ -630,7 +654,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_closed(&self, _vm: &mmu::Vm) { vm::closed(self) } @@ -645,9 +669,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } @@ -656,7 +681,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 3855bd0beb9cb..0f91f7c9f0610 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -96,7 +98,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b1..3da16c10696e3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -36,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -85,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -259,18 +261,30 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, - #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>>>, + garbage_contexts: Mutex>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -285,6 +299,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -297,14 +315,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); } if self.queue_limits().is_some() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + if self.soft_faults() && self.supports_soft_faults() { + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -312,35 +330,38 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} - /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS - /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 - /// T8122 runtime admits any client. + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Admit a client's parameter ABI before creating its VM. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -350,6 +371,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -358,13 +386,14 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -406,8 +435,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); + /// Register an unused context, with the objects it retains, for invalidation. + fn free_context(&self, ctx: workqueue::RetiredContext); } /// Private generic trait for functions that don't need to escape this module. @@ -752,6 +781,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -816,6 +846,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -831,6 +867,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -858,6 +895,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1089,7 +1127,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1139,20 +1177,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) @@ -1319,6 +1351,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } @@ -1401,6 +1437,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1418,6 +1455,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) @@ -1427,15 +1465,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } @@ -1460,7 +1498,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { + if self.invalidate_context(&ctx.data).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1469,6 +1507,9 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } + // The firmware no longer knows the context's queues, so their structures may go now, + // ahead of the context data they point to. + core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1737,7 +1778,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: KBox>) { + fn free_context(&self, ctx: workqueue::RetiredContext) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index dc828a460202d..437685bfad1de 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -109,20 +108,19 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +154,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -312,6 +368,7 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, + bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -357,19 +414,54 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { + /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. + /// + /// Map steps update this monotonically under the VM reservation before + /// linking the GPUVA. Remap pieces keep their original sequence. + newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -410,6 +502,9 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, + + /// Sequence assigned by the successful map for its related metadata. + bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -422,6 +517,7 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, + bind_sequence: 0, }) } @@ -452,6 +548,7 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, + bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -470,6 +567,11 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let bind_sequence = self.bind_sequence.fetch_update( + Ordering::AcqRel, + Ordering::Acquire, + |value| value.checked_add(1), + ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -478,22 +580,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -540,6 +636,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -547,12 +644,18 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; + op.vm_bo() + .data() + .newest_bind + .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), + bind_sequence, }, ); + ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -578,6 +681,7 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; + let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -618,7 +722,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -642,9 +747,11 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, ); @@ -659,7 +766,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -694,14 +806,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -711,6 +816,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -765,15 +883,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -801,10 +923,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -812,7 +959,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -870,8 +1024,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -879,9 +1032,11 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, + bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -948,7 +1103,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -956,8 +1111,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -998,6 +1154,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1038,10 +1200,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1084,29 +1243,90 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); + +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { + use kernel::drm::gem::BaseObject; + self.0.owner().size() + } + + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { + return; + } + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.0.owner().timestamp_lock.lock(); + let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let pointer = unsafe { base.add(offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: All eight bytes were checked above; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.0.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); + } +} + +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, + mapping: TimestampVmap, offset: usize, } +#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - pub(crate) fn write(&self, value: u64) { - let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1130,14 +1350,22 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + self.0.owner.lock_inner().page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1148,10 +1376,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1263,13 +1493,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1359,6 +1589,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1401,8 +1644,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1411,6 +1673,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1427,6 +1691,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1435,32 +1704,73 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1477,20 +1787,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1518,7 +1946,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1543,9 +1973,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1625,19 +2055,39 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result> { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1660,12 +2110,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_G15)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1691,6 +2141,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1704,6 +2155,7 @@ impl Vm { } else { None }; + let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -1716,6 +2168,7 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), + bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -1725,11 +2178,14 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, mapping_epoch, + bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -1742,7 +2198,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1753,15 +2212,18 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1773,9 +2235,18 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -1845,9 +2316,7 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -1862,26 +2331,22 @@ impl Vm { if stride <= block_size { return false; } - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return (false, None); - }; + // Every mapping change holds the VM reservation, so readers need only that one. The full + // exec lock would also lock every external BO, making each submit O(external BOs). + let mut inner = self.gpuvm.lock_inner(); let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -1894,6 +2359,10 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } + fn current_bind_sequence(&self) -> u64 { + self.bind_sequence.load(Ordering::Acquire) + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -1929,6 +2398,49 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1943,29 +2455,30 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -1986,7 +2499,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2004,9 +2517,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2024,33 +2544,32 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock(/*Some(&gem)*/ None, false) + .exec_lock_vm(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2085,9 +2604,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2105,14 +2631,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev: &device::Device = device.as_ref().as_ref(); + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2125,19 +2653,13 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2180,6 +2702,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2189,7 +2727,9 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2223,22 +2763,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2280,7 +2833,9 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2311,7 +2866,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2338,6 +2893,66 @@ impl Vm { Ok(()) } + fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { + let _residency = self.enter_retirement_residency(); + // Removing whole mappings only does unmaps, so no preallocated VAs + // are needed for the indexed path. + let mut resources = VmOpResources { + preallocated_gpuvas: [None, None, None], + }; + let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; + + // The BO's newest sequence is checked under the same lock as binds. If + // it is not newer than the close, every mapping belongs to that close + // and the per-object list is safe and substantially cheaper to walk. + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + Err(error) if error == ENOMEM => { + return self.drop_mappings_filtered(gem, sequence); + } + result => result?, + }; + let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { + return Ok(()); + }; + if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { + drop(vm_exec); + drop(bo); + return self.drop_mappings_filtered(gem, sequence); + } + + mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); + let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(vm_exec); + drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_filtered(gem, sequence); + } + result + } + + /// A newer binding shares the BO list, so select the closed generation by + /// walking the VM without allocating. This is also the ENOMEM fallback. + fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_object_where( + gem, + |va| va.data().bind_sequence <= sequence, + VmInner::unmap_gpuva, + ); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { @@ -2384,8 +2999,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2398,9 +3023,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2420,7 +3046,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(CONFIG_DRM_ASAHI_G15)] +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -2431,34 +3057,56 @@ pub(crate) struct AdoptedRegions { } impl Uat { - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // A carveout outside System RAM is mapped write-combined; a reserved-memory region - // inside System RAM (mapped by the kernel) through the kernel's own mapping, as - // `map_region` does for the other generations. - let mode = crate::g15::memory_mode(range.start, size as u64)?; - let ram = mode == kernel::bindings::MEMREMAP_WB as u32; - let flags = if ram { - io::mem::MemFlag::WB + let flags = io::mem::MemFlag::WC; + #[cfg(CONFIG_DRM_ASAHI_G15)] + let flags = if reserved_ram { + // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. + if crate::g15::memory_mode(range.start, size as u64)? + == kernel::bindings::MEMREMAP_WB as u32 + { + io::mem::MemFlag::WB + } else { + flags + } } else { - io::mem::MemFlag::WC + flags }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -2466,7 +3114,7 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -2480,23 +3128,52 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; + Self::new_adopted_inner(dev, geometry, regions, true, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + ) -> Result { + Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn new_adopted_inner( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + reserved_ram: bool, + ) -> Result { + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2505,10 +3182,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2520,8 +3201,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2548,13 +3235,13 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -2618,13 +3305,16 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2653,12 +3343,24 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = *module_parameters::robust_isolation.value() != 0; + let isolation = module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -2666,22 +3368,34 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2690,8 +3404,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2710,6 +3428,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2717,7 +3464,7 @@ impl Uat { return Err(EINVAL); } { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -2761,6 +3508,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2770,14 +3518,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2799,14 +3550,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -2819,11 +3575,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -2835,15 +3596,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2860,24 +3625,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -2887,21 +3661,30 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_G15)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2910,10 +3693,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -2929,12 +3716,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -2942,11 +3744,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -2997,11 +3799,18 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3013,7 +3822,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3022,8 +3831,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3041,3 +3854,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = self.gpuvm.lock_inner(); + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 87ec3bd73658b..325babdecb370 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only //! Support for Apple RTKit coprocessors. //! @@ -222,6 +222,7 @@ impl RtKit { } else { 0 }, + syslog_is_debug: None, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), @@ -230,11 +231,18 @@ impl RtKit { } else { None }, + shmem_setup_ep: None, shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { Some(shmem_destroy_callback::) } else { None }, + protocol_version: 0, + }; + + const COREANALYTICS_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_NEEDS_COREANALYTICS, + ..Self::VTABLE }; /// Creates a new RTKit client for a given device and optional mailbox name or index. @@ -243,6 +251,29 @@ impl RtKit { mbox_name: Option<&'static CStr>, mbox_idx: usize, data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::VTABLE) + } + + /// Creates a client that requires CoreAnalytics setup during boot. + /// + /// The caller must qualify this dependency for the particular firmware + /// instance. This selects immutable operations before mailbox RX starts. + pub fn new_with_coreanalytics( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) + } + + fn new_with_ops( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ops: &'static bindings::apple_rtkit_ops, ) -> Result { let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); let guard = ScopeGuard::new(|| { @@ -260,7 +291,7 @@ impl RtKit { None => ptr::null(), }, mbox_idx.try_into()?, - &Self::VTABLE, + ops, )) }?; @@ -274,6 +305,10 @@ impl RtKit { }) } + /// Borrows the C handle without transferring ownership. + /// The owner must outlive each unsafe C operation and its callbacks. + pub fn as_raw(&self) -> *mut bindings::apple_rtkit { self.rtk } + /// Requests acknowledged RTKit shutdown using the C core's bounded waits. /// /// The caller must not hold a lock needed by its receive callbacks: the @@ -334,6 +369,57 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } + + /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or + /// restores the scheduler's default with `None`. + /// + /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a + /// mailbox without a threaded receive handler. + pub fn set_rx_util_min(&self, util_min: Option) -> Result { + // INVARIANT: `rtk` is valid per the type invariant. + RxUtilMin { rtk: self.rtk }.set(util_min) + } + + /// Returns a handle that sets the receive thread's minimum utilization (see + /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this [`RtKit`] is dropped. + pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { + // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. + RxUtilMin { rtk: self.rtk } + } +} + +/// Sets the minimum utilization of an RTKit coprocessor's receive thread. +/// +/// The request is a single store that the receive thread applies to itself when it next runs, +/// so it may be made from any context. +/// +/// # Invariants +/// +/// `rtk` is valid while the handle is used. +#[derive(Clone, Copy)] +pub struct RxUtilMin { + rtk: *mut bindings::apple_rtkit, +} + +// SAFETY: The request is one store into the mailbox, valid from any thread. +unsafe impl Send for RxUtilMin {} +// SAFETY: As above; the handle has no state of its own. +unsafe impl Sync for RxUtilMin {} + +impl RxUtilMin { + /// See [`RtKit::set_rx_util_min`]. + pub fn set(&self, util_min: Option) -> Result { + let util_min = match util_min { + Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, + None => -1, + }; + // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. + to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) + } } // SAFETY: `RtKit` operations require a mutable reference From 3f4f1a4ffa285145c29c66d428ecaff051449c76 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:03:32 -0500 Subject: [PATCH 043/392] drm/asahi: use the existing F32 bit accessor on 7.3 Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/float.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 51cac02624d3d..782cd47fc7111 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,11 +44,6 @@ struct F32U { } impl F32 { - /// Return the IEEE-754 binary32 representation. - pub(crate) const fn to_bits(self) -> u32 { - self.0 - } - /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) From 40c0f578fa188d83dbe58f0cdd0780aff05696fa Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:13:17 -0500 Subject: [PATCH 044/392] selftests: asahi: match the 7.3 capability argument type Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index cfbbe8f36d4e6..1c5b4695c85f7 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -24,7 +24,7 @@ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; } mod bindings { pub const CAP_SYS_NICE:u32=23; - pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) } thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} } mod g17 {pub struct Gpu;} From 032b6040892ac4e56a3f21bd9966534de45be48c Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:13:18 -0500 Subject: [PATCH 045/392] selftests: asahi: accept explicitly registered test-prefix scripts Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..53f803bddc8e5 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -188,7 +188,7 @@ def jobs(work, conf, dtc): if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) missing = sorted(found - known) - gone = sorted(known - found) + gone = sorted(name for name in known if not (ASAHI / name).is_file()) if missing or gone: for name in missing: print(f'not registered in {Path(__file__).name}: tools/asahi/{name}', file=sys.stderr) From 02eeffd807a1f5787b4bb034fe94bfa9f718c40f Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:17:33 -0500 Subject: [PATCH 046/392] selftests: sep: include the read-only store error in the shim fixture Signed-off-by: Satya Benson --- tools/aurora-sep/tests/test_store_open_errors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/aurora-sep/tests/test_store_open_errors.py b/tools/aurora-sep/tests/test_store_open_errors.py index 7625f1ddffea0..23d99e3e7b6d3 100644 --- a/tools/aurora-sep/tests/test_store_open_errors.py +++ b/tools/aurora-sep/tests/test_store_open_errors.py @@ -140,6 +140,7 @@ def test_native_rust_c_boundary(self): impl Error { pub fn from_errno(n: i32) -> Self { Self(n) } } pub type Result = core::result::Result; pub const EIO: Error = Error(-5); + pub const EROFS: Error = Error(-30); pub const ENODEV: Error = Error(-19); pub fn to_result(n: i32) -> Result { if n < 0 { Err(Error(n)) } else { Ok(()) } } pub unsafe fn from_err_ptr(p: *mut T) -> Result<*mut T> { @@ -148,7 +149,7 @@ def test_native_rust_c_boundary(self): } pub mod prelude { pub use core::ffi::{c_char,c_int,c_void,c_long,c_uint,c_ulong}; - pub use crate::error::{Error, Result, EIO, ENODEV}; + pub use crate::error::{Error, Result, EIO, EROFS, ENODEV}; pub struct CStr(pub &'static [u8]); impl CStr { pub fn as_char_ptr(&self) -> *const c_char { self.0.as_ptr().cast() } } } From 10e8e3a92db108eeb3c540dbfc3187a25002226d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:24:03 -0500 Subject: [PATCH 047/392] drm/asahi: compose the carried timestamp adapter with Aurora 7.3 Keep Aurora current for all other backends. Use the carried public M3 firmware timestamp adapter and accept the new queue usage argument. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/asahi_core.rs | 86 +- drivers/gpu/drm/asahi/driver.rs | 256 ++-- drivers/gpu/drm/asahi/file.rs | 170 ++- drivers/gpu/drm/asahi/g15/command.rs | 43 +- drivers/gpu/drm/asahi/g15/mod.rs | 65 +- drivers/gpu/drm/asahi/g15/uat.rs | 12 +- drivers/gpu/drm/asahi/g17/buffer.rs | 16 + drivers/gpu/drm/asahi/g17/buffer/target.rs | 78 +- drivers/gpu/drm/asahi/g17/capacity.rs | 5 + drivers/gpu/drm/asahi/g17/completion.rs | 208 ++- drivers/gpu/drm/asahi/g17/context.rs | 200 ++- drivers/gpu/drm/asahi/g17/context/identity.rs | 256 +++- drivers/gpu/drm/asahi/g17/dependency.rs | 126 +- drivers/gpu/drm/asahi/g17/event.rs | 8 + drivers/gpu/drm/asahi/g17/feed.rs | 75 +- drivers/gpu/drm/asahi/g17/freelist.rs | 37 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 62 + drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 +- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 108 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 14 + drivers/gpu/drm/asahi/g17/fw/queue.rs | 59 - drivers/gpu/drm/asahi/g17/fw/render.rs | 80 +- drivers/gpu/drm/asahi/g17/fw/variant.rs | 23 +- drivers/gpu/drm/asahi/g17/initdata.rs | 299 ++++- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 44 +- drivers/gpu/drm/asahi/g17/job.rs | 76 +- drivers/gpu/drm/asahi/g17/kick.rs | 170 ++- drivers/gpu/drm/asahi/g17/mod.rs | 429 +++++- drivers/gpu/drm/asahi/g17/object.rs | 98 +- drivers/gpu/drm/asahi/g17/power.rs | 86 +- drivers/gpu/drm/asahi/g17/preparation.rs | 124 +- drivers/gpu/drm/asahi/g17/queue/admission.rs | 373 +++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 906 +++++++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 29 +- .../drm/asahi/g17/queue/compute/retirement.rs | 463 ++++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 177 ++- drivers/gpu/drm/asahi/g17/queue/render.rs | 655 ++++++++- .../gpu/drm/asahi/g17/queue/render/memory.rs | 85 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + .../drm/asahi/g17/queue/render/publication.rs | 10 +- .../drm/asahi/g17/queue/render/retirement.rs | 181 +++ drivers/gpu/drm/asahi/g17/recovery.rs | 200 ++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 49 +- drivers/gpu/drm/asahi/g17/replay.rs | 67 +- drivers/gpu/drm/asahi/g17/runtime.rs | 1185 ++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 94 +- .../drm/asahi/g17/runtime/compute_control.rs | 13 +- .../drm/asahi/g17/runtime/compute_replay.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 149 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 156 ++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 6 +- .../drm/asahi/g17/runtime/render/observe.rs | 118 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 38 +- .../drm/asahi/g17/runtime/render/prepare.rs | 10 +- .../drm/asahi/g17/runtime/render/settle.rs | 14 + .../drm/asahi/g17/runtime/render/submit.rs | 434 +++++- .../drm/asahi/g17/runtime/render/teardown.rs | 48 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 23 +- drivers/gpu/drm/asahi/g17/teardown.rs | 148 ++ drivers/gpu/drm/asahi/g17/timeout.rs | 69 + drivers/gpu/drm/asahi/g17/validation.rs | 174 ++- drivers/gpu/drm/asahi/gem.rs | 71 +- drivers/gpu/drm/asahi/gpu.rs | 70 +- drivers/gpu/drm/asahi/hw/t6040.rs | 76 +- drivers/gpu/drm/asahi/hw/t8140.rs | 6 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 6 +- drivers/gpu/drm/asahi/memacct.rs | 10 +- drivers/gpu/drm/asahi/mmu.rs | 418 ++++-- drivers/gpu/drm/asahi/mmu/bind.rs | 4 +- drivers/gpu/drm/asahi/mmu/m3.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- drivers/gpu/drm/asahi/object.rs | 42 +- drivers/gpu/drm/asahi/pgtable.rs | 62 +- drivers/gpu/drm/asahi/queue/mod.rs | 72 +- drivers/gpu/drm/asahi/regs.rs | 15 +- drivers/gpu/drm/asahi/sysfs.c | 94 +- drivers/gpu/drm/asahi/sysfs.h | 22 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 79 +- drivers/gpu/drm/asahi/timestamp.rs | 45 +- drivers/gpu/drm/asahi/util.rs | 25 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 26 +- include/drm/drm_gem_shmem_helper.h | 4 + include/uapi/drm/asahi_drm.h | 93 +- rust/bindings/bindings_helper.h | 23 + rust/kernel/drm/device.rs | 25 +- rust/kernel/drm/file.rs | 19 +- 88 files changed, 8718 insertions(+), 1824 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 8b782a6f8ffa7..decf16ea09647 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,6 +4,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; +mod binding; mod buffer; mod channel; mod cleanup; @@ -13,20 +14,20 @@ mod event; mod file; mod float; mod fw; -mod gem; -mod g17; +mod fwid; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; -// Without the G15 backend, only the UAT adoption the T8122 runtime shares with it. -#[cfg(all(CONFIG_DRM_ASAHI_UAT_ADOPTION, not(CONFIG_DRM_ASAHI_G15)))] -#[path = "g15/adopt.rs"] -mod g15; +mod g17; +mod gem; + +#[cfg(CONFIG_DRM_ASAHI_G16)] +mod g16; mod gpu; mod hw; mod initdata; +mod m3; mod mem; mod memacct; -mod m3; mod microseq; mod mmu; mod object; @@ -35,8 +36,10 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_raw; mod sysfs_exports; mod timestamp; +mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -48,12 +51,33 @@ use kernel::prelude::*; type G15Registration = kernel::driver::Registration>>; +#[cfg(CONFIG_DRM_ASAHI_G16)] +type G16Registration = kernel::driver::Registration>; +#[cfg(not(CONFIG_DRM_ASAHI_G16))] +type G16Registration = (); + +fn register_g16(module: &'static kernel::ThisModule) -> Result>>> { + #[cfg(CONFIG_DRM_ASAHI_G16)] + { + Ok(Some(KBox::pin_init( + kernel::driver::Registration::new(g16::DRIVER_NAME, module), + GFP_KERNEL, + )?)) + } + #[cfg(not(CONFIG_DRM_ASAHI_G16))] + { + let _ = module; + Ok(None) + } +} + #[pin_data] struct DriverModule { #[cfg(CONFIG_DRM_ASAHI_G15)] #[pin] _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. + _g16: Option>>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -63,15 +87,18 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), + _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, ), + _g16: register_g16(module)?, // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), @@ -81,9 +108,10 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "asahi", + name: "aurora", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", + alias: ["asahi"], params: { debug_flags: u64 { default: 0, @@ -145,22 +173,6 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, - g15_pb_blocks: u32 { - default: 64, - description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", - }, - g15_pb_reclaim: u32 { - default: 2, - description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", - }, - g15_pb_idle_ms: u32 { - default: 1000, - description: "G15: idle time before a PB reclaim, ms (at least 100)", - }, - g15_pb_reclaim_blocks: u32 { - default: 1024, - description: "G15: PB blocks posted above which an idle engine reclaims the PB", - }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", @@ -186,5 +198,29 @@ module! { default: 1, description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, + g16_deadline_ms: u32 { + default: 5000, + description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", + }, + g16_compute_deadline_ms: u32 { + default: 5000, + description: "G16: compute completion-stamp deadline, ms (0 permits unbounded compute; firmware recovery stays active)", + }, + g16_repoll_us: u32 { + default: 15, + description: "G16: repoll interval while publication waits for record consumption, us", + }, + g16_resnap_us: u32 { + default: 100, + description: "G16: completion repoll interval, us", + }, + g16_recovery: u32 { + default: 1, + description: "G16: answer qualified firmware recovery handshakes", + }, + g16_stream_gate: u32 { + default: 1, + description: "G16: validate CDM/VDM stream records", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 9b9b8fc4ccd15..6227bb4bf815b 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,11 +45,13 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) g16: Option>, m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -83,12 +85,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. @@ -142,72 +150,82 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] +// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on +// its entries, so the entries that the native G15 backend owns when it is +// built are passed in by configuration. +macro_rules! asahi_of_device_table { + ($($g15_owned:tt)*) => { + kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + $($g15_owned)* + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] + ); + }; +} + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +asahi_of_device_table!( + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), ); +#[cfg(CONFIG_DRM_ASAHI_G15)] +asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -215,19 +233,16 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { - // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } + this.close_binding(); } /// Device probe function. @@ -248,6 +263,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -264,6 +280,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -295,12 +314,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -308,14 +335,17 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && *module_parameters::stats_export.value() != 0, + stats.is_some() && module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev, raw_stats, stats, @@ -335,6 +365,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -344,10 +377,13 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -366,6 +402,7 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -380,6 +417,9 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { + binding, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -399,6 +439,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -414,9 +457,14 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); @@ -426,13 +474,58 @@ impl AsahiPlatformDriver { if let Err(error) = crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) { - dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev, + raw_stats, + stats, + }) + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl AsahiPlatformDriver { + pub(crate) fn probe_g16<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static crate::g16::Config, + ) -> Result> { + // SAFETY: the qualified UAT output width constrains this device's DMA. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G16Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: data_gpu, + resources, + }); + // SAFETY: registration is stored in bound platform data and never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; + Ok(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: None, + g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -440,8 +533,15 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + self.binding.close(); // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index c627bd9c6b8a0..24e8db30e580e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, - memacct, gpu, + memacct, mmu, queue, util::{ @@ -127,10 +127,23 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -151,6 +164,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -173,12 +188,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -193,28 +206,52 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], + anchor - in_count as usize, + )?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -238,11 +275,13 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// GPU memory this client created and holds. + mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, - /// GPU memory this client created and holds. - mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -263,9 +302,10 @@ impl drm::file::DriverFile for File { let id = gpu.ids().file.next(); let user_range = gpu.user_range(); let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) }) } @@ -279,6 +319,17 @@ impl drm::file::DriverFile for File { }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -292,17 +343,21 @@ impl File { id: u64, user_range: Range, mem: Arc, + usage: Arc, ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_vms, xarray::XArray::>::new(xarray::AllocKind::Alloc1), )?; let raw_queues = addr_of_mut!((*slot).queues); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_queues, xarray::XArray::>>>::new( xarray::AllocKind::Alloc1, @@ -310,15 +365,16 @@ impl File { )?; let raw_objects = addr_of_mut!((*slot).objects); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_objects, xarray::XArray::>::new(xarray::AllocKind::Alloc1), )?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); + addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) } @@ -413,13 +469,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; file.inner() @@ -632,7 +688,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - reg_data: &driver::AsahiDrmRegistrationData<'_>, + _reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -641,7 +697,6 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -685,26 +740,47 @@ impl File { }; let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; // Large G17 user objects are populated in full by the first VM_BIND. // Do not multiply direct reclaim/compaction latency by their folio // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = gem::user_pages_no_reclaim( - size, - reg_data.gpu.as_any().is::(), - ); + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); let gem = gem::new_object( - device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, )?; if reg_data.gpu.prepare_new_objects() { - gem::prepare_new_object(&gem)?; + #[cfg(CONFIG_DRM_ASAHI_G15)] + if reg_data.gpu.as_any().is::() { + gem::prepare_new_object_g15(&gem)?; + } else { + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let registered = device.registration_guard().ok_or(ENODEV)?; + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + { + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } } + let handle = gem.create_handle(file)?; data.handle = handle; @@ -1278,15 +1354,21 @@ impl File { if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH && !reg_data.gpu.as_any().is::() { - cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); return Err(EINVAL); } - // SAFETY: capable() only reads the calling task's credentials (ioctl process context). if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } && !file.is_current_master() { - cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); return Err(EACCES); } @@ -1309,6 +1391,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1397,8 +1480,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index b343aa3fe4e5b..66e52e6981197 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -360,8 +365,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(buffer) = object; - let mapping = buffer.firmware_mapping()?; + let file::Object::TimestampBuffer(mapping) = object; + let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); @@ -377,7 +382,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -407,7 +418,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -467,8 +482,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -552,11 +567,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -650,6 +665,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -658,12 +679,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -678,7 +699,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 48e6afa948c98..ae0512de0b156 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -143,22 +144,23 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &hw::t6030::CONFIG, + &&hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &hw::t6031::CONFIG, + &&hw::t6031::CONFIG, ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - MODULE_OF_TABLE_G15, - (), - [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] + @none, + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -177,6 +179,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -211,7 +219,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -374,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -439,7 +446,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -452,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -537,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, @@ -632,7 +654,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_destroyed(&self) { + fn vm_closed(&self, _vm: &mmu::Vm) { vm::closed(self) } @@ -647,9 +669,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 1d63fb83b9fa1..0f91f7c9f0610 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -36,14 +38,12 @@ use kernel::{ bindings, device, io::Io, prelude::*, + sync::Arc, time::{delay::fsleep, Delta, Instant, Monotonic}, }; -#[cfg(CONFIG_DRM_ASAHI_G15)] -use kernel::sync::Arc; use super::hw; use super::regs::{mb, rmb, wmb, Mapping}; -#[cfg(CONFIG_DRM_ASAHI_G15)] use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{mmu, pgtable}; @@ -69,7 +69,6 @@ const fn indices(va: u64) -> [usize; 3] { } /// Creates the adopted UAT over the ADT regions (step 2 above). -#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn adopt( dev: &AsahiDevice, cfg: &'static super::Config, @@ -202,7 +201,6 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). - #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 8f3a03fbb8dba..fd6460db2d1fb 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -677,6 +677,22 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + /// Read-only diagnostic while firmware is halted. In particular this + /// does not reset the high-water counter as automatic_target() does. + pub(crate) fn report_recovery(&self) { + let read = |offset| -> Result { + Ok(self.graph.word(offset)?.load(Ordering::Relaxed)) + }; + pr_warn!("G17: recovery tvb id={} blocks={} cursors={:?} pages={:?} capacity={:?} producer={:?} consumer={:?} max_pages={:?} reset={:?} initialized={}\n", + self.buffer_id, self.blocks(), self.cursors(), + read(core::mem::offset_of!(fw::State, pages)), + read(core::mem::offset_of!(fw::State, block_capacity)), + read(core::mem::offset_of!(fw::State, producer)), + read(core::mem::offset_of!(fw::State, consumer)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + self.initialized); + } pub(crate) fn state_va(&self) -> u64 { self.graph.gpu_va() } diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index 64c88da8eca14..bd96ecc9d460e 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -29,7 +29,12 @@ fn aligned(value: usize, alignment: usize) -> Result { /// the default parameter-buffer sizes. Sizes are /// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; -const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, GTP6_SIZE, 0x80]; +/// Room for the tiler's records across a partial render. macOS uses 0x1c00, so this is a superset, +/// and the size every partial-render test ran with. +const GTP_SLOTS: usize = 64; +const GTP6_SIZE: usize = GTP_SLOTS + * (((cfg::TVB_MAX_BLOCKS * cfg::TVB_BLOCK_SIZE).div_ceil(0x8000) + 0x17ff) / 24 * 8 + 8); /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] @@ -139,6 +144,16 @@ impl Layout { }) } pub(crate) fn tpc_capacity(&self) -> Result { + // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object + // also contains GTP sub-buffers, notably 64 times the tile map. Applying + // the cache limit to their combined extent rejects valid large targets. + // Keep the original cache and tile-record limits; they also bound the + // derived auxiliary extent before any backing allocation or replacement. + if aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN) > cfg::TPC_MAX + || self.record_stride()? > cfg::TARGET_RECORD_MAX + { + return Err(ERANGE); + } Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { @@ -188,7 +203,7 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + if stride > cfg::TARGET_RECORD_MAX { return Err(ERANGE); } let records = Object::new(dev, vm, stride * DEPTH)?; @@ -272,9 +287,6 @@ impl Target { let stride = layout.record_stride()?; let mut grew = false; if tpc > self.tpc.backing.size() { - if tpc > cfg::TPC_MAX { - return Err(ERANGE); - } Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; grew = true; } @@ -350,3 +362,59 @@ impl Target { } } } + + +#[kernel::macros::kunit_tests(asahi_g17_target_capacity)] +mod tests { + use super::*; + + fn pass(size: u32, utile: u8) -> RenderPass { + // SAFETY: RenderPass and its nested surfaces/programs contain only + // integers and bools, for which the all-zero representation is valid. + let mut pass: RenderPass = unsafe { core::mem::zeroed() }; + pass.width = size; + pass.height = size; + pass.layers = 1; + pass.utile_width = utile; + pass.utile_height = utile; + pass + } + + #[test] + fn t6040_auxiliary_tiler_storage_is_not_tail_pointer_cache() { + // CTS point_size_10000 renders to 10240 squared. Its GTP sub-buffer + // is required by the existing 25G83 program, not a larger cache. + for utile in [16, 32] { + let layout = Layout::new(&pass(10240, utile), 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.tpc_size < cfg::TPC_MAX); + assert!(layout.record_stride().unwrap() < cfg::TARGET_RECORD_MAX); + assert!(layout.tpc_end > cfg::TPC_MAX); + assert!(layout.tpc_capacity().unwrap() >= layout.tpc_end); + assert_eq!(layout.gtp[0], Some(0)); + assert!(layout.gtp[1].unwrap() >= layout.tpc_size); + } + } + + #[test] + fn core_and_tile_record_limits_still_reject_oversized_targets() { + let mut layout = Layout::new(&pass(10240, 32), 2, RegisterPrograms::V25G83).unwrap(); + layout.tpc_size = cfg::TPC_MAX + 1; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + let mut layered = pass(16384, 16); + layered.layers = 2; + let layout = Layout::new(&layered, 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.record_stride().unwrap() > cfg::TARGET_RECORD_MAX); + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } + + #[test] + fn neo_keeps_the_same_cache_extent_and_limit() { + let mut layout = Layout::new(&pass(10240, 32), 1, RegisterPrograms::G17P).unwrap(); + assert!(layout.gtp.iter().all(Option::is_none)); + assert_eq!(layout.tpc_capacity().unwrap(), + aligned(layout.tpc_size, mmu::UAT_PGSZ).unwrap().max(cfg::TPC_MIN)); + layout.tpc_size = cfg::TPC_MAX + 1; + layout.tpc_end = layout.tpc_size; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index bcbbb2e7e875d..57406e5487a37 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,11 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4f5b87bc853dc..90f6d77780bff 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,12 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; +use crate::{ + driver::AsahiDrmDevice, + file, + hw::t8140::{dynamic, resources as cfg}, + mmu, +}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -27,9 +32,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +50,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +66,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,20 +88,15 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) + } + + pub(crate) fn has_outputs(&self) -> bool { + self.0.iter().any(Option::is_some) } fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } @@ -254,6 +255,8 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + usage: Arc, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -267,6 +270,9 @@ pub(crate) struct Completion { /// A recovery blamed this command. Unlike `class`, which a later /// classification can change, this never clears. blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } @@ -286,16 +292,21 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, + usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { fence, member, destinations }; + let output = Output { + fence, + member, + destinations, + }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { @@ -313,6 +324,8 @@ impl Completion { terminal, compute, feed, + usage, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -322,6 +335,7 @@ impl Completion { replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -333,7 +347,9 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal + .as_ref() + .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -357,11 +373,51 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { + self.note_input_error(error); + } + } + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { + sources.check()?; + } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) + self.output + .output() + .destinations + .iter() + .any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -397,6 +453,9 @@ impl Completion { .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -485,6 +544,35 @@ impl Completion { self.blamed.load(Ordering::Acquire) } + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -515,6 +603,13 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { @@ -525,6 +620,9 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -533,7 +631,15 @@ impl Completion { drop(lease); } + /// Whether the command still holds its VM pin: firmware may still own or replay it. + pub(crate) fn retains_vm(&self) -> bool { + self.vm_job.lock().is_some() + } + fn release_healthy(&self) { + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -541,22 +647,40 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { + result = Err(error); + } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } + if let Ok(ticks) = &result { + use crate::usage::Engine; + let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; + if self.compute.is_some() { + self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); + } else { + self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); + self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); + } + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. - if self.spared() { - if error != ECANCELED { - self.status.report_failure(error); - } - } else if error != ECANCELED { - self.status.record(error); - } + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -570,7 +694,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); - if self.accepted.load(Ordering::Acquire) { + // As in complete(): a cancellation is not this client's device loss. A guilty VM + // already carries its own sticky error. + if self.accepted.load(Ordering::Acquire) && error != ECANCELED { self.status.report_failure(error); } self.release_healthy(); @@ -589,9 +715,11 @@ impl Completion { guard.quarantine(); } if self.spared() { - self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c93ab25d55ce3..8a6d9b572dec0 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,8 +14,9 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; +use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use super::job::Engine; use kernel::{ new_mutex, prelude::*, @@ -81,6 +82,19 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } + /// Whether the firmware completed every kick counted so far on node `word`. + fn drained(&self, word: u32) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let completed = self + .object + .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? + .load(Ordering::Relaxed); + fence(Ordering::Acquire); + Ok(completed >= self.counts[slot]) + } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -114,9 +128,24 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// A kill of this context was published, through a compute queue or on its render pairs' + /// behalf. A context is killed at most once. + kill_published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, + /// The domain refused compute publishers wait on; signalled with `state_returns` when the + /// admission turn moves on. + compute_returns: Arc, + /// The order in which this context's ioctls take work-state nodes. + order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, @@ -128,6 +157,10 @@ pub(crate) struct Context { policy: Policy, } impl Context { + #[expect( + clippy::too_many_arguments, + reason = "A context takes its allocators, identities and both admission domains." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -135,6 +168,8 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, + compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -183,15 +218,23 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + let order = + Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, scheduler, scheduler_generation, published: AtomicBool::new(false), + kill_published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), + state_returns, + compute_returns, + order, work, work_base, render, @@ -203,6 +246,22 @@ impl Context { pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } @@ -253,6 +312,28 @@ impl Context { _work: self.work.clone(), } } + /// Called once the firmware consumed this context's scheduler release. If its work list + /// still links a node, the firmware keeps referencing that node: on Neo a render command + /// that ran behind a failed compute command is never unlinked, and a later context given + /// the same work storage (the object cache hands it out again) made the firmware recover + /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether + /// it was linked. + pub(crate) fn retain_linked_work(&self, pool: &Arc) -> bool { + // An unreadable head counts as linked: retaining is the safe side. + let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]); + if linked { + self.work.lock().object.retain_until_stop(pool); + } + linked + } + + /// The firmware may still walk this context's work list into memory it names: a node is + /// linked and the scheduler page shows no kill. Unreadable state counts as linked. + pub(crate) fn firmware_may_link(&self) -> bool { + let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed); + must_retain(self.work_head().ok(), self.work_head_va(), killed) + } + pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } @@ -267,6 +348,16 @@ impl Context { Ok(head) } + /// Record that a kill of this context was published (see `kill_published`). + pub(crate) fn note_kill_published(&self) { + self.kill_published.store(true, Ordering::Release); + } + + /// A kill of this context was published. + pub(crate) fn kill_published(&self) -> bool { + self.kill_published.load(Ordering::Acquire) + } + pub(crate) fn is_published(&self) -> bool { self.published.load(Ordering::Acquire) } @@ -333,41 +424,60 @@ impl Context { pub(crate) struct WorkStateLease { context: Arc, word: AtomicU32, + /// Index of the engine whose admission took the node, valid once `word` is set. + engine: AtomicU8, + /// The ioctl's place in its context's submission order. + ordinal: u64, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { + let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), + engine: AtomicU8::new(0), + ordinal, } } - pub(crate) fn acquire(&self) -> Result { + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many + /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. + pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); if current != FREE { return Ok(current); } - let Some(candidate) = self.context.states.allocate() else { - let current = self.word(); - return if current != FREE { - Ok(current) - } else { - Err(EBUSY) - }; - }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + // The work-storage lock serializes the decision with this context's other + // admissions, and with the other engine of this ioctl taking its node. + let mut work = self.context.work.lock(); + let current = self.word(); + if current != FREE { + return Ok(current); + } + let mut order = self.context.order.lock(); + if !order.admits(self.ordinal) { + return Err(EBUSY); + } + let engine = engine.index(); + let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; + if let Err(error) = work.initialize(candidate) { + self.context.release_state(candidate, engine); return Err(error); } - match self - .word - .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => Ok(candidate), - Err(installed) => { - self.context.states.release(candidate); - Ok(installed) - } + self.engine.store(engine as u8, Ordering::Relaxed); + self.word.store(candidate, Ordering::Release); + let wake = order.admit(self.ordinal); + drop(order); + drop(work); + if wake { + self.context.pass_turn(); } + Ok(candidate) + } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) @@ -379,6 +489,16 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } + /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores + /// the completion stamps of the node's commands only then, so a command whose record is + /// present still waits for its stamp while other commands of its ioctl run. + pub(crate) fn drained(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + self.context.work.lock().drained(word) + } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work @@ -409,21 +529,55 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); + } else if self.context.order.lock().leave(self.ordinal) { + self.context.pass_turn(); } } } +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32, engine: usize) { + if self.states.release(word, engine) { + self.state_returns.notify(); + } + } + /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a + /// refusal means one that sampled before its refused acquire() never sleeps through it. + fn pass_turn(&self) { + self.state_returns.notify(); + self.compute_returns.notify(); + } +} + impl QosIds { pub(crate) fn used(&self) -> u32 { (*self.used.lock() & !3u128).count_ones() } } +/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or +/// an unreadable head) and no acknowledged kill. +fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool { + killed == 0 && head.is_none_or(|head| head != [0, head_va]) +} + #[kunit_tests(asahi_g17_qos_capacity)] mod capacity_tests { use super::*; + #[test] + fn linked_unkilled_work_is_retained() { + let va = 0xfffffc20_0000_4400; + assert!(!must_retain(Some([0, va]), va, 0)); + assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0)); + assert!(must_retain(None, va, 0)); + assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1)); + assert!(!must_retain(None, va, 1)); + } + #[test] fn failed_client_construction_returns_every_qos_id() { let ids = QosIds::new().unwrap(); diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 595c7154d42ed..7586f8474e602 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -4,23 +4,50 @@ //! the complete sequence distinguishes its owner until exact retirement. use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; pub(super) const FREE: u32 = u32::MAX; pub(super) const STATE_COUNT: usize = 63; pub(super) const NODE_STRIDE: usize = 0x100; +/// Engines whose admissions take work-state nodes: render (0) and compute (1). +pub(super) const ENGINES: usize = 2; + +/// Whether an engine that holds `held` nodes may take one of `free` nodes. +/// +/// All engines of one ioctl share its node, and the node returns only once every command of +/// the ioctl has retired. So render admission can keep taking nodes for newer ioctls while +/// their compute commands still wait in the compute queue behind an older ioctl that has no +/// node yet; once render held every node, nothing could retire and both queues stalled. An +/// engine therefore never takes the last free node while it already holds one. The other +/// engine can then always take a node, and an engine refused here waits only for nodes it +/// took itself, for ioctls ahead of its waiting command. Each engine admits a context's +/// ioctls in submission order, so those waits cannot form a cycle. +pub(super) const fn may_take(free: usize, held: u32) -> bool { + free > 1 || (free == 1 && held == 0) +} + pub(super) struct WorkStates { next: AtomicU32, live: [AtomicU32; 64], + /// Nodes taken by each engine's admissions and not yet released. + held: [AtomicU32; ENGINES], } impl WorkStates { pub(super) const fn new() -> Self { Self { next: AtomicU32::new(0), live: [const { AtomicU32::new(FREE) }; 64], + held: [const { AtomicU32::new(0) }; ENGINES], } } - pub(super) fn allocate(&self) -> Option { + /// Takes a node for `engine`, or none when [`may_take`] refuses. Callers serialize + /// allocations of one context; releases may run concurrently. + pub(super) fn allocate(&self, engine: usize) -> Option { + let held = self.held.get(engine)?; + if !may_take(STATE_COUNT.saturating_sub(self.used()), held.load(Ordering::Acquire)) { + return None; + } for _ in 0..64 { let word = self.next.fetch_add(1, Ordering::Relaxed); let index = (word & 63) as usize; @@ -29,17 +56,119 @@ impl WorkStates { .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) .is_ok() { + held.fetch_add(1, Ordering::AcqRel); return Some(word); } } None } - pub(super) fn release(&self, word: u32) -> bool { + pub(super) fn release(&self, word: u32, engine: usize) -> bool { let index = (word & 63) as usize; - index != STATE_COUNT + let released = index != STATE_COUNT && self.live[index] .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) - .is_ok() + .is_ok(); + if released { + if let Some(held) = self.held.get(engine) { + held.fetch_sub(1, Ordering::AcqRel); + } + } + released + } +} + +/// Ordinals past the turn remembered for ioctls that took a node or left early. +pub(super) const ORDER_WINDOW: u64 = 1024; + +/// How far past the oldest waiting ioctl an ioctl may take its node. +pub(super) const RUN_AHEAD: u64 = 8; + +/// The order in which a context's ioctls take work-state nodes. +/// +/// Each engine admits a context's ioctls in submission order, but the engines advance +/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still +/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render +/// ioctls in flight on one queue) the firmware then stops completing all of the context's work +/// within seconds. Native macOS schedules a queue's command buffers in commit order across +/// engines, at most 36 between scheduling and completion, and the same load runs there without +/// failure. +/// +/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have +/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; +/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler +/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that +/// without letting one engine get far ahead of the other. +pub(super) struct Order { + /// The next ordinal to hand out. + next: u64, + /// The oldest ordinal that has neither taken a node nor left. + turn: u64, + /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + done: [u64; (ORDER_WINDOW / 64) as usize], + /// Set once an ioctl left too far past the turn to remember; admission is unordered after. + unordered: bool, + /// An ioctl was refused since the turn last moved: its publisher waits for the next move. + refused: bool, +} +impl Order { + pub(super) const fn new() -> Self { + Self { + next: 0, + turn: 0, + done: [0; (ORDER_WINDOW / 64) as usize], + unordered: false, + refused: false, + } + } + /// The ordinal of a new ioctl, in submission order. + pub(super) fn assign(&mut self) -> u64 { + let ordinal = self.next; + self.next += 1; + ordinal + } + /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the + /// turn next moves, so that move wakes the refused publisher. + pub(super) fn admits(&mut self, ordinal: u64) -> bool { + let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + self.refused |= !admits; + admits + } + /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + pub(super) fn admit(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). + /// Returns whether a refused ioctl may now be admitted. + pub(super) fn leave(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + fn finish(&mut self, ordinal: u64) -> bool { + if self.unordered || ordinal < self.turn { + return false; + } + if ordinal == self.turn { + self.pass(); + return core::mem::take(&mut self.refused); + } + if ordinal - self.turn >= ORDER_WINDOW { + self.unordered = true; + return core::mem::take(&mut self.refused); + } + let bit = ordinal % ORDER_WINDOW; + self.done[(bit / 64) as usize] |= 1 << (bit % 64); + false + } + /// Moves the turn past the current ordinal and every following one already done. + fn pass(&mut self) { + loop { + self.turn += 1; + let bit = self.turn % ORDER_WINDOW; + let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); + if self.done[word] & mask == 0 { + break; + } + self.done[word] &= !mask; + } } } @@ -78,3 +207,122 @@ impl WorkStates { impl RenderSlots { pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } } + +#[kunit_tests(asahi_g17_work_states)] +mod tests { + use super::*; + + #[test] + fn last_node_stays_with_an_engine_holding_none() { + assert!(!may_take(0, 0)); + assert!(may_take(1, 0)); + assert!(!may_take(1, 3)); + assert!(may_take(2, 3)); + } + + #[test] + fn render_cannot_starve_compute_of_a_node() { + let states = WorkStates::new(); + let mut render = KVec::new(); + while let Some(word) = states.allocate(0) { + render.push(word, GFP_KERNEL).unwrap(); + } + // Render stops one short; compute takes the last node, then nothing is left. + assert_eq!(render.len(), STATE_COUNT - 1); + let compute = states.allocate(1).unwrap(); + assert_eq!(states.used(), STATE_COUNT); + assert!(states.allocate(0).is_none()); + assert!(states.allocate(1).is_none()); + // A lone free node goes to the engine holding none. + assert!(states.release(compute, 1)); + assert!(states.allocate(0).is_none()); + let compute = states.allocate(1).unwrap(); + // With both engines holding nodes, a lone free node waits for one of their returns. + assert!(states.release(render.pop().unwrap(), 0)); + assert!(states.allocate(0).is_none()); + assert!(states.allocate(1).is_none()); + // Two free nodes: render may take one again, never the last. + assert!(states.release(render.pop().unwrap(), 0)); + assert!(states.allocate(0).is_some()); + assert!(states.allocate(0).is_none()); + assert!(states.release(compute, 1)); + assert!(states.allocate(1).is_some()); + } + + #[test] + fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + let mut order = Order::new(); + let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); + let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); + // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. + for &ordinal in &ordinals[..=RUN_AHEAD as usize] { + assert!(order.admits(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // Ioctls ahead of the turn taking their nodes do not move it. + for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { + assert!(!order.admit(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // The oldest one moves the turn past everything already admitted, and wakes the refused. + assert!(order.admit(oldest)); + assert!(order.admits(last)); + let next = order.assign(); + assert!(order.admits(next)); + assert!(!order.admits(next + RUN_AHEAD + 1)); + } + + #[test] + fn ioctls_leaving_without_a_node_pass_the_turn() { + let mut order = Order::new(); + let (a, b, c) = (order.assign(), order.assign(), order.assign()); + let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); + let beyond = later[RUN_AHEAD as usize - 1]; + // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + assert!(!order.leave(b)); + assert!(!order.leave(c)); + assert!(!order.admits(beyond)); + // Once a takes its node, the turn passes b and c as well, and the refused one may retry. + assert!(order.admit(a)); + assert!(order.admits(beyond)); + // With nobody refused since, moving the turn wakes nobody; an ordinal already passed + // leaves without effect. + assert!(!order.leave(later[0])); + assert!(!order.leave(a)); + } + + #[test] + fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + let mut order = Order::new(); + let first = order.assign(); + let mut last = first; + for _ in 0..ORDER_WINDOW { + last = order.assign(); + } + assert!(!order.admits(last)); + assert!(order.leave(last)); + assert!(order.admits(last - 1)); + } + + #[test] + fn single_engine_admission_wakes_nobody() { + let mut order = Order::new(); + for _ in 0..3 * RUN_AHEAD { + let ordinal = order.assign(); + assert!(order.admits(ordinal)); + assert!(!order.admit(ordinal)); + } + } + + #[test] + fn releases_return_nodes_to_their_engine() { + let states = WorkStates::new(); + let word = states.allocate(1).unwrap(); + assert!(states.release(word, 1)); + assert!(!states.release(word, 1)); + assert_eq!(states.used(), 0); + assert_eq!(states.held[1].load(Ordering::Acquire), 0); + // An engine outside the table takes nothing. + assert!(states.allocate(ENGINES).is_none()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 0bc1d5024566e..5a0ed611b1dde 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -9,6 +9,7 @@ use super::{ use kernel::prelude::*; /// A failed producer does not establish a fault in its consumer's VM. +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn inherited_error(error: Error) -> Error { if error == EIO || error == ENODEV || error == ETIMEDOUT { ECANCELED @@ -17,14 +18,61 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its +/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with +/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too. +pub(crate) const PRODUCER_FAILED: Error = ECANCELED; + /// A replay's original host inputs must still be signalled. Treat an unmet /// input as a failed replay, never as a wait under the device mutex. pub(crate) fn replay_status(status: i32) -> Result { if status == 0 { Err(EBUSY) } - else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else if status < 0 { Err(PRODUCER_FAILED) } else { Ok(()) } } +/// Whether a failed command may still run or write: it holds its VM pin until it has physically +/// retired, or was never published. +pub(crate) trait Retention { + fn retained(&self) -> bool; +} + +/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix +/// covers one is cancelled, so none runs before a failed producer the firmware may still replay. +/// Once that producer has retired nothing of it can run or write any more, so it stops counting: +/// otherwise one cancelled command cancelled every later command of its queue for good. +pub(crate) struct FailureRecord { + retained: KVec<(u64, T)>, + /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did. + untracked: Option<(u64, T)>, +} + +impl FailureRecord { + pub(crate) const fn new() -> Self { + Self { retained: KVec::new(), untracked: None } + } + + /// A finished frontier entry, at pruning. + pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) { + if status >= 0 || !entry.retained() { + return; + } + if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err() + && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first) + { + self.untracked = Some((sequence, entry.clone())); + } + } + + /// The earliest failure later prefixes must not bypass. + pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> { + self.retained.retain(|(_, entry)| entry.retained()); + Iterator::chain(self.retained.iter(), self.untracked.iter()) + .min_by_key(|(sequence, _)| *sequence) + .map(|(sequence, entry)| (*sequence, entry)) + } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], @@ -89,3 +137,79 @@ impl Frontier { self.insert(sequence, dependency) } } + +#[kunit_tests(asahi_g17_dependency)] +mod tests { + use super::*; + + #[derive(Clone)] + struct Probe(kernel::sync::Arc); + + impl Probe { + fn new(retained: bool) -> Self { + Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap()) + } + fn retire(&self) { + self.0.store(false, core::sync::atomic::Ordering::Relaxed); + } + } + + impl Retention for Probe { + fn retained(&self) -> bool { + self.0.load(core::sync::atomic::Ordering::Relaxed) + } + } + + fn earliest(record: &mut FailureRecord) -> Option { + record.earliest().map(|(sequence, _)| sequence) + } + + #[test] + fn a_cancelled_command_does_not_poison_its_queue() { + // Cancelled or failed before publication: it released its pin and can never run. + let mut record = FailureRecord::new(); + record.note(5, ECANCELED.to_errno(), &Probe::new(false)); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_retained_failure_cancels_until_it_retires() { + let mut record = FailureRecord::new(); + let failed = Probe::new(true); + record.note(5, EIO.to_errno(), &failed); + assert_eq!(earliest(&mut record), Some(5)); + failed.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_later_retained_failure_survives_an_earlier_retirement() { + let mut record = FailureRecord::new(); + let (first, second) = (Probe::new(true), Probe::new(true)); + record.note(5, EIO.to_errno(), &first); + record.note(9, ECANCELED.to_errno(), &second); + assert_eq!(earliest(&mut record), Some(5)); + first.retire(); + assert_eq!(earliest(&mut record), Some(9)); + second.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn successes_are_not_recorded() { + let mut record = FailureRecord::new(); + record.note(5, 1, &Probe::new(true)); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn failed_predecessor_cannot_be_bypassed() { + let mut frontier = Frontier::new(); + for error in [ECANCELED, EIO, ETIMEDOUT] { + assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED)); + } + assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(())); + assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(())); + assert_eq!(frontier.count, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9508a827be4b1..a175e58392379 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -36,6 +36,8 @@ pub(super) trait Queues { fn retain_render_completions(&mut self, masks: [u64; 2]); fn complete_compute(&mut self, masks: [u64; 2]) -> Result; fn context_killed(&mut self, cookie: u64); + /// The firmware failed a compute command (see [`Event::ComputeFault`]). + fn compute_fault(&mut self, qid: u8, stamp: u32); } /// Work requested after releasing the device mutex. @@ -72,6 +74,12 @@ fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { } Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, Event::ContextKilled(cookie) => host.context_killed(cookie), + Event::ComputeFault { qid, stamp } => { + crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0); + host.compute_fault(qid, stamp); + // Its queue retires or settles its quarantine on this pass. + masks[usize::from(qid) / 64] |= 1 << (qid % 64); + } Event::Unknown(_) => {} } Ok(true) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 0045dff413922..bd1228a82e26f 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,6 +44,8 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, + /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). + bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -57,6 +59,7 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), + bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -108,17 +111,26 @@ impl Detector { Err(_) => 0, } }; - if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + if streak < STREAK_ON { + return; + } + if idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } + } else { + self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } + + fn bound(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) + } } #[cfg(CONFIG_CPU_FREQ)] @@ -410,12 +422,23 @@ impl Feed { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } } + /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps + /// under a quarter of their busy time, and the feed does not see host-refill-bound work. + /// Lingers like streaming, so a pause of more than a few milliseconds ends it. + pub(crate) fn gpu_bound(&self) -> bool { + if !cfg!(CONFIG_CPU_FREQ) { + return false; + } + let now = Self::now(); + self.detector.bound(now) && !self.detector.wanted(now) + } + /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -498,3 +521,51 @@ impl PinnedDrop for Feed { self.shutdown(); } } + +#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] +mod detector_tests { + use super::*; + + /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. + fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { + let mut start = 1000; + let mut now = 1_000_000; + for _ in 0..passes { + detector.note(start, start + busy, now); + start += busy + gap; + now += 1_000_000; + } + now + } + + #[test] + fn back_to_back_long_passes_are_gpu_bound() { + let detector = Detector::new(); + // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. + let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); + assert!(detector.bound(now)); + assert!(!detector.wanted(now)); + assert!(!detector.bound(now + LINGER_NS)); + } + + #[test] + fn refill_gaps_stream_instead() { + let detector = Detector::new(); + // 20 us passes with 60 us refill gaps: bound by host refills. + let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); + assert!(detector.wanted(now)); + assert!(!detector.bound(now)); + } + + #[test] + fn paced_or_short_runs_are_neither() { + let detector = Detector::new(); + // Each pass starts after a pause of at least the refill limit. + let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); + assert!(!detector.bound(now) && !detector.wanted(now)); + // Fewer back-to-back passes than the streak needs. + let detector = Detector::new(); + let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); + assert!(!detector.bound(now) && !detector.wanted(now)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3b31c3cead064..9015dc82e4076 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -670,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -707,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -772,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 524132c4ae7be..26c6adcf34c83 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -136,6 +136,9 @@ pub(crate) enum Event { TvbGrow(TvbGrowRequest), FreeListGrow(FreeListCompletion), ContextKilled(u64), + /// The firmware failed a compute command of queue `qid` and stores no completion stamp for + /// it: `stamp` is the value that command would have stored. + ComputeFault { qid: u8, stamp: u32 }, Unknown(u32), } @@ -145,6 +148,8 @@ impl EventRecord { const TVB_GROW: u32 = 6; const FREE_LIST_GROW: u32 = 13; const CONTEXT_KILLED: u32 = 12; + const COMPUTE_FAULT: u32 = 7; + const COMPUTE_QIDS: u32 = 128; const DIAGNOSTIC_PAIRS_MAX: u16 = 5; const TVB_SLOTS: u32 = 127; const SUBPIPES: u32 = 64; @@ -160,6 +165,16 @@ impl EventRecord { unsafe { self.words[0] } } + /// Type 7 is a retirement witness only on the firmware whose layout and + /// visibility semantics are qualified. 25G83 must keep consuming it as an + /// unknown event until its own fault contract is established. + pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { + if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { + return Ok(Event::Unknown(Self::COMPUTE_FAULT)); + } + self.decode() + } + /// Validates the record's fields without interpreting resource ownership. /// A malformed known event is an error, unlike an unknown tag. pub(crate) fn decode(&self) -> Result { @@ -198,6 +213,19 @@ impl EventRecord { halt_count: record.halt_count, })) } + Self::COMPUTE_FAULT => { + // SAFETY: Every constructor initializes all integer words. + let words = unsafe { self.words }; + // Only compute faults are known to use this type: any other layout stays an + // unknown event, consumed and ignored as before. + if words[3] >= Self::COMPUTE_QIDS || words[4] & 0xff != 0 { + return Ok(Event::Unknown(Self::COMPUTE_FAULT)); + } + Ok(Event::ComputeFault { + qid: words[3] as u8, + stamp: words[4], + }) + } Self::CONTEXT_KILLED => { // SAFETY: Every constructor initializes all integer words. let words = unsafe { self.words }; @@ -525,3 +553,37 @@ impl ContextKill { } } } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_event_records))] +mod tests { + use super::*; + + fn record(words: &[u32]) -> EventRecord { + let mut all = [0u32; EVENT_WORDS]; + all[..words.len()].copy_from_slice(words); + EventRecord::from_words(all) + } + + #[test] + fn a_compute_fault_names_its_queue_and_stamp() { + let event = record(&[7, 1, 2, 0x39, 0x300, 0xffff_fc00, 1]).decode(); + assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); + } + + #[test] + fn v25g83_does_not_accept_an_unqualified_fault_witness() { + use super::super::variant::InitGraph; + let fault = record(&[7, 1, 2, 0x39, 0x300]); + assert_eq!(fault.decode_for(InitGraph::G17P), + Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); + assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), + Ok(Event::Unknown(7))); + } + + #[test] + fn an_unrecognized_type_7_layout_stays_unknown() { + assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 9, 0x301]).decode(), Ok(Event::Unknown(7))); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0cb7304e30677..b1afdad816b6b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -507,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -522,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index c237896a49fb1..f4c3ee5db22e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: GPL-2.0-only //! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). //! @@ -17,9 +17,9 @@ //! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. use super::{ - fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, - Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, - REGISTER_MAP_SLOTS, + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, + RegisterMapEntry, Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, + COMPLETION_RING_INDEX, PERF_STATES, REGISTER_MAP_SLOTS, }; use kernel::prelude::*; @@ -381,26 +381,10 @@ impl HwData25G83 { /// Power configuration control period, in 24 MHz clocks (8 ms). const PERIOD_CLOCKS: u32 = 200_000; -/// Highest GPU power, in mW. -pub(crate) const MAX_POWER_MW: u32 = 69_116; -/// [`MAX_POWER_MW`] as an IEEE-754 single. -const MAX_POWER_MW_F32: u32 = 0x4786_fe00; /// Temperature sensors read by the die and MTR temperature limiters. const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; -/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays -/// of two dies. Not found in the ADT; constant for the SoC. -const LEAKAGE: [u32; 56] = { - let mut words = [0; 56]; - words[0] = 0x44f0_3000; // 1921.5 - words[1] = 0x44f3_0000; // 1944.0 - words[32] = 0x4373_0000; // 243.0 - words[33] = 0x4373_0000; // 243.0 - words[52] = 0x437b_0000; // 251.0 - words[54] = 0x43bd_c000; // 379.5 - words -}; -/// Offset of [`LEAKAGE`] in the power configuration. +/// Offset of the per-chip leakage coefficients in the power configuration. const LEAKAGE_OFFSET: usize = 0x642c; /// Offset in the power configuration of the GPU address of the configuration's status (the /// configuration named by the root, `+0x80`). @@ -433,12 +417,10 @@ const POWER_CONFIG_WORDS: Words = &[ (0x9c4, 0x3f7f2e9f, 1), (0x9cc, 0x3b516154, 1), (0x9d4, 0x3ca59586, 1), - (0x9e0, MAX_POWER_MW_F32, 1), (0x9e4, 0x40a90fdb, 1), (0x9ec, 0xbc6d0cfe, 1), (0x9f0, 0x64, 1), (0x9f4, 0x3e8, 2), - (0xa04, MAX_POWER_MW, 1), (0xa38, 0x3e8, 1), (0xa70, 0x8, 1), (0xa7c, 0x3f000000, 1), @@ -448,7 +430,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0xa9c, 0x3ecf2e49, 1), (0xaa8, 0x64, 1), (0xaac, 0x3e8, 1), - (0xabc, MAX_POWER_MW, 1), (0xac8, 0x10, 1), (0xad0, 0x5dc00, 1), (0xb28, 0x50, 1), @@ -508,7 +489,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x27b8, 0xc80000, 1), (0x2870, 0x3e8, 1), (0x28c0, 0x1, 1), - (0x28c8, MAX_POWER_MW, 3), (0x28e0, 0x3f800000, 1), (0x28e4, 0x4, 1), (0x28e8, 0x8, 1), @@ -521,8 +501,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x29e8, 0x3f4f2e49, 1), (0x29f4, 0x28, 1), (0x29f8, 0x3e8, 2), - (0x2a04, MAX_POWER_MW_F32, 1), - (0x2a08, MAX_POWER_MW, 1), (0x2a14, 0xf8, 1), (0x2a1c, 0x5ad200, 1), (0x2a3c, 0x3e8, 1), @@ -591,27 +569,34 @@ unsafe impl Zeroable for PowerConfig25G83 {} static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); impl PowerConfig25G83 { - /// Pointer and random words, which [`Self::hash`] leaves out. - const RUNTIME: [(usize, usize); 3] = - [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; - /// Initializes the power configuration. - pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + pub(crate) fn init( + &mut self, + config_va: u64, + aon_va: u64, + entropy: u32, + max_power_mw: u32, + leakage: &[u32; 56], + ) { + self.0.fill(0); fill_words(&mut self.0, POWER_CONFIG_WORDS); - for (index, word) in LEAKAGE.iter().enumerate() { + for (index, word) in leakage.iter().enumerate() { let at = LEAKAGE_OFFSET + 4 * index; self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); } + // All copies of the power ceiling must describe the same chip. + for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { + self.0[offset..offset + 4].copy_from_slice(&max_power_mw.to_le_bytes()); + } + let power = crate::float::F32::from(max_power_mw).to_bits(); + for offset in [0x9e0, 0x2a04] { + self.0[offset..offset + 4].copy_from_slice(&power.to_le_bytes()); + } self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] .copy_from_slice(&(config_va + 0x80).to_le_bytes()); self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); } - - /// Hash of the contents without the pointers and the random word. - pub(crate) fn hash(&self) -> u64 { - hash_without(&self.0, &Self::RUNTIME) - } } /// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU @@ -632,50 +617,3 @@ pub(crate) const AON_TABLE: [u32; 9] = [ /// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero /// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; - -impl HwData25G83 { - /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) - /// and what comes from the device: the performance tables built from its operating points, - /// which are binned per device, and its enabled core count. - const UNHASHED: [(usize, usize); 10] = [ - (COMPLETION_DESCRIPTORS, 0x80), - (QOS_POINTER, 8), - (core::mem::offset_of!(Self, unk_f88), size_of::()), - (core::mem::offset_of!(Self, unk_fc0), size_of::()), - (core::mem::offset_of!(Self, perf_a), size_of::()), - (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), - (core::mem::offset_of!(Self, cs_perf), size_of::()), - (core::mem::offset_of!(Self, afr_perf), size_of::()), - (core::mem::offset_of!(Self, perf_b), size_of::()), - (core::mem::offset_of!(Self, unk_2570), size_of::()), - ]; - - /// Hash of the constant contents (everything but [`Self::UNHASHED`]). - pub(crate) fn hash(&self) -> u64 { - // SAFETY: `HwData25G83` consists of integers only and has no padding. - let bytes = unsafe { - core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) - }; - hash_without(bytes, &Self::UNHASHED) - } -} - -/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. -pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { - let mut hash = 0xcbf2_9ce4_8422_2325u64; - for (offset, byte) in bytes.iter().enumerate() { - if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { - continue; - } - hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); - } - hash -} - -/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the -/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. -pub(crate) mod expected { - pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; - pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; - pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; -} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 4f3a70671eb87..ca2b6604a302b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 5708a2251a062..66e7d07db48a5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -96,8 +96,6 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, - /// [`FragmentBarrier`] (25G83 firmware). - FragmentBarrier = 4, /// [`InitBufferManager`] (25G83 firmware). InitBufferManager = 5, } @@ -569,63 +567,6 @@ impl QueueConfig { } } -/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work -/// waits for the paired tiling stamp, in addition to the kick-entry dependency. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FragmentBarrier { - /// [`CommandTag::FragmentBarrier`]. - pub(crate) tag: u32, - /// GPU address of the tiling queue's completion stamp word. - pub(crate) stamp_va: u64, - /// Timestamp of the tiling kick. - pub(crate) tiling_kick: u64, - /// QID of the tiling queue. - pub(crate) tiling_qid: u32, - /// Tiling stamp value waited for. - pub(crate) wait: u32, - pub(crate) unk_1c: [u8; 8], - /// Event of the wait: the tiling QID. - pub(crate) event: u32, - /// Fragment stamp value. - pub(crate) stamp_self: u32, - /// Current object ID of the fragment stage. - pub(crate) fragment_id: u64, - pub(crate) unk_34: u32, - pub(crate) unk_38: [u8; 0x10], -} - -static_assert!(size_of::() == 0x48); -static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); -static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); -static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); - -impl FragmentBarrier { - /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, - /// signals `stamp_value` at `stamp_va`. - pub(crate) fn new( - stamp_va: u64, - tiling_kick: KickTimestamp, - tiling_qid: u32, - stamp_value: u32, - fragment_id: u64, - ) -> Self { - Self { - tag: CommandTag::FragmentBarrier as u32, - stamp_va, - tiling_kick: tiling_kick.get(), - tiling_qid, - wait: stamp_value, - unk_1c: [0; 8], - event: tiling_qid, - stamp_self: stamp_value, - fragment_id, - unk_34: 1, - unk_38: [0; 0x10], - } - } -} - /// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling /// descriptor that uses a parameter buffer. The block count is read from the manager. #[repr(C, packed)] diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e270d40392a8..d0000282cef82 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -162,8 +162,14 @@ const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; /// The partial-render phases load and store every attached depth and stencil /// surface. -const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; -const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const ZLS_STORE_DEPTH: u64 = 1 << 19; +const ZLS_STORE_STENCIL: u64 = 1 << 18; +const ZLS_LOAD_DEPTH: u64 = 1 << 15; +const ZLS_LOAD_STENCIL: u64 = 1 << 14; +const ZLS_COMPRESS_DEPTH: u64 = 1 << 6; +const ZLS_COMPRESS_STENCIL: u64 = 1 << 8; +/// Bits of the main word a partial render sets for itself. +const ZLS_PARTIAL_CLEAR: u64 = ZLS_COMPRESS_DEPTH | ZLS_COMPRESS_STENCIL | 0xf << 56; const STENCIL_CLEAR_BASE: u32 = 0x300; /// Value of the ZLS swizzle registers and descriptor field. const ZLS_SWIZZLE: u32 = 0x0404_0404; @@ -443,6 +449,7 @@ struct PassWords { layer_mode: u64, tib_blocks: u64, partial_zls_control: u64, + partial_zls_reload: u64, aux_fb_flags: u64, bg_rsrc_spec: u64, bg_va: u64, @@ -478,12 +485,27 @@ impl PassWords { tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; } - let mut partial_zls_control = pass.zls_control; + // The partial store keeps the main word, loads included, and stores and compresses each + // plane it has. The reload arrays add the loads, which a clearing pass doesn't have. + let mut partial_zls_control = pass.zls_control & !ZLS_PARTIAL_CLEAR; if pass.depth.va != 0 { - partial_zls_control |= ZLS_PARTIAL_DEPTH; + partial_zls_control |= ZLS_STORE_DEPTH; + } + if pass.depth.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_STORE_STENCIL; + } + if pass.stencil.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_STENCIL; + } + let mut partial_zls_reload = partial_zls_control; + if pass.depth.va != 0 { + partial_zls_reload |= ZLS_LOAD_DEPTH; } if pass.stencil.va != 0 { - partial_zls_control |= ZLS_PARTIAL_STENCIL; + partial_zls_reload |= ZLS_LOAD_STENCIL; } let mut aux_fb_flags = AUX_FB_FLAGS_BASE; @@ -511,6 +533,7 @@ impl PassWords { * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, + partial_zls_reload, aux_fb_flags, bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), bg_va: usc(pass.bg.offset)?, @@ -636,6 +659,13 @@ fn ta_registers( ]) } +/// Tiler segment control of a two-cluster pass, what macOS writes. +const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; +const VDM_SEGMENT_INDEX: u64 = 4; +/// Pages the tiler keeps out of a partial render's segments: without them the pages a pass +/// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. +const MIN_FREE_TVB_PAGES: u64 = 0x3c; + /// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage /// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the /// M4 Pro has run with. Returns the array and its used entry count. @@ -686,15 +716,15 @@ fn ta_registers_25g83( push(find(0x1c880, 0)?); push(RegisterWrite::new(0x1c948, gtp(4)?)); push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, 0)); - push(RegisterWrite::new(0x1c918, 0)); + push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); + push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); push(find(0x1c079, 0)?); push(find(0x1c9d8, 0)?); push(RegisterWrite::new(0x1c089, gtp2)); push(RegisterWrite::new(0x1c9e0, gtp2)); push(RegisterWrite::new(0x16c41, gtp6)); push(RegisterWrite::new(0x1ca40, gtp6)); - push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c9a8, MIN_FREE_TVB_PAGES)); push(RegisterWrite::new(0x1c920, gtp(7)?)); for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { push(find(reg, 0)?); @@ -930,7 +960,7 @@ fn partial_resume_registers( RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -942,13 +972,15 @@ fn partial_load_registers( [ RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), - RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + // Only the main end-of-tile program resolves, and it has to reach tiles that only + // earlier segments drew to. Without this their resolve target is never written. + RegisterWrite::new(0x10039, w.tile_config | TILE_CONFIG_PROCESS_EMPTY_TILES), RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -1023,7 +1055,11 @@ pub(crate) struct TaDescriptor { /// QID of the tiling queue. pub(crate) qid: u32, pub(crate) unk_8be: u32, - pub(crate) unk_8c2: u32, + /// Non-zero on the first kick of a queue pair, like the fragment descriptor's word at the + /// same position. Set on every kick, the firmware also notifies the host of each pass's + /// tiling completion, which no retirement needs. The 25G83 firmware keeps it set on every + /// kick until this is measured there. + pub(crate) first_kick: u32, /// Completion value of the tiling kick (see /// [`KickTimestamp::completion_value`]). pub(crate) completion: u64, @@ -1072,8 +1108,6 @@ unsafe impl Zeroable for TaDescriptor {} impl TaDescriptor { const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; const UNK_876: u32 = u32::MAX; - const UNK_892: u32 = 1; - const UNK_8C2: u32 = 1; const UNK_932: u8 = 0x44; const UNK_93C: u8 = 1; const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; @@ -1110,9 +1144,7 @@ impl TaDescriptor { } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1124,12 +1156,17 @@ impl TaDescriptor { self.sampler_heap_va = pass.sampler_heap_va; self.sampler_count = pass.sampler_count; self.sampler_max = sampler_max; - self.unk_892 = Self::UNK_892; + // Asks the firmware to grow the heap. macOS keeps that for memoryless passes, which can't + // partial-render, and Honeykrisp has none. 26A428 keeps it: untested there. + self.unk_892 = u32::from(args.programs != RegisterPrograms::V25G83); self.stamp_va = args.ta.stamp_va; self.aux_stamp_va = args.ta.aux_stamp_va; self.stamp_value_2 = stamp_value; self.qid = args.ta.qid.into(); - self.unk_8c2 = Self::UNK_8C2; + self.first_kick = match args.programs == RegisterPrograms::V25G83 { + true => 1, + false => u32::from(args.ordinal == 0), + }; self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; @@ -1456,10 +1493,7 @@ impl FragmentSlot { } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 2f7b598733368..03d5bfccafeca 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -57,19 +57,24 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Tiling descriptors report whether the fragment queue still had earlier kicks + /// outstanding (`fragment_busy`). + pub(crate) fragment_busy: bool, /// Render item rings carry lead commands: InitBufferManager before the first tiling pass - /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + /// of a buffer manager. pub(crate) lead_commands: bool, /// Register programs of the render descriptors. pub(crate) programs: super::render::RegisterPrograms, } -/// A firmware build a SoC supports, identified by the macOS release it ships with. +/// A firmware build a SoC supports, identified by its loaded TEXT. pub(crate) struct Build { - /// macOS version (major, minor, patch). + /// macOS version (major, minor, patch), used to order image candidates. pub(crate) os_version: [u32; 3], /// Name, for the log. pub(crate) name: &'static str, + /// The admitted loaded images, measured from the `fw-text` region. + pub(crate) images: &'static [crate::fwid::FirmwareBuild], /// Layouts and rules of the build. pub(crate) variant: &'static Variant, } @@ -78,9 +83,8 @@ pub(crate) struct Build { pub(crate) enum Firmware { /// The SoC runs a single build, which is not identified (A18 Pro). Fixed(&'static Variant), - /// One of these builds, selected by the macOS version that the boot loader reports in - /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and - /// with it this firmware build. Any other version is refused. + /// One of these exact loaded images. The loader's `apple,firmware-version` is only + /// a candidate-ordering hint because multiple OS releases can share an iBoot string. Identified(&'static [Build]), } @@ -98,6 +102,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, }; @@ -121,8 +126,10 @@ pub(crate) const V25G83: Variant = Variant { mcache: false, // A fragment stage with the few-primitives flag hangs. few_primitives: false, - // A stalled ray traversal wedges the GPU without a firmware recovery. - ray_traversal: false, + ray_traversal: true, + // When a tiling pass runs out of memory with this set, the firmware resumes it without + // adding memory and the tiler stops silently; macOS always writes 0. + fragment_busy: false, lead_commands: true, programs: super::render::RegisterPrograms::V25G83, }; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 10af261aeb3ac..578bb7533f3c9 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -38,6 +38,30 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { + slots + } else { + 0x800 + } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -129,14 +153,6 @@ impl WorkPublication<'_> { ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -191,7 +207,13 @@ struct BuildEnv<'a> { impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn fixed( + &self, + offset: u64, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -203,7 +225,13 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn dynamic( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -308,7 +336,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { + pub(super) fn initialize_compute_pool_blocks( + &self, + slot: u16, + page_list_va: u64, + blocks: usize, + ) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -394,6 +427,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -510,7 +549,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -563,16 +602,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -671,21 +700,117 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. + pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { + use super::fw::{ + channels, + queue::{DataMaster, Policy}, + }; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = + |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let rings = + |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; + Ok(super::port::Views { + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( + &self, + start: usize, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( &self, + seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + let mut newest = None; + for (slot, seen_control) in seen.iter_mut().enumerate() { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })?; + }; + let control = completion.read(0)?; + *seen_control = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } + } + record(completion)?; } - Ok(()) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -709,6 +834,10 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. + #[expect( + clippy::too_many_arguments, + reason = "The firmware graph builder keeps its independent input objects explicit." + )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -786,7 +915,8 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = + self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -870,7 +1000,11 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + let pages = self.add(KernelObject::backing( + env.dev, + low::RENDER_PAGE_LIST_SIZE, + WB, + )?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -878,10 +1012,18 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing( + env.dev, + low::RENDER_RUN_LIST_SIZE, + WB, + )?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing( + env.dev, + abi::FREE_LIST_BLOCK_SIZE, + WB, + )?)?; } self.render_pages = pages; self.render_runs = runs; @@ -889,7 +1031,12 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; + self.alias( + first_block + block, + env.render, + low::render_block(block), + GS, + )?; } Ok(first_block) } @@ -926,12 +1073,8 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; + let ready = + self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1223,7 +1366,11 @@ impl InitData { /// The GPU's operating points: `count` of them, each with one voltage cell for the whole /// GPU or one per cluster. - fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + pub(super) fn opps( + dev: &AsahiDrmDevice, + count: usize, + clusters: u32, + ) -> Result> { let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; if opps.len() != count { return Err(EINVAL); @@ -1318,15 +1465,15 @@ impl InitData { /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). pub(super) fn init_accepted(&self) -> Result { - Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + Ok(self + .status_word(Role::Primary, 0x14)? + .load(Ordering::Acquire) + == 1) } pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn status_read_words( @@ -1334,11 +1481,8 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1431,13 +1575,15 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset + .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1450,18 +1596,28 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? + let mask = self + .object(self.cluster)? .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; + // Progress is independently available even when this firmware's key + // address is unknown. Zero keeps ownership unknown and cannot override + // firmware blame, but preserves the progress evidence for diagnostics. + let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); + let monitor = self + .recovery_view_words::<6>( + abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, + ) + .ok() + .map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); + sources.reason = self + .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok() + .map(|words| words[0]); Ok(sources) } @@ -1517,17 +1673,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index 85aa6468fa311..670f4b96dcb76 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -8,7 +8,7 @@ //! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; -use crate::g17::{Config, Role}; +use crate::g17::{calibration::Calibration, Config, Role}; use crate::mmu; use abi::v25g83 as v; use core::sync::atomic::{fence, Ordering}; @@ -74,7 +74,7 @@ impl InitData { }) } - /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// Writes the 25G83 firmware objects using the chip calibration, then /// stages the opening control-ring state. fn initialize_25g83( &mut self, @@ -101,12 +101,14 @@ impl InitData { let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let calibration = Calibration::read(env.dev)?; + calibration.check_opps(&opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: &calibration.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -117,41 +119,28 @@ impl InitData { }; // SAFETY: Plain C entropy source. let entropy = unsafe { kernel::bindings::get_random_u32() }; - let mut hashes = [0; 3]; self.object_mut(self.bundle)? .initialize::(0, |hw| { hw.init(&hw_args, &hw_args_25g83)?; - hashes[0] = hw.hash(); Ok(()) })?; self.object_mut(self.bundle)? .initialize::(v::POWER_CONFIG, |power| { - power.init(config_va, aon_va, entropy); - hashes[1] = power.hash(); + power.init( + config_va, + aon_va, + entropy, + calibration.max_power_mw, + &calibration.leakage, + ); Ok(()) })?; self.object_mut(self.region_c)? .initialize::(0, |region| { region.init(cfg.region_c); + region.max_power_mw = calibration.max_power_mw; Ok(()) })?; - hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); - // The constant contents must equal what the Gravity Linux encoders produce on the proven - // setup; the operating points and the core count come from the device and are not hashed. - let expected = [ - v::expected::HW_DATA, - v::expected::POWER_CONFIG, - v::expected::REGION_C, - ]; - if hashes != expected { - dev_err!( - env.dev.as_ref(), - "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", - hashes, - expected - ); - return Err(EINVAL); - } for (index, word) in v::AON_TABLE.iter().enumerate() { self.object_mut(objects.aon)?.write(index * 4, *word)?; } @@ -167,7 +156,9 @@ impl InitData { v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); let cluster = self.object(self.cluster)?; cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); - cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + cluster + .dword(fwctl + 8)? + .store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); // One main configuration carrying both roles. let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; @@ -218,7 +209,8 @@ impl InitData { )?; for &(offset, value, repeat) in v::POWER_STATE_WORDS { for index in 0..repeat { - self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + self.object_mut(objects.power_state)? + .write(offset + 4 * index, value)?; } } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 61b8c3b7b4737..7e43c163ac4a7 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -14,7 +14,10 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -const RENDER_CREDITS: u32 = 32; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -150,6 +153,7 @@ impl Packet { super::dependency::replay_status(fence_status(fence))?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -173,8 +177,9 @@ impl Packet { kernel::time::msecs_to_jiffies(10) as isize, ) }; + // An interrupted wait is not a producer failure. if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + return Err(Error::from_errno(waited as i32)); } if waited > 0 { break; } } @@ -183,6 +188,7 @@ impl Packet { for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -252,7 +258,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 { fn check_status(fence: &Fence) -> Result { let status = fence_status(fence); if status < 0 { - Err(super::dependency::inherited_error(Error::from_errno(status))) + Err(super::dependency::PRODUCER_FAILED) } else { Ok(()) } @@ -262,6 +268,9 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } @@ -277,12 +286,16 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; + /// A long job of another queue is running beside `packet`. + fn long_job_beside(&self, _packet: &Arc) -> bool { false } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -297,6 +310,7 @@ struct Timeout { backend: Arc, victim_renewals: u8, execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -314,13 +328,14 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = - Some(&crate::THIS_MODULE); + Some(kernel::module::this_module::()); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -334,6 +349,13 @@ impl sched::JobImpl for Job { ) { return true; } + // Other work retiring means the GPU is shared, not hung. So does another long job + // running beside a long job, though neither retires until it ends. + let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) + || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); + if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred { job.victim_renewals = job.victim_renewals.saturating_add(1); @@ -342,10 +364,17 @@ impl sched::JobImpl for Job { } fn prepare(job: &mut Self) -> Option { - job.backend.prepare(&job.packet) + super::trace::schedule(&job.packet, 5, 0); + let wait = job.backend.prepare(&job.packet); + super::trace::schedule(&job.packet, 6, i32::from(wait.is_some())); + wait } fn run(job: &mut Self) -> Result> { - job.backend.publish(job.packet.clone()).map(Some) + super::trace::schedule(&job.packet, 3, 0); + let result = job.backend.publish(job.packet.clone()).map(Some); + super::trace::schedule(&job.packet, 4, + result.as_ref().err().map_or(0, |error| error.to_errno())); + result } fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep @@ -373,13 +402,32 @@ pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, pub(crate) publication: Option>, + pub(crate) owner: Arc, +} + +impl super::dependency::Retention for Fences { + fn retained(&self) -> bool { + self.owner.retains_vm() + } +} + +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } } impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; @@ -397,6 +445,7 @@ impl Scheduler { early: bool, publication: Option, ) -> Result { + super::trace::schedule(&packet, 0, dependencies.len() as i32); let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); // Only commands which actually write timestamps become host-prefix @@ -429,10 +478,15 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + let inline = early && self.backend.try_early(&packet); + super::trace::schedule(&packet, if inline { 1 } else { 2 }, 0); + if !inline { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed, publication: host_publication }) + Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() }) } } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index c337a8b298f19..6499cae106309 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -24,22 +24,38 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -// Protect one tiling/fragment pair for each logical execution context, plus -// the two unpublished bootstrap reservations. The remaining ten compute -// channels are shared after exact retirement, rather than pinned per client. -const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; -/// Logical queues that can each hold a render pair: what the render reserve leaves beside the -/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone -/// (every execution context with no floor). +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT = "y")] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. pub(super) const fn queue_limit(floor: usize) -> u32 { - ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 } const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - assert!(COMPUTE_CHANNELS >= 4); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -71,7 +87,6 @@ struct Identity { pub(crate) struct QueueIds { entries: [Option; QID_COUNT], initial_render: [Option; 2], - publication: u64, leased: usize, /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). floor: usize, @@ -82,10 +97,9 @@ impl QueueIds { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], - publication: 0, leased: 0, // The reserve must keep render capacity above the floor. - floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -113,6 +127,58 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a render pair not yet built can still reserve both of its identities. + pub(crate) fn render_pair_fits(&self) -> bool { + QID_COUNT - self.floor - self.leased >= 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -129,10 +195,8 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never the host's; the reserve shrinks - // with them, so the compute channel count stays the same. + // The QIDs below the floor are never available to the host. let total = QID_COUNT - self.floor; - let reserve = COMPUTE_RENDER_RESERVE - self.floor; let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() @@ -145,27 +209,9 @@ impl QueueIds { } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if total - self.leased + replacement - <= reserve.saturating_sub(render - replacement) - { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); - } + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred .map(|index| usize::from(index) + self.floor) @@ -196,13 +242,6 @@ impl QueueIds { Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. Compute and - /// render publications share it, so a watchdog can order work across both engines. - pub(crate) fn next_publication(&mut self) -> Result { - self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -249,10 +288,10 @@ mod accounting_tests { #[test] fn render_identities_satisfy_the_compute_reserve() { let mut ids = table(); - for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + for owner in 1..=FULL_RENDER_RESERVE as u64 { ids.reserve(owner, DataMaster::Tiling, None).unwrap(); } - for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.leased, QID_COUNT); @@ -263,14 +302,49 @@ mod accounting_tests { #[test] fn compute_cannot_consume_the_render_minimum() { let mut ids = table(); - for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); + } + + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4, 32] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4, 32] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } } #[test] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2999daec47015..1dbfcc8fd3ce5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,12 +12,18 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. - +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. + +mod availability; mod buffer; +mod calibration; mod channel; mod command; pub(crate) use command::Attachments; @@ -25,29 +31,36 @@ mod completion; pub(crate) mod context; mod data_segment; mod dependency; +mod sync_input; +pub(crate) mod sync_output; +mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -mod ids; pub(crate) mod hazard; +mod ids; mod initdata; mod job; -mod capacity; mod kick; mod object; +mod port; pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; mod replay; +mod reports; mod runtime; +pub(crate) mod stats; pub(crate) mod status; mod teardown; +pub(crate) mod telemetry; mod timeout; mod validation; @@ -55,6 +68,8 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, + AtomicU64, Ordering, // }; @@ -62,12 +77,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, + impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, - impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -192,9 +207,10 @@ pub(crate) struct Config { /// Layouts of the firmware command buffers that follow a SoC's GPU core layout. pub(crate) struct CoreLayout { - /// The cluster and enabled core counts the layouts were measured on, which the GPU must - /// report at probe; `None` where the layouts hold for every configuration of the SoC. + /// The measured cluster and enabled core counts; `None` admits every configuration. pub(crate) cores: Option<(u32, u32)>, + /// An experimental layout: core slots and enabled cores per cluster. + pub(crate) alternate_core_masks: Option<(u32, u32)>, /// A compute queue's operand-state buffer. pub(crate) operand_state: fw::compute::OperandState, /// A render's deflake buffer. @@ -359,7 +375,7 @@ impl rtkit::Operations for RoleOps { if !is_events_message(ep, msg) { return false; } - data.shared.queue_events(); + data.shared.queue_event_records(); true } @@ -394,13 +410,58 @@ impl rtkit::Operations for RoleOps { } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } + + /// A handle on the receive thread's minimum utilization, usable from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this mailbox is dropped. + unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { + // SAFETY: The caller's obligation is the handle's. + self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, _crash_buffer: Option, } @@ -455,7 +516,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -484,19 +545,20 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() + self.rtkit.with(|rtkit| rtkit.boot()) } /// Starts the firmware endpoints and hands the firmware its initialization root. An /// acknowledgment message is awaited here; one in status A is the caller's to poll. fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; match ack { fw::variant::InitAck::Message => self.state.wait_for_ack(), fw::variant::InitAck::Status => Ok(()), @@ -504,8 +566,9 @@ impl Coprocessor { } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -572,6 +635,10 @@ struct Firmware { secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, @@ -595,6 +662,7 @@ impl Drop for Firmware { secondary.stop(); } self.primary.stop(); + self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. @@ -641,42 +709,71 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. + fn observe_statistics_power(&mut self) { + if self.rings.statistics_suspended() { + return; + } + self.rings.observe_statistics_power( + self.bridge.regs.g17_core_power() & 0xf != 0, + self.port.statistics_epoch(), + ); + } + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let effort = match self.effort.idle(now_ns, in_flight) { + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; None } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }) + Ok( + match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + ) } /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A @@ -685,7 +782,10 @@ impl Firmware { const RETRY_NS: u64 = 1_000_000; let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { - match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + match self + .hold + .step(now_ns, in_flight, self.effort.submissions(), hold_ms) + { power::HoldAction::None => return None, power::HoldAction::Recheck(delay) => return Some(delay), power::HoldAction::Assert => { @@ -773,8 +873,28 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, + /// The next event pass runs every housekeeping step: set by every [`Self::queue_events`] + /// request, not by the firmware's event notification ([`Self::queue_event_records`]). + housekeeping: AtomicBool, + /// When an event pass last ran every housekeeping step, in ns. + housekeeping_ns: AtomicU64, #[pin] poll: DelayedWork, #[pin] @@ -795,6 +915,8 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -884,8 +1006,37 @@ impl Shared { drop(retired); } - /// Queues the event worker. Callable from any context. + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changed.notify_all(); + self.renders.notify(); + } + + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + + /// Queues the event worker for a pass that runs every housekeeping step. Callable from any + /// context. fn queue_events(self: &Arc) { + self.housekeeping.store(true, Ordering::Release); + self.queue_event_records(); + } + + /// Queues the event worker for firmware records alone: completions, or records that arrived + /// during a pass. The pass skips the housekeeping that nothing needs (see + /// `runtime_event_turn`). Callable from any context. + fn queue_event_records(self: &Arc) { // Already queued if this fails, which is just as good. let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); } @@ -912,6 +1063,7 @@ impl Shared { // Context release returns its execution root off-lock. The next // device pass can now detach idle compute aliases and their QoS // leases, without depending on a new client's submission/event. + self.queues.notify(); self.queue_events(); } } @@ -991,7 +1143,10 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + stats: Arc, _capacity: Pin>>, + _reports: Pin>>, + _power: Pin>>, } impl Gpu { @@ -1002,8 +1157,22 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { + Some(calibration::Calibration::read(dev).inspect_err(|_| { + dev_err!( + dev.as_ref(), + "Missing or invalid T6040 GPU calibration handoff\n" + ); + })?) + } else { + None + }; let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; + if let Some(calibration) = calibration { + calibration.check_cores(&id.core_masks)?; + calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; + } if id.family != cfg.identity.family || id.variant != cfg.identity.variant || id.num_dies != cfg.identity.num_dies @@ -1023,14 +1192,37 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let idle_delay = + power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; + power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); + dev_info!( + dev.as_ref(), + "GPU idle power-off delay: {} ms\n", + idle_delay + ); + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; + let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { + clusters == id.num_clusters + && cores != active_cores + && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, enabled)| { + id.num_cores == slots + && id.core_masks.len() == clusters as usize + && id.core_masks.iter().all(|mask| mask.count_ones() == enabled) + }) + }); if cfg .core_layout .cores .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + && !alternate { dev_err!( dev.as_ref(), @@ -1040,6 +1232,12 @@ impl Gpu { ); return Err(ENODEV); } + if alternate { + dev_warn!( + dev.as_ref(), + "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" + ); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1054,7 +1252,14 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, + queues, + renders, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), + housekeeping: AtomicBool::new(true), + housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), grow <- new_work!("g17::Shared::grow"), @@ -1063,15 +1268,31 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let directory = kernel::debugfs::Dir::new(c"aurora-g17"); let capacity = KBox::pin_init( directory.read_only_file(c"capacity", capacity::View(shared.clone())), GFP_KERNEL, )?; + let reports = KBox::pin_init( + directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), + GFP_KERNEL, + )?; + let power = KBox::pin_init( + directory.read_only_file(c"power", reports::PowerView(shared.clone())), + GFP_KERNEL, + )?; + let stats = Arc::new( + crate::stats::StatsSnapshot { + g17_reader: Some(stats::Reader(shared.clone())), + ..Default::default() + }, + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1080,11 +1301,17 @@ impl Gpu { geometry, ids: Default::default(), shared, + stats, _capacity: capacity, + _reports: reports, + _power: power, }, GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) @@ -1101,21 +1328,43 @@ impl Gpu { fw::variant::Firmware::Identified(builds) => builds, }; let version = Self::os_firmware_version(dev); - if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok(build.variant); + // iBoot strings can be shared by OS releases. Prefer the reported release's + // candidate, but admit only an exact loaded image, regardless of that hint. + let preferred = builds + .iter() + .filter(|build| Some(build.os_version) == version); + let others = builds + .iter() + .filter(|build| Some(build.os_version) != version); + for build in Iterator::chain(preferred, others) { + if Self::identify_image(dev, build).is_ok() { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } } dev_err!( dev.as_ref(), - "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + "Unsupported GPU firmware image (reported macOS version {:?})\n", version ); - for build in builds.iter() { - dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); - } Err(ENODEV) } + /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size + /// of the `fw-data` region. The version property is only a selection hint. + fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; + let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let start = text.start(); + let end = start.checked_add(text.size()).ok_or(EINVAL)?; + crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) + })?; + Ok(()) + } + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { @@ -1172,13 +1421,21 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let primary = + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) + })?; let secondary = match cfg.secondary_root_offset { Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) - })?, + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( + |e| { + dev_err!( + dev.as_ref(), + "Secondary coprocessor setup failed: {:?}\n", + e + ) + }, + )?, ), None => None, }; @@ -1196,17 +1453,24 @@ impl Gpu { secondary, primary, bridge, - rings: channel::Rings::new(), + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, + seen_submissions: 0, + rings: channel::Rings::new(shared.variant.graph)?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), registration_power_release_pending: false, variant: shared.variant, - queues <- runtime::Registry::new( + queues <- runtime::Registry::with_waiters( shared.clusters, shared.deflake, shared.descriptor_flags, shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, @@ -1242,12 +1506,19 @@ impl Gpu { if let Some(root_entries) = root_entries { session.uat.confirm_firmware_root_entries(root_entries)?; } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } @@ -1260,7 +1531,12 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; session .uat .activate_render_context(&session._render_global)?; @@ -1353,7 +1629,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - None + Some(self.stats.clone()) } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1377,11 +1653,11 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } if variant.ray_traversal { params.features |= @@ -1463,6 +1739,7 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); @@ -1475,16 +1752,25 @@ impl gpu::Gpu for Gpu { let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, Err(error) if error == ENOSPC => { // Closed logical queues can still own execution/QoS IDs // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. - if now_ns().saturating_sub(admission_started) - >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + let waited = now_ns().saturating_sub(admission_started); + let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; + if waited >= limit { + return Err(EBUSY); + } self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A returned queue slot or reclaimed context wakes this. + self.shared.queues.block( + observed, + (limit - waited).div_ceil(1_000_000) as u32, + true, + )?; } result => break result?, } @@ -1495,6 +1781,7 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, + usage, ) { Ok(frontend) => frontend, Err(error) => { @@ -1510,18 +1797,22 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - crate::timestamp::Buffer::firmware(bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )?) + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index f1b22ad5d7331..87fdb0e8a62ad 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -466,8 +466,17 @@ struct PoolInner { reserved: [usize; POOL_FAMILIES], bytes: usize, stopped: bool, + /// Objects the firmware may still reference after their owner dropped them, kept mapped + /// and out of reuse until firmware stop (see [`PooledObject::retain_until_stop`]). + kept: KVec, + kept_bytes: usize, + /// Kept objects still owned by their users, whose list entries are reserved. + kept_reserved: usize, } +/// Retained objects past this many are still retained, with one warning. +const KEPT_WARN: usize = 256; + /// Per-device cache-safe object recycling. Every converted mapping reserves /// one return slot before its page-table attributes change. Active leases and /// poisoned mappings count against the same capacity as reusable objects. @@ -490,6 +499,9 @@ struct CacheLease { pub(crate) struct PooledObject { object: core::mem::ManuallyDrop, lease: Option, + /// The pool keeping this object until firmware stop, and whether `reserve_kept` took its + /// list entry. + keep: Option<(kernel::sync::Arc, bool)>, } impl core::ops::Deref for PooledObject { @@ -503,11 +515,28 @@ impl core::ops::DerefMut for PooledObject { &mut self.object } } +impl PooledObject { + /// The firmware may still reference this object after its owner drops it: keep it mapped + /// and never hand it out again before firmware stop. It leaves the reuse cache, so it + /// does not take cache capacity. Call from sleepable context: this reserves the list + /// entry, so the drop, wherever it happens, does not allocate. + pub(crate) fn retain_until_stop(&mut self, pool: &kernel::sync::Arc) { + if self.keep.is_none() { + let reserved = pool.reserve_kept(); + self.keep = Some((pool.clone(), reserved)); + } + } +} + impl Drop for PooledObject { fn drop(&mut self) { // SAFETY: This destructor is the only operation that moves `object` // out; ManuallyDrop prevents a second implicit drop of the field. let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; + if let Some((pool, reserved)) = self.keep.take() { + pool.keep_object(object, self.lease.take(), reserved); + return; + } match self.lease.take() { Some(lease) => lease.pool.return_object(object, &lease), None => drop(object), @@ -523,7 +552,10 @@ impl Pool { } kernel::sync::Arc::pin_init( pin_init!(Self { - inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), + inner <- kernel::new_mutex!(PoolInner { + returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), + kept_bytes: 0, kept_reserved: 0, + }, "g17::object::Pool"), }), GFP_KERNEL, ) @@ -572,6 +604,7 @@ impl Pool { return Ok(PooledObject { object: core::mem::ManuallyDrop::new(entry.object), lease: Some(lease()), + keep: None, }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; @@ -596,6 +629,7 @@ impl Pool { let mut result = PooledObject { object: core::mem::ManuallyDrop::new(object), lease: reserved.then(lease), + keep: None, }; if !reserved { return Ok(result); @@ -655,10 +689,67 @@ impl Pool { core::mem::forget(object); } + /// Room for one more kept object, taken by the matching `keep_object`. Returns whether + /// it was reserved; after a failed allocation that drop leaks the object instead. + fn reserve_kept(&self) -> bool { + let mut inner = self.inner.lock(); + let pending = inner.kept_reserved + 1; + if inner.kept.capacity() - inner.kept.len() < pending + && inner.kept.reserve(pending, GFP_KERNEL).is_err() + { + return false; + } + inner.kept_reserved = pending; + true + } + + fn keep_object(&self, object: KernelObject, lease: Option, reserved: bool) { + let mut inner = self.inner.lock(); + if reserved { + inner.kept_reserved = inner.kept_reserved.saturating_sub(1); + } + if let Some(lease) = &lease { + // The object leaves the reuse cache and its capacity. + inner.reserved[lease.family as usize] -= 1; + inner.bytes -= object.size(); + } + if inner.stopped { + drop(inner); + drop(object); + return; + } + let size = object.size(); + if inner.kept.len() == KEPT_WARN { + pr_warn!("G17 keeps {} firmware-linked objects until the GPU firmware restarts\n", KEPT_WARN); + } + if !reserved { + // Another kept object's reservation must not be used: leak this one instead. + pr_err!("G17 could not record a firmware-linked object; leaking it\n"); + core::mem::forget(object); + return; + } + // retain_until_stop reserved the entry; this never allocates. + match inner.kept.push_within_capacity(object) { + Ok(()) => inner.kept_bytes += size, + Err(error) => { + // Never unmap an object the firmware may still reference. + pr_err!("G17 could not record a firmware-linked object; leaking it\n"); + core::mem::forget(error.0); + } + } + } + /// Releases retained mappings after both firmware processors have stopped. /// Active objects returned later observe the same stop witness. pub(crate) fn stop_after_firmware(&self) { - self.inner.lock().stopped = true; + let kept = { + let mut inner = self.inner.lock(); + inner.stopped = true; + inner.kept_bytes = 0; + core::mem::take(&mut inner.kept) + }; + // Mapping teardown must not run while the pool mutex is held. + drop(kept); for index in 0..POOL_FAMILIES * POOL_SLOTS { let entry = { let mut inner = self.inner.lock(); @@ -684,6 +775,7 @@ impl Pool { inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), entries.iter().flatten().filter(|entry| entry.poisoned).count())?; } - writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; + writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) } } diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 7fef15084c50a..a2e293df91510 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -6,16 +6,28 @@ use core::sync::atomic::AtomicU32; pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after /// the last job. Not measured on other firmware; at or below it the hold stays off. pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; -/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU -/// cores powered until the GPU has been idle this long ([`Hold`]). +/// Idle policy initialized from the existing DT `apple,idle-off-delay-ms` property. +/// The existing `idle_off_delay_ms` sysfs attribute may override it. Above +/// [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU cores powered until it has been idle +/// this long ([`Hold`]). pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); +/// Use the same bounds as the existing idle-delay attribute. Missing DT policy +/// preserves the native delay; malformed or out-of-range values fail probe. +pub(super) fn idle_delay(requested: Option) -> kernel::error::Result { + let delay = requested.unwrap_or(FIRMWARE_IDLE_OFF_MS); + if !(FIRMWARE_IDLE_OFF_MS..=1000).contains(&delay) { + return Err(kernel::error::code::EINVAL); + } + Ok(delay) +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -237,6 +249,34 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { mod tests { use super::*; + #[test] + fn dt_idle_delay_is_bounded_and_absence_preserves_native_policy() { + assert_eq!(idle_delay(None), Ok(FIRMWARE_IDLE_OFF_MS)); + for delay in [2, 16, 1000] { + assert_eq!(idle_delay(Some(delay)), Ok(delay)); + } + for delay in [0, 1, 1001, u32::MAX] { + assert!(idle_delay(Some(delay)).is_err()); + } + } + + #[test] + fn frame_hold_survives_depth_one_gaps_but_releases_at_idle() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 16), HoldAction::Assert); + hold.asserted(); + for submission in 1..=8 { + // A completed depth-one dispatch followed by another short gap. + let now = submission * 3_000_000; + assert_eq!(hold.step(now, false, submission, 16), HoldAction::Recheck(16_000_000)); + assert!(hold.active()); + } + assert_eq!(hold.step(39_999_999, false, 8, 16), HoldAction::Recheck(1)); + assert_eq!(hold.step(40_000_000, false, 8, 16), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + #[test] fn holds_until_idle_for_the_delay_then_releases() { let mut hold = Hold::new(); @@ -260,3 +300,43 @@ mod tests { assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); } } + +unsafe extern "C" { + fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// DT-selected memory-demand provider. Its independent allocation outlives a +/// detached provider; a device link orders supplier removal and suspend. +pub(super) struct Demand(*mut core::ffi::c_void); + +// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. +unsafe impl Send for Demand {} +// SAFETY: shared access only changes that locked request, never its lifetime. +unsafe impl Sync for Demand {} + +impl Demand { + pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { + let mut out = core::ptr::null_mut(); + // SAFETY: live platform device and writable result; C returns an owned reference. + let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(Self(out)) + } + } + + pub(super) fn set_active(&self, active: bool) { + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_active(self.0, active) }; + } +} + +impl Drop for Demand { + fn drop(&mut self) { + // SAFETY: releases exactly the reference returned by get, clearing demand first. + unsafe { asahi_clpc_gpu_put(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0297c619b4f93..c7d1761699a24 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -6,6 +6,7 @@ //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is //! stored after its tentative mappings so those mappings are destroyed before the join completes. +use core::cell::Cell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::{ new_condvar, new_mutex, @@ -24,25 +25,37 @@ const EPOCH: u64 = CLOSED << 3; /// packet cancellation, logical close, VM status and device loss even when no /// capacity/recovery event arrives. Wakes and nested constructors never renew it. pub(super) struct Admission<'a> { - deadline: u64, + deadline: Cell, + started: u64, check: &'a dyn Fn() -> Result, } impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { - Self { deadline, check } + Self { deadline: Cell::new(deadline), started: super::now_ns(), check } + } + + /// A recovery is pending or this owner's spared work drains: wait as long as the + /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing. + pub(super) fn extend_for_recovery(&self) { + let limit = self + .started + .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000); + if self.deadline.get() < limit { + self.deadline.set(limit); + } } pub(super) fn check(&self) -> Result { (self.check)()?; - if super::now_ns() >= self.deadline { return Err(EBUSY); } + if super::now_ns() >= self.deadline.get() { return Err(EBUSY); } Ok(()) } /// Ignore condvar remaining-time reports: only the monotonic deadline counts. pub(super) fn slice_ms(&self) -> Result { self.check()?; - let remaining = self.deadline.saturating_sub(super::now_ns()); + let remaining = self.deadline.get().saturating_sub(super::now_ns()); if remaining == 0 { return Err(EBUSY); } Ok(remaining.div_ceil(1_000_000).min(10) as u32) } @@ -55,20 +68,37 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -120,19 +150,20 @@ impl Gate { } } - /// Packet admission may wait for recovery only within its original budget. + /// Only a recovery closes the gate: packet admission waits for it up to + /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery). /// Called with no device mutex or existing preparation lease. pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let mut wait = self.wait.lock(); loop { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let _ = self.changed.wait_timeout( @@ -162,20 +193,55 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } /// Runtime failure/recovery must not join an indefinitely stalled builder. @@ -199,7 +265,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -213,6 +279,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, @@ -268,6 +336,20 @@ mod growth_gate_tests { mod compute_deadline_tests { use super::*; + #[test] + fn a_recovery_extends_the_deadline_once_from_the_start() { + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert_eq!(admission.check(), Err(EBUSY)); + admission.extend_for_recovery(); + assert_eq!(admission.check(), Ok(())); + let limit = admission.started + + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000; + assert_eq!(admission.deadline.get(), limit); + admission.extend_for_recovery(); + assert_eq!(admission.deadline.get(), limit); + } + #[test] fn spent_budget_refuses_even_an_open_gate() { let gate = Gate::new().unwrap(); @@ -278,13 +360,16 @@ mod compute_deadline_tests { } #[test] - fn closed_gate_expires_without_any_recovery_wake() { + fn closed_gate_waits_past_the_budget_for_recovery() { let gate = Gate::new().unwrap(); gate.close(); let start = super::super::now_ns(); - let check = || Ok(()); + // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting. + let check = || { + if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) } + }; let admission = Admission::new(start + 20_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); assert!(super::super::now_ns() - start < 500_000_000); gate.wait_drained(); assert!(gate.reopen()); @@ -307,9 +392,9 @@ mod compute_deadline_tests { #[test] fn recovery_and_binding_retry_churn_cannot_renew_budget() { let gate = Gate::new().unwrap(); - let start = super::super::now_ns(); + let deadline = super::super::now_ns() + 20_000_000; let check = || Ok(()); - let admission = Admission::new(start + 20_000_000, &check); + let admission = Admission::new(deadline, &check); loop { // Re-entry models a channel taken between admission/publication. // Recovery joins each dropped lease, then wakes the same budget. @@ -317,12 +402,17 @@ mod compute_deadline_tests { gate.changed.notify_all(); gate.wait_drained(); assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); match gate.enter_until(&admission) { - Ok(lease) => drop(lease), + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } Err(error) => { assert_eq!(error, EBUSY); break; } } } - assert!(super::super::now_ns() - start < 500_000_000); assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ebdc79d2b2dd3..f09e629aa047e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,6 +44,9 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; @@ -67,11 +71,19 @@ struct State { /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. spared: u64, kill_hold: u64, + /// Slots whose kill hold ended at their killed context's consumed scheduler release, + /// and when. A new owner takes such a pair last, the one released longest ago first. + killed: u64, + killed_at: [u64; cfg::RENDER_SLOTS], + /// The physical queue table still has room for a pair built in an unowned slot. The + /// runtime updates this whenever the table changes. + build_room: bool, released: u64, epoch: u64, footprints: KVec, next_ticket: u64, } + impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; const fn new() -> Self { @@ -83,6 +95,9 @@ impl State { quarantined: 0, spared: 0, kill_hold: 0, + killed: 0, + killed_at: [0; cfg::RENDER_SLOTS], + build_room: true, released: 0, epoch: 0, footprints: KVec::new(), @@ -116,6 +131,15 @@ impl State { self.epochs[slot] = self.epoch; } fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { + if self.owners[slot] != owner { + crate::g17::trace::teardown( + 6, + slot as u64, + self.owners[slot], + owner | u64::from(self.killed & (1u64 << slot) != 0) << 63, + ); + self.killed &= !(1u64 << slot); + } self.refs[slot] += 1; self.active |= 1u64 << slot; self.released &= !(1u64 << slot); @@ -161,13 +185,23 @@ impl State { } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; - // Reuse a closed, exactly retired owner before consuming a new graph. + let released = |slot: usize| self.released & (1u64 << slot) != 0; + let unowned = || (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0); + // Reuse a closed, exactly retired owner before consuming a new graph. A killed + // context's pair comes after a new graph while one can still be built: on Neo, new + // owners of such pairs had renders that executed but never stored their stamps. + // Once the queue table is full it is reused, the one released longest ago first, + // so killed pairs never strand physical queues. let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) + .filter(|slot| eligible(*slot) && released(*slot) && self.killed & (1u64 << slot) == 0) .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| if self.build_room { unowned() } else { None }) .or_else(|| { - (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) - })?; + (0..cfg::RENDER_SLOTS) + .filter(|slot| eligible(*slot) && released(*slot)) + .min_by_key(|slot| self.killed_at[*slot]) + }) + .or_else(unowned)?; let first = self.take_idle(slot, owner); self.claim_sibling(owner); Some(first) @@ -176,7 +210,8 @@ impl State { if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { return; } - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + // A speculative sibling never takes a killed context's pair. + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold & !self.killed; let eligible = |slot: usize| free & (1u64 << slot) != 0; let sibling = (0..cfg::RENDER_SLOTS) .filter(|slot| { @@ -186,6 +221,10 @@ impl State { }) .min_by_key(|slot| self.epochs[*slot]) .or_else(|| { + // Nor an unowned slot once no pair can be built there. + if !self.build_room { + return None; + } (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) }); if let Some(slot) = sibling { @@ -219,6 +258,7 @@ impl State { let bit = 1u64 << slot; let old = self.owners[slot]; if old == 0 + || self.killed & bit != 0 || owner == Some(old) || expected.is_some_and(|expected| expected != old) || self.refs[slot] != 0 @@ -294,6 +334,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -332,6 +394,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -339,6 +402,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -366,9 +435,22 @@ pub(crate) struct Pool { queues: AtomicU32, /// Logical queues admitted at most: each must be able to hold a render pair. max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { + #[cfg(CONFIG_KUNIT = "y")] pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + renders: Option>, + ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); } @@ -378,10 +460,19 @@ impl Pool { released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), max_queues, + waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } /// Logical queues currently counted, including one being created. pub(crate) fn queues(&self) -> u32 { self.queues.load(Ordering::Acquire) @@ -403,6 +494,16 @@ impl Pool { } } } + /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. + pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { + let state = self.state.lock(); + state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) + } + pub(crate) fn acquire_render( self: &Arc, owner: u64, @@ -434,17 +535,22 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if now >= quarantine_deadline { - // Behind an innocent pair's quarantine (another VM wedged the GPU) only this - // publication fails, not the VM: it recovers once the pair drains. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); - return Err(if spared { EBUSY } else { EIO }); + // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the + // pair to drain until the caller's deadline, which the caller extends while it + // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + if now >= quarantine_deadline && !spared { + return Err(EIO); } - let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + let slice = if spared { + remaining_ms + } else { + remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?) + }; if matches!(self.released.wait_interruptible_timeout( &mut state, kernel::time::msecs_to_jiffies(slice) ), CondVarTimeoutResult::Signal { .. }) { @@ -522,6 +628,24 @@ impl Pool { record, })) } + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } pub(crate) fn warmable_sibling( self: &Arc, owner: u64, @@ -558,7 +682,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -575,9 +699,27 @@ impl Pool { state.quarantined &= !released; state.spared &= !released; state.kill_hold = 0; + state.killed = 0; drop(state); - self.released.notify_all(); + self.returned(); + } + + /// Whether the physical queue table can hold another pair (see `State::build_room`). + pub(crate) fn note_build_room(&self, room: bool) { + let mut state = self.state.lock(); + if state.build_room != room { + state.build_room = room; + drop(state); + self.returned(); + } } + + /// A killed context's pair is free for a new owner. + pub(crate) fn killed_pair_free(&self) -> bool { + let state = self.state.lock(); + state.killed & state.released & !state.active & !state.quarantined & !state.kill_hold & State::MASK != 0 + } + } #[kunit_tests(asahi_g17_admission_accounting)] @@ -653,9 +795,18 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } @@ -691,7 +842,7 @@ impl RenderLease { } state.quarantined |= bit; drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. @@ -705,24 +856,83 @@ impl RenderLease { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } impl State { - fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64, executed: u64) -> bool { if owner == 0 || slots & !Self::MASK != 0 { return false; } @@ -731,10 +941,10 @@ impl State { if slots & bit == 0 { continue; } + // A pair whose commands all ran is held with them (see `Pair::executed_for_kill`). if self.owners[slot] != owner - || self.refs[slot] != 0 - || self.active & bit != 0 - || self.released & bit == 0 + || (executed & bit == 0 + && (self.refs[slot] != 0 || self.active & bit != 0 || self.released & bit == 0)) { return false; } @@ -745,18 +955,39 @@ impl State { } impl Pool { /// Hold the departed owner's idle pairs before publishing its exit record. - /// Successful publication retains these bits until both processors stop. - pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { - self.state.lock().hold_for_exit_kill(owner, slots) + /// Successful publication retains these bits until the owner's scheduler release is + /// consumed (`release_killed_slots` at detach), or until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64, executed: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots, executed) } - /// Undo only a hold whose control record was not published. + /// Undo a hold whose control record was not published. pub(crate) fn release_exit_kill_hold(&self, slots: u64) { if slots == 0 { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); + } + + /// End the kill hold of `slots` once their killed context's scheduler release was + /// consumed (the firmware no longer names these pairs). A new owner takes them last. + pub(crate) fn release_killed_slots(&self, slots: u64) { + let mut state = self.state.lock(); + let held = state.kill_hold & slots & State::MASK; + if held == 0 { + return; + } + let now = crate::g17::now_ns(); + state.kill_hold &= !held; + state.killed |= held; + for slot in 0..cfg::RENDER_SLOTS { + if held & (1u64 << slot) != 0 { + state.killed_at[slot] = now; + } + } + drop(state); + self.returned(); } } @@ -785,10 +1016,13 @@ mod fault_capacity_tests { let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); + assert!(!pool.spared_quarantined(1)); lease.quarantine(true); assert_ne!(pool.state.lock().spared & bit, 0); + assert!(pool.spared_quarantined(1)); lease.quarantine(false); assert_eq!(pool.state.lock().spared & bit, 0); + assert!(!pool.spared_quarantined(1)); // Once guilty, a later spared failure does not make it spared again. lease.quarantine(true); assert_eq!(pool.state.lock().spared & bit, 0); @@ -831,13 +1065,86 @@ mod fault_capacity_tests { let bit = 1u64 << lease.slot(); pool.release_owner(1); drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); // A failed (unpublished) control may roll back its hold. Published kills // keep the existing processor-stop rule; this test must not weaken it. pool.release_retained_owners_after_processor_stop(); assert_eq!(pool.state.lock().kill_hold, 0); } + + #[test] + fn consumed_scheduler_release_ends_the_kill_hold() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let slot = lease.slot(); + let bit = 1u64 << slot; + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); + assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); + pool.release_killed_slots(bit); + assert_eq!(pool.state.lock().kill_hold, 0); + assert_eq!(pool.state.lock().killed, bit); + assert!(pool.killed_pair_free()); + // The killed context's pair is never reassigned as a spare, and a new owner takes + // a new graph before it. + assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); + let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_ne!(next.slot(), slot); + drop(next); + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().killed, 0); + } + + #[test] + fn killed_pairs_are_taken_last_oldest_first() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = 100 + slot as u64; + state.released |= 1u64 << slot; + state.killed |= 1u64 << slot; + state.killed_at[slot] = 1000 - slot as u64; + } + // One pair released without a kill. + state.killed &= !(1u64 << 7); + } + let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(first.slot(), 7); + let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); + assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); + } + + #[test] + fn a_full_queue_table_reuses_killed_pairs_before_unowned_slots() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..8 { + state.owners[slot] = 100 + slot as u64; + state.released |= 1u64 << slot; + state.killed |= 1u64 << slot; + state.killed_at[slot] = 1000 - slot as u64; + } + } + // With room in the queue table a new owner builds a pair in an unowned slot. + let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + assert!(usize::from(built.slot()) >= 8); + // Without room it takes the killed pair released longest ago. + pool.note_build_room(false); + let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(reused.slot(), 7); + assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); + // It claims no sibling: neither another killed pair nor an unowned slot. + let state = pool.state.lock(); + for slot in 1..7 { + assert_eq!(state.owners[slot], 100 + slot as u64); + } + assert_eq!(state.owners.iter().filter(|owner| **owner == 2).count(), 1); + } } impl Pool { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 713809907dd4b..55994066825f8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -15,7 +15,7 @@ use crate::g17::{ queue::*, variant::PointerLayout, }, - object::{Allocator, CpuMap, KernelObject}, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -30,10 +30,21 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) + .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -83,24 +94,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -112,10 +131,14 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, /// Host words of the pointer block in this firmware build. @@ -127,6 +150,7 @@ struct Graph { impl Graph { fn new( alloc: &Allocator<'_>, + objects: &Arc, context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, @@ -134,7 +158,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; @@ -144,11 +168,18 @@ impl Graph { support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), @@ -159,9 +190,21 @@ impl Graph { view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -172,6 +215,7 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, @@ -237,9 +281,12 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self + .graph + .word(CURSORS + self.pointers.producer)? + .load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -250,12 +297,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -289,11 +336,13 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + self.graph + .dword(ITEMS + index * 8)? + .store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); self.graph - .word(POINTERS + self.pointers.producer)? + .word(CURSORS + self.pointers.producer)? .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); @@ -519,12 +568,24 @@ pub(crate) struct Queue { replay_error: Option, replay_prepared: bool, exit: Option, + /// Since when the kill of this failed queue's context waits for other contexts' compute, + /// and whether that wait was reported. + kill_wait: Option<(u64, bool)>, installed: bool, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// The stamp of the newest command the firmware reported failed ([`Self::note_fault`]), + /// while it is newer than the stored stamp. + fault_stamp: Option, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -538,6 +599,8 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -545,6 +608,7 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, @@ -552,11 +616,13 @@ impl Queue { operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, @@ -579,11 +645,16 @@ impl Queue { replay_error: None, replay_prepared: false, exit: None, + kill_wait: None, installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, + fault_stamp: None, + claims: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -597,8 +668,44 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), + }) + } + /// The firmware failed the command whose completion stamp is `stamp`. It writes a + /// zero-duration completion record for it, which quarantines the queue, but stores no + /// stamp: without this the quarantine would wait for its retirement witness forever and + /// keep the queue's backing. Commands published after the failed one still need their own + /// stamp or record. + pub(crate) fn note_fault(&mut self, stamp: u32) { + if self + .fault_stamp + .is_none_or(|old| retirement::stamp_newer(stamp, old)) + { + self.fault_stamp = Some(stamp); + } + } + + /// The firmware failed a command of this quarantined queue, and the queue still waits for + /// its retirement witness: commands published after the failed one, which the firmware + /// never runs, or the failed command itself, whose ring entry it does not consume (a ray + /// traversal's, on Neo). Only a kill of the queue's context gives them back. + pub(crate) fn fault_pending(&self) -> bool { + self.fault_stamp.is_some() + && self.quarantined + && self.awaiting_witness + && self.exit.is_none() + } + + /// The stamp that proves retirement: the stored one, or a newer reported fault's. + fn witness_stamp(&self) -> Result { + let stored = self.graph.stamp()?; + Ok(match self.fault_stamp { + Some(fault) if retirement::stamp_newer(fault, stored) => fault, + _ => stored, }) } + pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } @@ -626,12 +733,24 @@ impl Queue { } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self + .context() + .is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } + /// The oldest command is a published long job (ray traversal or AS build) whose + /// completion record the driver has not seen yet. + pub(crate) fn long_job_running(&self) -> bool { + !self.quarantined + && self.active.front().is_some_and(|active| { + active.observed.is_none() + && active.packet.is_published() + && active.packet.command.long_job() + }) + } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { @@ -659,6 +778,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -672,7 +812,10 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), + qos: self + .binding + .as_ref() + .map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -689,7 +832,8 @@ impl Queue { } pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + self.pool_release_pending() + && crate::g17::timeout::background_stalled(self.release_sent_ns, now) } /// A closed owner can still retain its last commands or their completion @@ -698,7 +842,12 @@ impl Queue { pub(crate) fn release_settling(&self) -> bool { // release_owner keeps the old owner while its last commands retire. // Waiting only once owner is None misses precisely that interval. - release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + release_pending( + self.released, + self.quarantined, + self.active.len, + self.retire_pending, + ) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -706,33 +855,32 @@ impl Queue { || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { @@ -836,6 +984,7 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { + let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -855,7 +1004,18 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = retirement::record_aged(front.observed_at, now_ns()); + let record_aged = !retirement::stamp_covers(stamp, ticket.kick) + && match retirement::stamp_wait(front.observed_at, now_ns(), || { + // A node that cannot be read keeps the bound from the record. + front.work.drained().unwrap_or(true) + }) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + restart = Some(now); + false + } + retirement::StampWait::Aged => true, + }; let timestamps = retirement::poll( &ticket, front.observed, @@ -887,6 +1047,16 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); + crate::g17::trace::complete( + crate::g17::trace::COMPUTE, + ticket.qid, + ticket.ordinal, + &front.packet, + [timestamps[0], timestamps[1]], + ); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -906,6 +1076,9 @@ impl Queue { } } Ok(None) | Err(EAGAIN) => { + if restart.is_some() { + self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; + } self.retire_pending = self.front_finished(); break; } @@ -956,7 +1129,7 @@ impl Queue { if !retirement::quarantined_retired( &newest, self.graph.cursors()?, - self.graph.stamp()?, + self.witness_stamp()?, self.active.len as u32, self.submitted, )? { @@ -1000,6 +1173,7 @@ impl Queue { drop(self.active.pop()); } self.awaiting_witness = false; + self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; @@ -1102,7 +1276,9 @@ impl Queue { /// Validate the complete unpublished item budget before changing descriptors, /// work counts or queue producers. Leave one item empty across ring wrap. fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { return Err(ERANGE); } @@ -1111,14 +1287,18 @@ fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); return Err(EAGAIN); } - Ok(()) + Ok(()) } /// Retained commands are capacity, not a failed VM. Nothing has been written /// when this check runs, so the outer publisher can wait for retirement under /// its original admission deadline, just as it does for a full firmware ring. fn retained_capacity( - quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, + quarantined: bool, + released: bool, + active: usize, + replays: usize, + replay_front: bool, ) -> Result { if quarantined || released { return Err(EBUSY); @@ -1139,17 +1319,68 @@ fn release_pending(released: bool, quarantined: bool, active: usize, retiring: b mod admission_tests { use super::*; + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start( + slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + CACHED_RECORD_STRIDE, + ) + .unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!( + record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), + Err(EINVAL) + ); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!( + record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1) + ); + } + #[test] fn retained_ring_pressure_waits_without_discarding_work() { let mut active = Fifo::new(); for sequence in 0..IN_FLIGHT { active.push(sequence).unwrap(); } - assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Err(EAGAIN) + ); assert_eq!(active.front(), Some(&0)); assert_eq!(active.len, IN_FLIGHT); assert_eq!(active.pop(), Some(0)); - assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Ok(()) + ); assert_eq!(active.front(), Some(&1)); } @@ -1157,28 +1388,114 @@ mod admission_tests { fn replay_front_and_terminal_ownership_remain_distinct() { assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, IN_FLIGHT, 1, true), + Err(EAGAIN) + ); for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + assert_eq!( + retained_capacity(quarantined, released, 0, 1, true), + Err(EBUSY) + ); } } #[test] fn item_refusal_is_retryable_and_preserves_the_empty_slot() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 10, + consumer: 10, + count: n + }, + true + ), + Ok(()) + ); } #[test] fn bad_item_cursors_are_not_backpressure() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); - assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 0, + count: 0 + }, + false + ), + Err(EIO) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n, + consumer: 0, + count: n + }, + false + ), + Err(ERANGE) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: n, + count: n + }, + true + ), + Err(ERANGE) + ); } #[test] @@ -1214,7 +1531,9 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { return false; } + if self.growing { + return false; + } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1230,11 +1549,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1274,6 +1597,13 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind( + crate::g17::trace::COMPUTE, + self.qid(), + context.id(), + if same_vm { 2 } else { 1 }, + ); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1284,6 +1614,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1338,7 +1671,10 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated + && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1396,6 +1732,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1405,6 +1742,7 @@ impl Queue { self.spared_deferred = false; self.retire_pending = false; self.awaiting_witness = false; + self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.replay_error = None; @@ -1430,9 +1768,8 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); self.release_sent_ns = Some(crate::g17::now_ns()); @@ -1447,7 +1784,9 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { return Ok(None); } + if self.growing { + return Ok(None); + } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1456,11 +1795,18 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let alias = self + .binding + .as_mut() + .and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let descriptors = self + .binding + .as_mut() + .and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1475,7 +1821,9 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { return Err(EAGAIN); } + if !self.can_rearm() { + return Err(EAGAIN); + } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1489,27 +1837,50 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { return Err(EAGAIN); } - if self.quarantined || self.released || self.exit.is_some() - || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + pub(crate) fn begin_growth( + &mut self, + init: &crate::g17::initdata::InitData, + blocks: usize, + ) -> Result { + if self.growing { + return Err(EAGAIN); + } + if self.quarantined + || self.released + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + || self.pool_phase != PoolPhase::Populated + { + return Err(EAGAIN); + } if self.installed { - let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; + } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth( + &mut self, + init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, blocks: usize) -> Result { - if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + alias: &mut Option, + blocks: usize, + ) -> Result { + if !self.growing || blocks < self.growth_blocks { + return Err(EAGAIN); + } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1517,7 +1888,9 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { return Err(EINVAL); } + if alias.is_none() { + return Err(EINVAL); + } let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. @@ -1529,11 +1902,19 @@ impl Queue { self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + Ok(Vacated { + _alias: old_alias, + _pages: pages.take(), + _descriptors: None, + _ring: None, + }) } pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + self.growing.then_some(Growth { + qid: self.qid(), + phase: self.pool_phase, + }) } /// Transfer an abandoned builder's release back to the event worker. Do not @@ -1634,11 +2015,11 @@ impl Queue { let producer = self .graph .graph - .word(POINTERS + self.graph.pointers.producer)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + target)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1675,9 +2056,7 @@ impl Queue { let ticket = active.ticket.ok_or(EIO)?; let state = crate::g17::job::fence_status(&active.packet.completion.fence()); if state < 0 { - return Err(crate::g17::dependency::inherited_error(Error::from_errno( - state, - ))); + return Err(crate::g17::dependency::PRODUCER_FAILED); } if state > 0 { return Ok(None); @@ -1690,6 +2069,17 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride) + .and_then(|start| start.checked_add(offset)) + .ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1699,12 +2089,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - (&mut self.records, slot * RECORD_STRIDE + offset) + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) + } else { + (&mut self.cached_records, CACHED_RECORD_STRIDE) + }; + (object, record_start(slot, offset, window, stride)?) } else { - (&mut self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1730,7 +2123,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1756,8 +2149,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -1775,6 +2167,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1809,11 +2212,26 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { + return Err(EAGAIN); + } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { + return Err(EINVAL); + } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1858,6 +2276,10 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self + .config_cache + .and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1879,6 +2301,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -1904,7 +2327,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( @@ -1921,8 +2344,26 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes( + crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence, + ); + let address = self.graph.record( + record_slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + record, + )?; + wait_vas.push(address, GFP_KERNEL)?; + } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1948,7 +2389,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -1957,15 +2398,25 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { + })() + .or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; - Err(error) - })?; + } + Err(error) + })?; visible = true; self.unpublished_binding = false; let qos = host.qos_publish( @@ -1978,14 +2429,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -2015,10 +2468,12 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -2043,13 +2498,13 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // Every publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { @@ -2087,8 +2542,13 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity(self.quarantined, self.released, self.active.len, - self.replays.len, replay_front)?; + retained_capacity( + self.quarantined, + self.released, + self.active.len, + self.replays.len, + replay_front, + )?; if !packet.context.is_current() || !self .binding @@ -2155,7 +2615,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -2176,17 +2636,30 @@ impl Queue { u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + Ok(cursors) if cursors.count != 0 => writeln!( + f, + "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, + cursors.count - 1 + ), _ => writeln!(f, "items_unavailable=1"), }?; - writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), - u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + writeln!( + f, + "compute_settling qid={} release={} pool_release={}", + self.qid(), + u8::from(self.release_settling()), + u8::from(self.pool_release_pending()) + )?; if let Some(binding) = self.binding.as_ref() { - binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + binding + .context + .write_capacity(f, "compute", usize::from(self.qid()))?; } if let Some(previous) = self.previous.as_ref() { - previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + previous + .context + .write_capacity(f, "compute_previous", usize::from(self.qid()))?; } Ok(()) } @@ -2262,21 +2735,66 @@ impl Queue { /// Logical queue/context ownership is unchanged. The existing binding is /// retained through the successor's exact replacement witness. pub(crate) fn share_idle(&mut self) -> Result { - if !self.installed || self.owner.is_none() || !self.retirement_ready - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.previous.is_some() || self.pool_phase != PoolPhase::Populated - { + if !self.shareable() { return Ok(false); } self.release_owner()?; Ok(self.reusable()) } + + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.claims == 0 + && self.installed + && self.owner.is_some() + && self.retirement_ready + && !self.growing + && !self.quarantined + && !self.awaiting_witness + && self.exit.is_none() + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } } impl Queue { pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + self.pool_phase == PoolPhase::Vacant + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } } @@ -2286,19 +2804,35 @@ impl Queue { /// caller validates the current preparation epoch and cancels only a true /// result's never-published QID; installed rings and identities stay intact. pub(crate) fn rollback_unpublished( - &mut self, owner: u64, context: &Arc, + &mut self, + owner: u64, + context: &Arc, ) -> Result<(bool, Option)> { - if self.owner != Some(owner) || !self.unpublished_binding - || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.retire_pending || self.failure_status_pending + if self.owner != Some(owner) + || !self.unpublished_binding + || self + .binding + .as_ref() + .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing + || self.quarantined + || self.awaiting_witness + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.retire_pending + || self.failure_status_pending || self.pool_phase != PoolPhase::Populated - || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + || self + .previous + .as_ref() + .is_some_and(|previous| previous.publication.is_some()) { return Ok((false, None)); } - if !self.installed { return Ok((true, None)); } + if !self.installed { + return Ok((true, None)); + } let mut retired = None; if let Some(previous) = self.previous.as_ref() { self.graph.set_owner(&previous.context)?; @@ -2318,3 +2852,83 @@ impl Queue { Ok((false, retired)) } } + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source( + &self, + source: &Arc, + status: &Arc, + epoch: u64, + ) -> bool { + !self.quarantined + && self.replays.len == 0 + && self.active.iter().any(|active| { + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source) + }) + } + + pub(crate) fn source_dependency( + &mut self, + source: &crate::g17::sync_input::Source, + status: &Arc, + epoch: u64, + ) -> Result> { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { + continue; + } + if !Arc::ptr_eq(active.packet.completion.status(), status) { + return Err(EINVAL); + } + if self.quarantined || self.replays.len != 0 { + return Err(EAGAIN); + } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { + return Err(EAGAIN); + } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { + return Err(ECANCELED); + } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new( + self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, + ) + .ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some(( + active.published, + dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), + hold, + ))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} + +/// Publication, dependency, stamp address, queue and the retained source hold. +pub(crate) type SourceDependency = ( + u64, + crate::g17::fw::kick::KickDependency, + u64, + u32, + crate::g17::sync_input::Hold, +); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index d7386e335e167..719a74787ca70 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,11 +83,12 @@ impl Transaction { } impl Queue { - /// A command with ray traversal scratch is never killed. On Neo, killing a + /// A running command with ray traversal scratch is never killed. On Neo, killing a /// traversal that stopped making progress either crashed the GPU firmware /// when the queue's pool was then released, or stalled every later render /// when it was retained; left alone, the rest of the GPU kept working. Such - /// a queue stays quarantined and retained until its commands retire. + /// a queue stays quarantined and retained until its commands retire. A + /// traversal the firmware reported failed no longer runs (see `fault_pending`). fn active_traversal(&self) -> bool { self.active.iter().any(|active| { matches!( @@ -101,7 +102,7 @@ impl Queue { && self.released && self.active.len != 0 && self.exit.is_none() - && !self.active_traversal() + && (!self.active_traversal() || self.fault_pending()) && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { @@ -110,6 +111,10 @@ impl Queue { CANCELLED_DELAY_NS } } + /// A kill of `context` was published through this queue. + pub(crate) fn exit_published(&self, context: &Arc) -> bool { + self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } pub(crate) fn exit_owner(&self, context: &Arc) -> Option { if !self .active @@ -124,6 +129,23 @@ impl Queue { .and_then(|active| active.ticket.map(|ticket| ticket.owner)) }) } + /// The kill of this queue's failed context waits; returns how long it has waited, and + /// whether this is the first time it crossed `slow`. + pub(crate) fn note_kill_wait(&mut self, now: u64, slow: u64) -> (u64, bool) { + let (since, warned) = self.kill_wait.get_or_insert((now, false)); + let waited = now.saturating_sub(*since); + let crossed = waited >= slow && !*warned; + *warned |= crossed; + (waited, crossed) + } + /// How long the kill of this queue's failed context has been waiting, if it is. + pub(crate) fn kill_waiting(&self, now: u64) -> Option { + self.kill_wait.map(|(since, _)| now.saturating_sub(since)) + } + /// The kill was published: how long it waited, if it did. + pub(crate) fn take_kill_wait(&mut self, now: u64) -> u64 { + self.kill_wait.take().map_or(0, |(since, _)| now.saturating_sub(since)) + } pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { self.exit = Some(Transaction { cookie, @@ -162,6 +184,7 @@ impl Queue { return false; } exit.phase = Phase::Acknowledged; + crate::g17::trace::teardown(4, cookie, u64::from(self.qid()), 0); true } pub(crate) fn settle_exit( diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index e938ec566dbac..d7f821bbce68d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -52,6 +52,12 @@ impl Cursors { } } +/// Whether stamp `a` was stored after stamp `b` (both kick stamps, modulo wrap). +pub(crate) const fn stamp_newer(a: u32, b: u32) -> bool { + let delta = a.wrapping_sub(b); + delta != 0 && delta < 0x8000_0000 +} + /// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); @@ -66,15 +72,16 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. +/// before it retires on the record alone, counted from when its ioctl's work-state node drained. /// /// The firmware stores the stamp after the cache maintenance that follows the record, so the /// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. While other commands of the same submit ioctl are still running, the stamp -/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms -/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as -/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail -/// its VM. +/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on +/// the ioctl's work-state node has completed: while other commands of the same submit ioctl +/// still run, the stamp is not late, however long they run (a command buffer of a dozen render +/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the +/// node drains. Only a stamp still missing well past that is treated as never stored. The bound +/// stays far below the scheduler timeout, so such a command cannot fail its VM. pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; // A command that waits out the fallback must still retire well inside the scheduler timeout. @@ -86,15 +93,46 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) } +/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StampWait { + /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. + Waiting, + /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not + /// late, and the wait counts again from the given host time. + Restart(u64), + /// The node drained and the stamp is still missing after the bound: retire on the record. + Aged, +} + +/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and +/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the +/// command's work-state node completed; it is asked only once the bound has passed. +pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) + -> StampWait +{ + if !record_aged(observed_at, now) { + StampWait::Waiting + } else if drained() { + StampWait::Aged + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(crate) fn stampless_total() -> u64 { + STAMPLESS.load(core::sync::atomic::Ordering::Relaxed) +} + /// Counts a command that retired on its record because its completion stamp never arrived; /// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: lifetime total {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, STAMP_FALLBACK_NS / 1_000_000, qid, @@ -106,7 +144,9 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery +/// generation. Recovery may consume items without completing their writes, so an old record +/// and consumed items cannot replace the stamp across a restart. In both cases its items /// must be consumed. #[expect( clippy::too_many_arguments, @@ -125,7 +165,9 @@ pub(crate) fn poll( if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let completed = stamp_covers(stamp, ticket.kick) + || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } @@ -161,35 +203,122 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, + unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 + } + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -201,65 +330,119 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) + } + + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; if count == 0 { - return Ok(()); + return Ok(!self.published_after_staging(qid, kick)); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(request.observed.is_some()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; } - let kick = - (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); + } + if request.observed.is_none() { + self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) } } @@ -306,6 +489,15 @@ mod tests { assert!(stamp_covers(0, 0xff_ffff)); } + #[test] + fn a_reported_fault_stamp_is_newer_only_until_stored() { + assert!(stamp_newer(own(KICK), own(KICK - 1))); + assert!(!stamp_newer(own(KICK), own(KICK))); + assert!(!stamp_newer(own(KICK - 1), own(KICK))); + // Across the wrap of the 24-bit kick in the stamp word. + assert!(stamp_newer(own(0), own(0xff_ffff))); + } + #[test] fn a_late_stamp_is_waited_for() { // Under games on J700 the stamp lands up to about 4 ms after the record while the @@ -374,6 +566,22 @@ mod tests { assert_eq!(result, Ok(Some([0x1111, 0x2222]))); } + #[test] + fn a_stamp_waits_for_the_rest_of_its_ioctl() { + let at = 1_000_000; + let late = at + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); + assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), + StampWait::Waiting); + // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. + assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // The node drained and the stamp is still missing after the bound. + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); + assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); + } + #[test] fn an_aged_record_still_waits_for_consumed_items() { let record = Some(Observation { timestamps: [10, 20] }); @@ -386,6 +594,30 @@ mod tests { assert_eq!(result, Ok(None)); } + #[test] + fn recovery_cannot_turn_an_aged_record_into_a_release_witness() { + let record = Some(Observation { timestamps: [10, 20] }); + // Recovery can advance the consumer without completing cache maintenance. + // Even a record observed before the restart and aged past the fallback + // must retain its command's mappings until a real stamp covers it. + for stamp in [0, own(KICK - 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + } + // A real stamp restores the release witness; completion's interrupted + // classification independently prevents false successful fences. + for stamp in [own(KICK), own(KICK + 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(Some([10, 20]))); + } + let busy = Cursors { producer: 4, consumer: 2, count: 64 }; + assert_eq!(poll(&ticket(), record, busy, own(KICK), + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + // The existing same-generation fallback remains available. + assert_eq!(poll(&ticket(), record, consumed(), own(KICK - 1), + Recordless::Allowed, 0, None, true), Ok(Some([10, 20]))); + } + #[test] fn a_zero_duration_record_fails() { let record = Some(Observation { timestamps: [5, 5] }); @@ -402,3 +634,156 @@ mod tests { assert_eq!(result, Err(ENODATA)); } } + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t, false).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 6a9b15789339f..b7037a1c35f75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,8 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + dependency::FailureRecord, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -27,7 +28,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -70,7 +70,7 @@ impl Frontend { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, @@ -85,6 +85,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -96,13 +97,19 @@ pub(crate) struct Queue { proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], - failures: [Option<(u64, Fence)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, + failures: [FailureRecord; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -111,6 +118,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -123,6 +131,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -134,7 +143,8 @@ impl Queue { proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), - failures: [None, None], + finished: KVec::new(), + failures: [FailureRecord::new(), FailureRecord::new()], _slot: slot, }) } @@ -155,16 +165,25 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } - // One earliest failed ordinal covers every later prefix without - // retaining its packet, mappings, or every failed fence. - if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { - *failure = Some((*sequence, fences.completed.clone())); + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); } + failure.note(*sequence, status, fences); false }); } @@ -179,7 +198,7 @@ impl Queue { command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -189,22 +208,62 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + let Validated::Compute { scratch, .. } = &command else { + return Err(EINVAL); + }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + if engine == Engine::Compute { + super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); + } + let sources = if engine == Engine::Compute + && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || fences.status() > 0 + }) + && self.failures.iter_mut().enumerate().all(|(index, failure)| { + failure.earliest().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; + if engine == Engine::Compute { + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); + } + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -216,9 +275,12 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), + sources.clone(), + self.usage.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -249,14 +311,17 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); - let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { - self.failures[index].as_ref().and_then(|(sequence, fence)| { - Order::contains(order.wait_through[index], *sequence).then_some(fence) - }) - }); + let mut failures: [Option; 2] = [None, None]; + for (index, failure) in self.failures.iter_mut().enumerate() { + failures[index] = failure.earliest().and_then(|(sequence, fences)| { + Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone()) + }); + } let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -277,7 +342,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -294,13 +359,26 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), @@ -314,10 +392,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -384,8 +463,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -399,22 +480,28 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { + self.context.note_render(); + } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -429,6 +516,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fb6c77fbdf04f..2436cfd33ed35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -78,12 +83,13 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; @@ -136,6 +142,8 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, + /// First observation of this group executed and consumed while its stamps lagged. + stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -186,6 +194,9 @@ pub(crate) struct Pair { /// An owner change in flight: the previous scheduler page and the ordinal of /// the new owner's first command, until that command leaves the pair. switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] @@ -237,8 +248,7 @@ impl Pair { let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = - kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -284,6 +294,7 @@ impl Pair { context_update: true, executed: 0, switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -324,7 +335,11 @@ impl Pair { self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self + .replay + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { return Err(EIO); } self.replay.pop(); @@ -351,14 +366,20 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) + || self + .replay + .front() + .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, attachments, ray_query, .. + pass, + attachments, + ray_query, + .. } = &packet.command else { return Err(EINVAL); @@ -368,6 +389,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); @@ -452,11 +474,9 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = self.memory.write_descriptors( - &args, - attachments, - packet.context.id().try_into()?, - ); + let result = + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -484,7 +504,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -495,7 +515,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -521,6 +541,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -571,12 +596,16 @@ impl Pair { let stamps = host.stamps(qids)?; fence(Ordering::Acquire); let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory - .completion_links(ordinal, kicks, payload, !covered[1])?; + self.memory.completion_links( + ordinal, + kicks, + payload, + self.variant.fragment_busy && !covered[1], + )?; let leads = match self.variant.lead_commands { true => { let init_manager = !self.memory.manager.initialized(); - self.memory.write_leads(ordinal, kicks[0], init_manager)? + self.memory.write_leads(ordinal, init_manager)? } false => [false; 2], }; @@ -637,10 +666,28 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -649,7 +696,10 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, ray_query, .. } = &packet.command else { + let Validated::Render { + pass, ray_query, .. + } = &packet.command + else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -752,10 +802,7 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); @@ -780,6 +827,7 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, + stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -790,10 +838,12 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; @@ -840,6 +890,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -848,6 +901,7 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -903,7 +957,11 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + if self + .registered + .is_some_and(|class| class != policy.priority()) + && self.in_flight() + { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -941,6 +999,12 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish( + crate::g17::trace::RENDER, + self.qids()[1], + self.ordinal as u32, + packet, + ); host.note_submission() })(); if let Err(error) = publish { @@ -993,12 +1057,124 @@ impl Pair { }, ] } + /// Bounded first-fault evidence while the recovery owner has firmware + /// halted, before it acknowledges state 1. Never changes retirement. + pub(crate) fn report_recovery(&self, stamps: Result<[u32; 2]>) { + pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", + self.slot, self.qids(), self.active.len(), self.prepared.is_some(), + self.quarantined, stamps, self.free_list.counters()); + self.memory.manager.report_recovery(); + for active in self.active.iter().take(4) { + if let Validated::Render { pass, .. } = &active.packet.command { + pr_warn!("G17: recovery render geometry ordinal={} width={} height={} layers={} layout={:?}\n", + active.ticket.ordinal, pass.width, pass.height, pass.layers, + target::Layout::new(pass, self.clusters, self.variant.programs)); + } + pr_warn!( + "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, + active.outer_published, + active.packet.context.id(), + active.packet.order.sequence, + observation::snapshot(&self.memory, active.ticket.ordinal) + ); + } + } + + /// Latches when the front group executed and the firmware consumed its ring entries while + /// `stamps` still lag it, and reports whether that has held for + /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it + /// on its pass record instead of waiting for stamps no later event will move. + pub(crate) fn note_stamp_wait( + &mut self, + stamps: [u32; 2], + state: u32, + now: u64, + ) -> Result { + if self.quarantined || self.prepared.is_some() || state != 0 { + return Ok(false); + } + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { + front.stamp_wait_since = None; + return Ok(false); + } + let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; + if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { + front.stamp_wait_since = None; + return Ok(false); + } + let since = *front.stamp_wait_since.get_or_insert(now); + Ok(Self::stamp_wait_expired(front, since, now)) + } + + /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks + /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair + /// stays latched for the rescan. + fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { + let drained = || { + // A node that cannot be read keeps the bound from the latch. + front + .packet + .completion + .work_state() + .and_then(|work| work.drained()) + .unwrap_or(true) + }; + match retirement::stamp_wait(since, now, drained) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + front.stamp_wait_since = Some(now); + false + } + retirement::StampWait::Expired => true, + } + } + + /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair + /// without a completion notification, which may never come for such a group. + pub(crate) fn stamp_waiting(&self) -> bool { + self.active + .front() + .is_some_and(|front| front.stamp_wait_since.is_some()) + } + + /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed + /// it and consumed its ring entries while the stamps lag, it retires on its pass record now + /// instead of failing. Failing would give up on the stamps as well. + pub(crate) fn expire_stamp_wait( + &mut self, + packet: &Arc, + stamps: [u32; 2], + state: u32, + ) -> Result { + if !self + .active + .front() + .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) + { + return Ok(false); + } + if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { + return Ok(false); + } + if let Some(front) = self.active.front_mut() { + front.stamp_wait_since = Some(0); + } + Ok(true) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { - masks.is_none_or(|masks| { - self.qids() - .into_iter() - .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) - }) + masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } /// The oldest published group still running, for cross-VM timeout attribution /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains @@ -1048,7 +1224,9 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory.manager.note_required(required, crate::g17::now_ns()); + self.memory + .manager + .note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1073,6 +1251,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1089,7 +1289,10 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + let ordinal = self + .active + .iter() + .map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1101,6 +1304,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1120,7 +1324,11 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { + if self + .active + .iter() + .any(|active| active.packet.completion.spared()) + { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1187,6 +1395,9 @@ impl Pair { { return Err(EIO); } + let stampless = front + .stamp_wait_since + .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); let mut observer = Observer { memory: &self.memory, qids, @@ -1196,7 +1407,13 @@ impl Pair { if observed.ticket != front.ticket { return Err(EIO); } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) + } else { + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } + } else if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) } else { retirement::poll(&mut front.ticket, &mut observer) }; @@ -1211,6 +1428,9 @@ impl Pair { }; let ordinal = front.ticket.ordinal; let context_update = front.context_update; + if stampless { + retirement::note_stampless(u16::from(self.slot), ordinal); + } self.note_retirement(ordinal, context_update); let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { @@ -1223,6 +1443,14 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::recovery::note_retired(); + crate::g17::trace::complete( + crate::g17::trace::RENDER, + qids[1], + ordinal as u32, + &front.packet, + [timestamps[0], timestamps[3]], + ); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1254,6 +1482,9 @@ impl Pair { if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } use crate::g17::replay::{self, Decision, Group}; let qids = self.qids(); let memory = &self.memory; @@ -1264,10 +1495,9 @@ impl Pair { let blamed = active.packet.completion.blamed(); let exhausted = active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is not abandoned, an error included. - let abandoned = innocent - && !blamed - && (|| -> Result { + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { let mut observer = Observer { memory, qids, @@ -1278,20 +1508,43 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. - Ok(matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) + let witness = |executed| { + matches!( + retirement::abandoned( + &active.ticket, + snapshot, + stamps, + state, + executed + ), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok(( + unexecuted && !lost, + !unexecuted && witness(true), + unexecuted && lost, )) })() .unwrap_or_else(|error: Error| { failure = Some(error); - false - }); + (false, false, false) + }) + } else { + (false, false, false) + }; Group { innocent, blamed, abandoned, exhausted, + ran, + lost, } }); let decision = replay::decide(groups); @@ -1306,7 +1559,7 @@ impl Pair { front.abandoned_since = None; return Ok(false); } - Decision::Replay => {} + Decision::Replay | Decision::Fail => {} } if grace { let Some(since) = front.abandoned_since else { @@ -1317,6 +1570,9 @@ impl Pair { return Ok(false); } } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } // decide() lets only the front be exhausted. An exhausted front fails, and its // successors, which consume its output, fail with it rather than run without it. let mut failing = self.active.front().is_some_and(|front| { @@ -1369,11 +1625,126 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); + self.published_completions = None; self.replay.push(packet).map_err(|_| EIO)?; } Ok(true) } + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + Ok(true) + } + + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); + } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; + let mut observer = Observer { + memory: &self.memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } + } + if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(Some(true)) + } + /// The pending replay cannot proceed (closed owner, cancellation or a /// publication refusal). Its successors cannot run without it and fail with /// it. ECANCELED leaves the VM usable; any other error is the ordinary @@ -1415,7 +1786,44 @@ impl Pair { let counters = self.free_list.counters()?; let stamps = host.stamps(self.qids())?; let (_, state) = host.epoch()?; - Ok(retirement::drained(self.ordinal, stamps, counters, state)) + Ok(retirement::drained(self.ordinal, stamps, counters, state) + || (state == 0 && self.executed_unstamped() && self.context_killed())) + } + + /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer + /// walks that context's work list, so the commands it never unlinked can be released. + fn context_killed(&self) -> bool { + crate::g17::queue::compute::exit::State::read(&self.memory.context) + .is_ok_and(|state| state.killed != 0) + } + + /// The pair's context may be killed although its commands were not retired: they all ran + /// (`executed_unstamped`), so the kill leaves nothing queued for the firmware to run. + /// Its memory stays until the kill is acknowledged. + pub(crate) fn executed_for_kill(&self, context: &Arc) -> bool { + Arc::ptr_eq(context, &self.memory.context) && self.executed_unstamped() + } + + /// Every command of this quarantined pair ran, its ring entries were consumed and its + /// free-list blocks returned, yet the firmware never stores the stamp of the last one. + /// On Neo this is the render work of a context whose compute command failed: it runs + /// about a second after the failure, but its stamp never arrives and it stays linked in + /// the firmware's job list, so neither the stamp proof nor the abandoned-group proof + /// ever holds and the pair would stay quarantined, keeping its context, until the + /// processors stop. + fn executed_unstamped(&self) -> bool { + self.quarantined + // Only for a closed owner: a live owner would go on publishing into a context + // whose work list still links the unstamped node. + && self.owner.is_none() + && self.prepared.is_none() + && self.active.len() != 0 + && self.free_list.counters().is_ok_and(|counters| counters[0] == counters[1]) + && self.active.iter().all(|active| { + active.outer_published + && observation::snapshot(&self.memory, active.ticket.ordinal) + .is_ok_and(retirement::Snapshot::executed_and_consumed) + }) } /// Account one witnessed tracker, preserving all completion ownership. @@ -1429,6 +1837,10 @@ impl Pair { let qids = self.qids(); let qos = self.qos_owners(); let buffer = self.buffer_id(); + // Commands the firmware never unlinked are released only after it acknowledged the + // kill of their context, on every path (see `firmware_drained`). + let unstamped = self.executed_unstamped(); + let killed = unstamped && self.context_killed(); let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { return Ok(false); }; @@ -1439,6 +1851,12 @@ impl Pair { }; let result = match retirement::poll(&mut active.ticket, &mut observer)? { retirement::Poll::Complete(ticks) => Ok(ticks), + // Its commands ran without a stamp (see `executed_unstamped`): they already failed + // with the quarantine, and the client sees them cancelled. + retirement::Poll::Pending | retirement::Poll::Retry if killed => { + crate::g17::trace::teardown(2, u64::from(self.slot), active.ticket.ordinal, 0); + Err(ECANCELED) + } retirement::Poll::Pending | retirement::Poll::Retry => { core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); let snapshot = observer.snapshot(active.ticket.ordinal)?; @@ -1515,15 +1933,13 @@ impl Pair { // own even with successors queued, so the pair is not quarantined into a sticky EIO. It // fails with ECANCELED, which spared work never reports as device loss; consecutive // abandoned spared successors are retired the same way. - let front_spared = self.active.front().is_some_and(|a| { - Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() - }); + let front_spared = self + .active + .front() + .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); if !self.quarantined && (self.active.len() == 1 || front_spared) { let mut retired = false; - loop { - let Some(active) = self.active.front() else { - break; - }; + while let Some(active) = self.active.front() { let spared = active.packet.completion.spared(); if retired && !spared { break; @@ -1770,6 +2186,71 @@ impl Pair { } /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + false, + )?; + for dependency in frontier.as_slice() { + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) } @@ -1777,8 +2258,12 @@ impl Pair { /// not left the pair yet. pub(crate) fn switch_in_flight(&self) -> Option { let (from, first) = self.switch?; - (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) - .then_some(from) + (self.replay.len() != 0 + || self + .active + .iter() + .any(|active| active.ticket.ordinal == first)) + .then_some(from) } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { @@ -1943,7 +2428,9 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { self.graph_started = false; } + if foreign { + self.graph_started = false; + } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -1952,3 +2439,45 @@ impl Pair { Ok(()) } } + +/// Whether `masks`, the stamp notifications collected for a pair whose tiling and fragment +/// queues are `qids`, can mean a finished pass: the fragment queue's bit. A fragment kick starts +/// only after its fragment barrier saw the tiling stamp, and retirement still checks both stamps +/// in memory, so the tiling notification is not needed to retire. +pub(crate) fn completion_noted(qids: [u8; 2], masks: [u64; 2]) -> bool { + let fragment = usize::from(qids[1]); + masks[fragment / 64] & (1u64 << (fragment % 64)) != 0 +} + +#[kunit_tests(asahi_g17_render_completion_noted)] +mod completion_noted_tests { + use super::completion_noted; + + fn bit(qid: u8) -> [u64; 2] { + let mut masks = [0; 2]; + masks[usize::from(qid) / 64] |= 1u64 << (qid % 64); + masks + } + + #[test] + fn fragment_notification_alone_selects() { + assert!(completion_noted([3, 4], bit(4))); + assert!(completion_noted([70, 100], bit(100))); + } + + #[test] + fn tiling_notification_alone_does_not_select() { + assert!(!completion_noted([3, 4], bit(3))); + assert!(!completion_noted([70, 100], bit(70))); + assert!(!completion_noted([3, 4], [0; 2])); + } + + #[test] + fn other_queues_do_not_select() { + assert!(!completion_noted([3, 4], bit(5))); + assert!(!completion_noted([3, 68], bit(4))); + let mut both = bit(3); + both[1] |= bit(68)[1]; + assert!(completion_noted([3, 68], both)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 80ee0b871cc82..90a11ca977684 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,8 +31,8 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; -/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the -/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +/// The lead command of a stage (InitBM before tiling), after the stage's configuration record +/// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. @@ -528,15 +528,44 @@ impl Memory { } const _: () = { + // Every unretired render of a pair holds its own command record. + assert!(DEPTH <= RECORDS); assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); assert!(size_of::() <= LEAD); - assert!(LEAD + size_of::() <= CONFIG_STRIDE); assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -571,21 +600,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, @@ -629,14 +647,8 @@ impl Memory { } /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the - /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages - /// have one. - pub(super) fn write_leads( - &mut self, - ordinal: u64, - tiling_kick: fw::kick::KickTimestamp, - init_manager: bool, - ) -> Result<[bool; 2]> { + /// tiling descriptor when `init_manager`. Returns which stages have one. + pub(super) fn write_leads(&mut self, ordinal: u64, init_manager: bool) -> Result<[bool; 2]> { if !self.owns(ordinal) { return Err(EIO); } @@ -645,23 +657,10 @@ impl Memory { .descriptors .pointer(Graph::offset(0, ordinal), size_of::())? .cast::(); - let fragment = self - .graph - .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? - .cast::(); - // SAFETY: Both views were checked during preparation and the ordinal still owns them; - // only integer fields are read. - let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + // SAFETY: The view was checked during preparation and the ordinal still owns it; only + // integer fields are read. + let ta = unsafe { &*ta }; let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; - let barrier = queue::FragmentBarrier::new( - ta.stamp_va, - tiling_kick, - ta.qid, - ta.stamp_value, - fragment_id, - ); - self.graph.config.write(CONFIG[1] + slot, barrier)?; if init_manager { let init = queue::InitBufferManager::new( ta.context_id, @@ -671,7 +670,7 @@ impl Memory { ); self.graph.config.write(CONFIG[0] + slot, init)?; } - Ok([init_manager, true]) + Ok([init_manager, false]) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 209ce0b8176ec..81d4306eda28c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -252,6 +252,7 @@ impl ParkedPair { context_update: true, executed: self.executed, switch: None, + published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 9f4cb05b6bede..d65c40c7cfade 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 @@ -41,6 +41,10 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { + #[expect( + clippy::too_many_arguments, + reason = "The publication lane describes independent fields of the firmware wire record." + )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -94,7 +98,9 @@ impl<'a> Lane<'a> { push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph + .queues + .word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 24aeed6bcd0ec..4377bbfe3e311 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,17 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } + + /// The command ran (tiler start and fragment end written) and the firmware consumed both + /// queues' ring entries. + pub(crate) fn executed_and_consumed(self) -> bool { + self.executed() && self.cursors.iter().all(|cursors| cursors.done == cursors.write) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -144,6 +155,85 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result bool) -> StampWait { + if now.saturating_sub(since) < STAMP_FALLBACK_NS { + StampWait::Waiting + } else if since == 0 || drained() { + StampWait::Expired + } else { + StampWait::Restart(now) + } +} + +static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Counts a render group that retired on its pass record because its completion stamps never +/// covered it; logs the first and every 64th. +pub(crate) fn note_stampless(slot: u16, ordinal: u64) { + let count = STAMPLESS.fetch_add(1, Ordering::Relaxed) + 1; + if count == 1 || count % 64 == 0 { + pr_info!( + "G17: lifetime total {} render groups retired on their pass record after {} ms without completion stamps (latest: slot {} ordinal {})\n", + count, + STAMP_FALLBACK_NS / 1_000_000, + slot, + ordinal + ); + } +} + +/// Retire an executed, consumed front group without its stamps, once [`STAMP_FALLBACK_NS`] +/// has passed. The caller decides that bound; this re-checks execution and consumption. Unlike +/// the stamped path, it has no post-maintenance evidence; see [`STAMP_FALLBACK_NS`]. +pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { + fence(Ordering::Acquire); + let snapshot = memory.snapshot(ticket.ordinal)?; + let Ok(observation) = observe(ticket, snapshot) else { + return Ok(Poll::Retry); + }; + if !observation.complete { + return Ok(Poll::Pending); + } + let [start, end] = memory.pass(ticket.ordinal)?; + Ok(Poll::Complete([start, start, start, end])) +} + /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -350,3 +440,94 @@ impl Tracker { .filter_map(Option::as_mut) } } + +#[kunit_tests(asahi_g17_render_retirement)] +mod executed_tests { + use super::*; + + fn snapshot(done: u32, write: u32, pass: [u64; 2]) -> Snapshot { + Snapshot { + cursors: [Cursors { done, read: done, write }; 2], + job_list_empty: false, + pass, + } + } + + #[test] + fn executed_and_consumed_needs_both_passes_and_consumed_rings() { + assert!(snapshot(3, 3, [1, 2]).executed_and_consumed()); + assert!(!snapshot(3, 3, [0, 0]).executed_and_consumed()); + assert!(!snapshot(3, 3, [1, 0]).executed_and_consumed()); + assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); + assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); + } + + /// The firmware state of a stalled pair as read on hardware: the front group executed and + /// its two ring entries per queue were consumed, but both stamps hold the previous kick. + struct Stalled { + snapshot: Snapshot, + stamps: [u32; 2], + } + impl Memory for Stalled { + fn snapshot(&mut self, _ordinal: u64) -> Result { + Ok(self.snapshot) + } + fn stamps(&mut self) -> Result<[u32; 2]> { + Ok(self.stamps) + } + fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { + Ok(self.snapshot.pass) + } + } + + fn stalled_ticket() -> Ticket { + let window = Window::new(1097, 2, 1280).unwrap(); + let previous = Cursors { done: 1097, read: 1097, write: 1097 }; + Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } + } + + fn stalled(pass: [u64; 2]) -> Stalled { + let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; + Stalled { + snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, + stamps: [406 << 8; 2], + } + } + + #[test] + fn lagging_stamps_keep_an_executed_group_pending() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stampless_poll_retires_an_executed_consumed_group() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); + } + + #[test] + fn stampless_poll_waits_for_the_fragment_end() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 0]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stamps_wait_for_the_rest_of_the_ioctl() { + let since = 1_000_000; + let late = since + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); + // Other kicks of the group's ioctl still run: the bound restarts. + assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // A drained node with lagging stamps is the stall the bound exists for. + assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); + // The watchdog's last resort does not wait for the node. + assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), + StampWait::Expired); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 89f8d33bdb80c..ca8da8b03545f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,6 +10,7 @@ mod attribution; mod slot; mod safety; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -24,20 +25,34 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -47,9 +62,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -71,6 +99,12 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; + /// Optional diagnostics only; absence cannot alter attribution or release. + fn report_blamed_render(&self, _qid: u8) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -85,7 +119,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -119,7 +155,9 @@ impl State { restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted @@ -192,6 +230,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -203,6 +243,15 @@ pub(super) fn service( host.clear_timestamps()?; } let mut sources = host.fault_sources()?; + sources.report(request.generation, super::queue::compute::retirement::stampless_total()); + if let Some(qid) = request.blamed { + host.report_blamed_render(qid); + } + // The keys, monitor and pass-start evidence must describe the same halted + // instant. Firmware may update pass timestamps as soon as state 3 is + // acknowledged; sampling them during classification can blame a bystander. + let attribution = sources.attribute(request.blamed, + |qid, stamp| host.render_pass_started(qid, stamp)); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; @@ -217,17 +266,31 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); + let (cancelled, blame) = classify(host, entries, request.blamed, attribution); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { pr_warn!( "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", @@ -247,16 +310,35 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. /// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed /// render that had not started while others had (or with no culprit at all) is innocent, and its /// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { +fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; - let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} Attribution::Queues(qids) => guilty_qids = qids, @@ -316,3 +398,101 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }); (cancelled & !(guilty_qids & !collateral) & spared, blame) } + + +#[kernel::macros::kunit_tests(asahi_g17_recovery_snapshot)] +mod tests { + use super::*; + use core::cell::Cell; + + struct Halted { + recovery: State, + state: Cell, + reads: Cell, + known_keys: bool, + blamed: bool, + } + + impl Halted { + fn new(known_keys: bool) -> Self { + let mut recovery = State::new(); + assert!(recovery.request(0, 38).unwrap()); + Self { recovery, state: Cell::new(1), reads: Cell::new(0), + known_keys, blamed: false } + } + } + + impl Memory for Halted { + fn recovery_state(&self) -> Result { Ok(self.state.get()) } + fn host_recovery(&self) -> Result { Ok(0) } + fn fault_sources(&self) -> Result { + assert_eq!(self.state.get(), 1); + let mut sources = Sources::sample(3, |slot| { + Some((if self.known_keys { ((38 + slot as u64) << 40) | 1 } else { 0 }, 0)) + }); + sources.reason = Some(2); + Ok(sources) + } + fn transition(&self, expected: u32, next: u32) -> Result { + assert_eq!(self.state.get(), expected); + self.state.set(next); + Ok(()) + } + fn clear_timestamps(&self) -> Result { panic!("active render scheduler") } + fn entry(&self, _index: usize) -> Result<(u32, u32)> { + assert_eq!(self.state.get(), 3); + Ok((0, 0)) + } + fn wait_tick(&self) { + assert_eq!(self.state.get(), 2); + self.state.set(3); + } + } + + impl Queues for Halted { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, qid: u8, _stamp: u64) -> Option { + // This is the regression oracle: state 0/2/3 is too late. A + // resumed peer may have started after the fault we are attributing. + assert_eq!(self.state.get(), 1); + self.reads.set(self.reads.get() + 1); + Some(qid == 39) + } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, _qid: u8) -> Option { None } + fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } + fn settle_blamed_render(&mut self, qid: u8, _error: Error) { + assert_eq!(qid, 38); + self.blamed = true; + } + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + } + + impl Host for Halted { + fn recovery(&self) -> &State { &self.recovery } + fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } + } + + #[test] + fn pass_evidence_is_frozen_before_the_first_acknowledgement() { + let mut host = Halted::new(true); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 2); + assert_eq!(host.state.get(), 0); + assert_eq!(host.recovery.generation(), 1); + assert!(!host.blamed); + } + + #[test] + fn unknown_25g83_keys_still_preserve_firmware_blame() { + let mut host = Halted::new(false); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 0); + assert!(host.blamed); + assert_eq!(host.state.get(), 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index c30e45e22579a..d71d3cc81553f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -2,17 +2,52 @@ //! Fail closed on repeated watchdog/restart traffic, even across client loss. +use core::sync::atomic::{AtomicU64, Ordering}; + const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + pub(super) struct Storm { start: Option, count: u8, stopped: bool, + progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false } + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) } pub(super) fn note(&mut self, now: u64) -> bool { if self.stopped { return true; } @@ -74,11 +109,23 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); assert_eq!(replay_status(1), Ok(())); assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED)); } } diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs index 94daa5ec0dfeb..7668871376bad 100644 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -16,6 +16,12 @@ pub(crate) struct Group { pub(crate) abandoned: bool, /// Already republished the maximum number of times. pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -25,26 +31,54 @@ pub(crate) enum Decision { Wait, /// Never replay this set; the render timeout settles it. Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, } /// Groups are read in order and the first one that decides stops the walk, so a caller can /// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its /// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. pub(crate) fn decide(groups: impl Iterator) -> Decision { let mut front_exhausted = None; let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; for group in groups { if !group.innocent || group.blamed { return Decision::Refuse; } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } if !group.abandoned { return Decision::Wait; } + dropped = true; match front_exhausted { None => front_exhausted = Some(group.exhausted), Some(_) => successor_exhausted |= group.exhausted, } } + if unordered { + return Decision::Fail; + } match front_exhausted { Some(true) => Decision::Replay, Some(false) if !successor_exhausted => Decision::Replay, @@ -56,10 +90,13 @@ pub(crate) fn decide(groups: impl Iterator) -> Decision { mod tests { use super::*; fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted } + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } } - const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; - const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; #[test] fn every_group_abandoned_and_innocent_is_replayed() { assert_eq!(decide([OK].into_iter()), Decision::Replay); @@ -106,4 +143,28 @@ mod tests { Decision::Replay ); } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6f4a5355d6eb3..d391fbb3001bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -31,9 +31,32 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -45,13 +68,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -62,6 +175,10 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -69,21 +186,34 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -97,12 +227,21 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -110,12 +249,16 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a @@ -131,15 +274,80 @@ impl Registry { Rebind::Fresh }); } - // Physical channels are a device budget, not a permanent entitlement - // of each open logical queue. Only exact idle retirement can yield one. - for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); + } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let shared = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .share_idle()?; + if !shared { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -159,9 +367,9 @@ impl Registry { } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -172,7 +380,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -184,7 +392,7 @@ impl Registry { pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -252,7 +460,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -282,7 +490,7 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -313,7 +521,7 @@ impl Registry { || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -328,14 +536,16 @@ impl Registry { .ok_or_else(|| self.compute_admission_error( crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) .map_err(|error| self.compute_admission_error(error))?; + self.note_build_room(); let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -367,7 +577,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -375,6 +585,7 @@ impl Registry { self.reserved(reservation)?; self.qids.cancel_unpublished(reservation.id)?; self.compute[reservation.slot] = None; + self.note_build_room(); Ok(()) } @@ -382,20 +593,32 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; + self.note_build_room(); } Ok(()) } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -436,12 +659,28 @@ mod uninstalled_cleanup_tests { /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -459,6 +698,7 @@ impl Backend { const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; let started = now_ns(); let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -466,6 +706,11 @@ impl Backend { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( @@ -493,7 +738,10 @@ impl Backend { return Err(ERESTARTSYS); } shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; } Err(error) => return Err(error), } @@ -506,7 +754,16 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new( + &alloc, + &pool, + &qos, + execution, + owner_pid, + policy, + shared.renders.clone(), + shared.progress.clone(), + )?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -514,15 +771,21 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), - }, + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), + }), GFP_KERNEL, )?, slot, @@ -592,7 +855,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -652,7 +915,7 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } @@ -667,11 +930,14 @@ impl Backend { drop(queue); let mut state = self.shared.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation) + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) } fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - use kernel::time::{delay::fsleep, Delta}; admission.check()?; { let state = self.shared.state.lock(); @@ -689,6 +955,9 @@ impl Backend { let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); admission.check()?; @@ -696,7 +965,7 @@ impl Backend { if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -714,9 +983,9 @@ impl Backend { match result { Ok(old) => { drop(old); return Ok(()); } Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } })(); @@ -727,8 +996,9 @@ impl Backend { if let Some(firmware) = state.as_deref_mut() { for index in 0..firmware.queues.compute.len() { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(queue) = entry.queue.as_deref_mut() else { continue; }; - if queue.abort_growth(token)? == Some(true) { + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { // Keep the empty entry and QID reserved until all // graph aliases have been destroyed outside the lock. retired = entry.queue.take().map(|queue| (entry.reservation, queue)); @@ -762,9 +1032,11 @@ impl Backend { } loop { admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -780,7 +1052,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -788,16 +1060,17 @@ impl Backend { match rebound { Rebind::Done => return Ok(()), Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -811,7 +1084,7 @@ impl Backend { } Err(error) => return Err(error), }; - (firmware.uat.clone(), Ok(reservation)) + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -826,12 +1099,12 @@ impl Backend { // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc)?)) + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -881,6 +1154,7 @@ impl Backend { self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new( &alloc, + &objects, backing, self.owner, self.context.clone(), @@ -920,7 +1194,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -932,6 +1206,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -967,6 +1242,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1020,10 +1296,18 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1035,11 +1319,33 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1060,15 +1366,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1085,54 +1391,151 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + self.port.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.port.notify(message) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() @@ -1150,26 +1553,42 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -1181,10 +1600,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1196,7 +1614,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -1223,33 +1641,53 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( + registry.completion_cursor, + seen, + |record| observations.fold(record), + )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.latch_observations(&mut registry.observations, masks)?; + let busy = !queue.channel_free(); + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1257,26 +1695,37 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } @@ -1286,18 +1735,17 @@ impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; @@ -1307,14 +1755,308 @@ impl Backend { result => return result, } } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted; if the + // admission deadline passes, the spared queue and its retained work stay + // intact until a later admission observes its witness. + admission.extend_for_recovery(); + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - admission.check()?; - Err(EBUSY) + } + + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } + + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { + return Err(render::FALLBACK); + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } + } + } + + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) } /// Called after taking this packet's publication claim. All waits and work @@ -1326,8 +2068,17 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let mut recovery_polls = 0; - let mut state_delay = 20i64; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. + let mut bound = true; + let mut demand = None; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1336,24 +2087,37 @@ impl Backend { Ok(()) }; let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { admission.check()?; + let observed = progress.observe(); + if !self.sources_published(packet)? { + let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { + firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + }); + if recovering { + admission.extend_for_recovery(); + } + self.shared.queue_events(); + self.wait_release(progress, observed, &admission)?; + continue; + } // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; + if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. - // Wait for one under the packet's admission deadline, as render - // publication does, instead of failing the packet. - match packet.completion.work_state()?.acquire() { + match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { Err(error) if error == EBUSY => { drop(lease); - admission.check()?; self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); + self.wait_release(progress, observed, &admission)?; continue; } result => { @@ -1376,18 +2140,18 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; + admission.extend_for_recovery(); drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; - if firmware.queues.compute(self.owner).is_none() { + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; drop(state); drop(lease); continue; @@ -1400,29 +2164,27 @@ impl Backend { dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + admission.extend_for_recovery(); } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; + if result.is_ok() { + self.remember_direct(&firmware.queues); } return result; } @@ -1435,7 +2197,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1491,6 +2253,7 @@ impl super::Shared { dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); return; } + self.forget_port(); let firmware = self.state.lock().take(); if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } } @@ -1504,6 +2267,7 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -1543,20 +2307,41 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + let mut freed = false; + let mut releasing = false; + for mut queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.pool_release_stalled(now)) { return Err(ETIMEDOUT); @@ -1568,6 +2353,11 @@ impl super::Firmware { // retained compute pool stays warm: render and new compute need it more. let cap = if super::freelist::lower_pressure() { 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS } else { COMPUTE_WARM_CAP }; @@ -1594,12 +2384,19 @@ impl Registry { writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; self.render.write_capacity(f)?; + let now = now_ns(); + let waits = || self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.kill_waiting(now)); + compute_exit::write_kill_waits(f, waits().count(), waits().max().unwrap_or(0))?; + compute_exit::write_render_kills(f, self.teardown.kill_waits(now).count(), + self.teardown.kill_waits(now).max().unwrap_or(0))?; writeln!(f, "compute_slots used={} total={} reusable={} building={}", self.compute.iter().flatten().count(), self.compute.len(), self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} total={}", - self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { queue.write_capacity(f)?; @@ -1609,13 +2406,42 @@ impl Registry { } impl Backend { - pub(super) fn wait_capacity(&self, started: u64) -> Result { - let check = || self.admission_open(); - let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission) + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() } - fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -1630,14 +2456,15 @@ impl Backend { if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { return Ok(()); } - for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; - } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - if !firmware.render_control_backpressured() { + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } } @@ -1649,8 +2476,6 @@ impl Backend { Err(error) if error == EAGAIN => {} result => result?, } - admission.check()?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); admission.check() } } @@ -1669,12 +2494,17 @@ mod shared_capacity_tests { ) .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - // Physical compute channels are device-owned and bounded independently - // of how many logical clients have queued unsignalled work. - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); @@ -1695,9 +2525,6 @@ mod shared_capacity_tests { GFP_KERNEL, ) .unwrap(); - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { - registry.reserve_compute(owner).unwrap(); - } let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { @@ -1726,11 +2553,31 @@ mod shared_capacity_tests { leases.push(lease, GFP_KERNEL).unwrap(); } } - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert_eq!(leases.len(), 36); } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } } impl Backend { diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7d690470cd7b5..bf0f989f042a5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,14 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } @@ -48,6 +55,7 @@ impl Backend { drop(state); drop(binding); self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); self.shared.queue_events(); return Ok(()); } @@ -85,6 +93,7 @@ impl Backend { // A builder may own only an off-lock reservation when close arrives. // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -99,12 +108,63 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -156,8 +216,13 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute and render admission wait for their shared capacity here, as a + /// fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => self.admit_render(packet), + } } fn publish(&self, packet: Arc) -> Result { @@ -185,6 +250,9 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -213,6 +281,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -226,6 +297,16 @@ impl job::Backend for Backend { } } + fn long_job_beside(&self, _packet: &Arc) -> bool { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|other| other.owner() != Some(self.owner) && other.long_job_running()) + } + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); @@ -274,11 +355,14 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index c407d9576e9c4..4f519743aed53 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,11 +20,10 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; let computes = firmware.queues.compute.iter().flatten() @@ -64,7 +63,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -91,7 +90,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d71f410c76fb4..32b9acef94c0f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -90,7 +84,7 @@ impl Firmware { let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -110,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4fd6e56741986..3e779bba4fbbf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -15,6 +15,9 @@ use kernel::{prelude::*, sync::Arc}; /// Wake interval while a tiler heap may still shrink: the native check period. const SHRINK_WAKE_NS: u64 = 1_000_000_000; +/// Longest interval between two event passes that run every housekeeping step while +/// completion records alone keep the worker busy. +const HOUSEKEEPING_NS: u64 = 1_000_000; struct Service<'a> { firmware: &'a mut Firmware, @@ -35,16 +38,19 @@ impl event::Transport for Service<'_> { } fn statistics(&mut self, role: Role) { let firmware = &mut *self.firmware; + firmware.observe_statistics_power(); firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + firmware.observe_statistics_power(); } fn drain( &mut self, role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = channel::Events::new(role); + let mut events = + channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -72,6 +78,15 @@ impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { self.firmware.queues.acknowledge_compute_exit(cookie); } + fn compute_fault(&mut self, qid: u8, stamp: u32) { + for entry in self.firmware.queues.compute.iter_mut().flatten() { + if let Some(mut queue) = entry.queue.as_deref_mut() { + if queue.qid() == qid { + queue.note_fault(stamp); + } + } + } + } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() } @@ -131,8 +146,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -203,6 +218,27 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } + fn report_blamed_render(&self, qid: u8) { + self.firmware + .queues + .report_render_recovery(qid, &self.firmware.port); + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -225,19 +261,17 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed( - state, - |owner| registry.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; + queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { + deferred.push(event) + })?; } Ok(()) } @@ -266,7 +300,11 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } + self.rings.invalidate_statistics(); + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -276,7 +314,15 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service + .firmware + .port + .set_generation(service.firmware.recovery.generation()); + service.firmware.rings.invalidate_statistics(); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -319,8 +365,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -333,6 +381,10 @@ impl crate::g17::Shared { } fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { + // Taken before consuming records: a request made during this pass gets the next one. + let requested = self + .housekeeping + .swap(false, core::sync::atomic::Ordering::AcqRel); let (recover, masks) = { let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -348,8 +400,10 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { - self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations + .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -392,19 +446,44 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; - firmware.reclaim_drained_renders(deferred)?; - firmware.service_compute_exit(deferred)?; - let pools = firmware.service_compute_pools(deferred, now)?; + // Expiries, drained pairs, exits, pool and heap trimming and teardown give back what + // retirement freed; a completion needs none of them to retire or to signal. While work + // is in flight and nothing waits for capacity, a pass for firmware records alone skips + // them. Any requested pass, recovery, teardown, a waiter, the last retirement and, at + // the latest, the first pass after HOUSEKEEPING_NS run them all. + let housekeeping = requested + || recover + || !in_flight + || firmware.recovery.pending() + || !firmware.queues.teardown.is_empty() + || self.channels.has_waiters() + || self.progress.has_waiters() + || self.queues.has_waiters() + || self.renders.has_waiters() + || now.saturating_sub( + self.housekeeping_ns + .load(core::sync::atomic::Ordering::Relaxed), + ) >= HOUSEKEEPING_NS; + let mut lingers = None; + let mut pools = None; let mut shrinking = false; - if !firmware.recovery.pending() { - firmware - .queues - .vacate_evicted_render_pools(&firmware.init, deferred)?; - shrinking = firmware - .queues - .shrink_render_heaps(&firmware.init, now, deferred); - } - { + if housekeeping { + self.housekeeping_ns + .store(now, core::sync::atomic::Ordering::Relaxed); + if !firmware.recovery.pending() { + lingers = firmware.expire_render_lingers(now)?; + } + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + pools = firmware.service_compute_pools(deferred, now)?; + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); + } let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; if !reclaim.is_empty() { @@ -434,11 +513,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } @@ -453,13 +535,18 @@ impl crate::g17::Shared { { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. - self.queue_events(); + self.queue_event_records(); } let grow_ready = firmware.render_grow_ready(); drop(state); if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 59db76b9f109a..6657f83e9e738 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; @@ -16,6 +17,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -74,6 +76,9 @@ pub(super) struct RenderState { pools: pools::Pools, grow: grow::Grow, retired: KVec, + /// Graphs of released contexts the firmware may still walk (see `detach_context`): kept + /// until the firmware stops, when this state is dropped. + kept_graphs: KVec, next_pm_generation: u64, } @@ -100,6 +105,7 @@ impl RenderState { pools: pools::Pools::new()?, grow: grow::Grow::new()?, retired: KVec::with_capacity(128, GFP_KERNEL)?, + kept_graphs: KVec::new(), next_pm_generation: 1, }) } @@ -298,6 +304,7 @@ impl Registry { if let Some(pair) = entry.pair.as_ref() { self.buffers.unreserve_owner(pair.buffer_id())?; } + self.note_build_room(); } return Ok(Some(UnusedRenderSpare { _entry: entry, @@ -325,6 +332,28 @@ impl Registry { Ok(None) } + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries @@ -333,11 +362,25 @@ impl Registry { .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) } + /// Tell admission whether another render pair can still be built. + pub(super) fn note_build_room(&self) { + self.admission.note_build_room(self.qids.render_pair_fits()); + } + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } - if self.render.entries[usize::from(slot)].is_none() { + if self.render.entries[usize::from(slot)].is_some() { + return Ok(()); + } + let built = self.build_render_ids(slot, owner); + self.note_build_room(); + built + } + + fn build_render_ids(&mut self, slot: u8, owner: u64) -> Result { + { let identity = self.identities.reserve(2)?.start; let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { @@ -381,6 +424,13 @@ impl Registry { /// ensure_render_ids cancels a first ID if the second cannot be reserved; /// validation, identity overflow, and cancellation errors still propagate. fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { + // A full queue table refuses a sibling without scanning it on every preparation. + if optional + && self.render.entries.get(usize::from(slot)).is_some_and(Option::is_none) + && !self.qids.render_pair_fits() + { + return Ok(false); + } match self.ensure_render_ids(slot, owner) { Ok(()) => Ok(true), Err(error) if optional && error == ENOSPC => Ok(false), @@ -507,27 +557,16 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration @@ -558,7 +597,7 @@ impl crate::g17::queue::render::Host for Host<'_> { self.firmware.queues.qids.publish(id) } fn next_publication(&mut self) -> Result { - self.firmware.queues.qids.next_publication() + Ok(self.firmware.port.next_publication()) } fn publish_outer_pair( &mut self, @@ -599,6 +638,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary @@ -618,13 +660,20 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -657,6 +706,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -725,6 +798,19 @@ impl crate::g17::Firmware { { continue; } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; + } self.retire_render_slot(slot, None, deferred)?; self.queues.render.collect_retired(slot)?; self.settle_render_terminal(slot, deferred)?; @@ -783,6 +869,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, @@ -968,10 +1070,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && entry.reservation.ids.into_iter().all(|id| { - let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 - }) + && (crate::g17::queue::render::completion_noted( + entry.reservation.ids.map(kick::Id::qid), + entry.event_masks, + ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) }) } pub(in crate::g17) fn defer_render_timeout( @@ -1056,6 +1158,8 @@ impl Registry { impl RenderState { pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + writeln!(f, "firmware_linked_render_graphs kept={} leaked={}", self.kept_graphs.len(), + teardown::LEAKED_GRAPHS.load(core::sync::atomic::Ordering::Relaxed))?; self.pools.write_capacity(f)?; for (slot, entry) in self.entries.iter().enumerate() { if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index e3cb174721aaa..277534328406e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -14,8 +14,8 @@ use kernel::prelude::*; const RETRY_LIMIT: u16 = 1000; pub(super) struct Grow { - requests: KVec, - active: Option, + pub(super) requests: KVec, + pub(super) active: Option, pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, release_attempts: u16, } @@ -280,7 +280,7 @@ impl crate::g17::Shared { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 4ede20d316c64..57543be54e835 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -1,21 +1,22 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Event-driven render snapshots outside the device mutex. The preparation -//! epoch and exact pair owner survive every read, cancellation and retry. +//! Event-driven render snapshots. A pair's completion memory is read and the +//! pair retired in one device-mutex hold, inside the preparation epoch. use super::super::DeferredBatch; -use crate::g17::queue::render::Pair; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { - fn take_render_observation(&mut self, start: usize) -> Result)>> { + /// The first slot from `start` whose pair has news to observe: published work in flight, the + /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. + fn next_render_observation(&mut self, start: usize) -> Result> { if self.recovery.pending() { return Ok(None); } let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; for slot in start..self.queues.render.entries.len() { - let Some(entry) = self.queues.render.entries[slot].as_mut() else { + let Some(entry) = self.queues.render.entries[slot].as_ref() else { continue; }; if entry.preparing.is_some() || entry.growing { @@ -26,9 +27,8 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || !qids - .into_iter() - .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) + && !pair.stamp_waiting()) { continue; } @@ -37,35 +37,22 @@ impl crate::g17::Firmware { self.init.stamp(qids[1])?.load(Ordering::Relaxed), ]; core::sync::atomic::fence(Ordering::Acquire); - let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { - continue; - }; - entry.borrowed_active = pair.published_in_flight(); - entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.borrowed_switch = pair.switch_in_flight(); - entry.preparing = Some(packet); - return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); + if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { + return Ok(Some(slot)); + } + // The stamps gate this pair out. If its front executed and was consumed anyway, the + // stamps may never move: after the bound, observe it and retire it on its record. + let pair = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { + return Ok(Some(slot)); + } } Ok(None) } - fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { - let returning = pair.as_ref().ok_or(EINVAL)?; - let entry = self.queues.render.entry_mut(slot as u8)?; - if entry.pair.is_some() - || entry.preparing.is_none() - || returning.slot() as usize != slot - || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) - { - return Err(EIO); - } - entry.pair = pair.take(); - entry.borrowed_dependencies = None; - entry.borrowed_switch = None; - entry.preparing = None; - Ok(()) - } - pub(super) fn settle_render_terminal( &mut self, slot: usize, @@ -122,57 +109,36 @@ impl crate::g17::Shared { Err(error) if error == EAGAIN || error == ENODEV => break, result => result?, }; - let loan = { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { - break; - } - firmware.take_render_observation(cursor)? + // Selecting, reading and retiring the front share one hold: a + // separate hold for the read would let publishers and the + // retirement contend twice per observed pair. + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); }; - let Some((slot, pair)) = loan else { + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; + } + let Some(slot) = firmware.next_render_observation(cursor)? else { break; }; if slot != cursor { remaining = crate::hw::t8140::queues::RENDER_DEPTH; retries = 0; } - let observed = pair.observe_front(); - let mut pair = Some(pair); - let mut state = self.state.lock(); - // Return installed ownership before examining read errors or epoch. - let restored = (*state) - .as_deref_mut() - .ok_or(ENODEV) - .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); - if let Err(error) = restored { - // A failed invariant must not drop firmware-owned storage. Join - // the other preparers only after releasing our own epoch, and - // keep this detached owner until both processors have stopped. - drop(state); - drop(epoch); - self.lose_device_quietly(); - self.stop_runtime(); - if let Some(mut pair) = pair { - pair.stopped(&mut |event| { - event.finish(); - Ok(()) - })?; - } - return Err(error); - } - let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); + let observed = firmware.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .ok_or(EIO)? + .observe_front(); + // Publishers waiting for render capacity recheck once this hold ends. + self.note_changed(); if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { - self.queue_events(); - } - if !epoch.is_current() - || firmware.recovery.pending() - || self.crashed.load(Ordering::Acquire) - { - break; + // Observation and the growth check run on every pass. + self.queue_event_records(); } let result = (|| { let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 4713eb5b6b152..1e3354d90dbfb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -186,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -199,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e2034c3176595..62ca7895a86f8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -426,7 +426,7 @@ impl super::super::Backend { } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -548,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -593,6 +593,10 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) @@ -711,7 +715,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 24c724cb32470..71f0bfc052696 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -79,6 +79,20 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if error == ETIMEDOUT { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + pair.expire_stamp_wait(packet, stamps, state)?; + } + } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index b77fc13b7ed2f..970035547dfee 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,22 +8,18 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the /// command instead. Never returned to userspace. -pub(super) const FALLBACK: Error = EINPROGRESS; +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { /// The submitting ioctl holds the logical queue's mutex and never sleeps /// for contention or a closed preparation gate: it returns FALLBACK, and /// the scheduler's worker publishes the command and owns the waits. - pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { if sleep { return self.shared.preparations.enter(false); } @@ -48,21 +44,32 @@ impl Backend { /// Reserve identities before attaching a lease to the packet. An empty /// logical slot must not hide reusable physical channels when the shared /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. fn acquire_render_with_ids( &self, packet: &Arc, mut next: Option, sleep: bool, - ) -> Result { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; let started = super::super::now_ns(); - let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let mut deadline_ns = + started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let recovery_deadline_ns = started.saturating_add(recovery_wait_ns()); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; + let mut capacity = WaitClock::default(); loop { self.admission_open()?; - if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if super::super::now_ns() >= deadline_ns { + if deadline_ns < recovery_deadline_ns && self.waiting_for_recovery()? { + deadline_ns = recovery_deadline_ns; + } else { + return Err(EBUSY); + } + } if packet.cancel_requested() { return Err(ECANCELED); } @@ -97,11 +104,13 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); continue; } + // The deadline check above decides whether a recovery extends the wait. + Err(error) if error == EBUSY => continue, Err(error) => return Err(error), } }; @@ -109,7 +118,10 @@ impl Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => return Ok(lease), + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; if reclaimed.is_some() { @@ -130,8 +142,9 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), true)?; let ready = firmware.queues.has_render_ids(self.owner); drop(state); - if reclaimed.is_none() && !ready { - self.wait_render_capacity(started, sleep)?; + // A full queue table makes admission reuse a free killed pair instead. + if reclaimed.is_none() && !ready && !pool.killed_pair_free() { + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); } @@ -140,31 +153,198 @@ impl Backend { } } - /// A pair is borrowed, still settling, or waiting for the release of the - /// scheduler it last ran for to be consumed and the pair parked. The event - /// worker observes both, so kick it on every poll; time out only after the - /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { - if !sleep { - return Err(FALLBACK); + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire(crate::g17::job::Engine::Render) { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire(crate::g17::job::Engine::Render) { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); } - let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { return Err(ETIMEDOUT); } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A recovery is pending, or this owner's pair is spared and draining after one: an + /// innocent publication waits up to RECOVERY_WAIT_MS for it instead of failing. + fn waiting_for_recovery(&self) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + Ok(firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + || firmware.queues.admission.spared_quarantined(self.owner)) + } + + /// The deadline of the wait `clock` measures (WaitClock::deadline). + fn wait_deadline(&self, clock: &mut WaitClock, bound_ns: u64) -> Result { + let recovering = self.waiting_for_recovery()?; + Ok(clock.deadline(super::super::now_ns(), recovering, bound_ns)) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + clock: &mut WaitClock, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { + if !sleep { + return Err(FALLBACK); + } + let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; self.shared.queue_events(); - fsleep(Delta::from_micros(200)); - Ok(()) + self.wait_domain(&self.shared.renders, sample, deadline, packet) } - /// A closing owner may still hold its pair until exact retirement. Wait - /// outside both the device mutex and preparation gate so the event worker - /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity( + &self, + observed: &mut Option, + clock: &mut WaitClock, + sleep: bool, + ) -> Result { if !sleep { return Err(FALLBACK); } - self.wait_capacity(started) + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = self + .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } } fn render_with_lease( @@ -185,14 +365,45 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early, sleep)?; + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() + } + lease => lease, + }; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; - let mut busy_since = None; - let mut recovery_polls = 0; - let capacity_started = super::super::now_ns(); + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } + let mut busy = WaitClock::default(); + let mut capacity = WaitClock::default(); + let mut state_clock = WaitClock::default(); + let mut recovery_since = None; + let mut seen = None; + let mut states = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -210,7 +421,7 @@ impl Backend { } else { self.enter_preparation(false)? }; - let result = packet.completion.work_state().and_then(|state| state.acquire()); + let result = packet.completion.work_state().and_then(|state| state.acquire(crate::g17::job::Engine::Render)); if preparation.is_current() { result } else { @@ -223,54 +434,72 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if state_polls == 1000 { - return Err(ETIMEDOUT); + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + // Work states free as earlier, possibly spared, work retires. + Some(sample) => { + let deadline = self + .wait_deadline(&mut state_clock, super::super::WORK_RING_WAIT_NS)?; + self.wait_domain(work.returns(), sample, deadline, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - match self.ensure_render(packet, slot, sleep) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { Err(error) if error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { if !sleep { return Err(FALLBACK); } - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. + let since = *recovery_since.get_or_insert(super::super::now_ns()); self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared + .preparations + .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } @@ -284,19 +513,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -448,15 +667,28 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; // An owner change taken from a live owner which closed meanwhile: its // first command waits until that scheduler's release has been consumed. @@ -505,3 +737,57 @@ impl crate::g17::Firmware { result } } + +fn recovery_wait_ns() -> u64 { + u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 +} + +/// One bounded wait (Backend::wait_deadline). +#[derive(Default)] +struct WaitClock { + first: Option, + calm: u64, +} + +impl WaitClock { + /// `bound_ns` after the later of the first call and the last call that saw a recovery + /// pending or this owner's spared pair draining (so the usual bound applies afresh once a + /// recovery ends), and at most RECOVERY_WAIT_MS after the first call. + fn deadline(&mut self, now: u64, recovering: bool, bound_ns: u64) -> u64 { + let first = *self.first.get_or_insert(now); + self.calm = if recovering { now } else { self.calm.max(first) }; + first + .saturating_add(recovery_wait_ns()) + .min(self.calm.saturating_add(bound_ns)) + } +} + +#[kunit_tests(asahi_g17_wait_clock)] +mod tests { + use super::*; + + const BOUND: u64 = 2_000_000_000; + + #[test] + fn a_calm_wait_keeps_its_first_deadline() { + let mut clock = WaitClock::default(); + assert_eq!(clock.deadline(100, false, BOUND), 100 + BOUND); + assert_eq!(clock.deadline(500, false, BOUND), 100 + BOUND); + } + + #[test] + fn the_bound_starts_afresh_once_a_recovery_ends() { + let mut clock = WaitClock::default(); + clock.deadline(100, false, BOUND); + assert_eq!(clock.deadline(1_000, true, BOUND), 1_000 + BOUND); + assert_eq!(clock.deadline(2_000, false, BOUND), 1_000 + BOUND); + } + + #[test] + fn a_recovery_cannot_extend_past_the_recovery_wait() { + let mut clock = WaitClock::default(); + clock.deadline(100, true, BOUND); + let limit = 100 + recovery_wait_ns(); + assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index eb04f80361955..fffa6d28dd289 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -38,7 +38,7 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? { + if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { return Ok(false); } } @@ -46,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self @@ -115,6 +123,7 @@ impl RenderState { if !out.has_room(count, 1) { return Err(EBUSY); } + let linked = context.firmware_may_link(); for entry in self.entries.iter_mut().flatten() { if !entry .context @@ -132,7 +141,14 @@ impl RenderState { let (mut parked, retired) = KBox::into_inner(pair).park(); // This park follows the consumed release of `context`. parked.release_consumed(context.scheduler_va()); - *target = Some(retired); + if linked { + // Guard (kill before free): the firmware may still walk this context's work + // list into the graph, so it is not freed or recycled before the firmware stops. + kernel::pr_warn_once!("G17: keeping render graphs of an unkilled context that still links work\n"); + keep_graph(&mut self.kept_graphs, retired); + } else { + *target = Some(retired); + } entry.parked = Some(parked); // Detached backing retains these references until the worker runs. entry.context = None; @@ -147,8 +163,12 @@ impl RenderState { if let Some(id) = self.retired[index].buffer_id() { buffers.unreserve_owner(id)?; } - let target = out.binding_slot()?; - *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + let retired = self.retired.remove(index).map_err(|_| EIO)?; + if linked { + keep_graph(&mut self.kept_graphs, retired); + } else { + *out.binding_slot()? = Some(retired); + } } Ok(()) } @@ -173,3 +193,23 @@ impl Registry { Ok(()) } } + +/// Graphs that could not be recorded in `kept_graphs` and were leaked instead, for debugfs. +pub(super) static LEAKED_GRAPHS: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + +/// Keeps `graph` until the firmware stops; if even that cannot be recorded, it is leaked +/// rather than freed. +fn keep_graph( + kept: &mut KVec, + graph: crate::g17::queue::render::RetiredBinding, +) { + if kept.reserve(1, GFP_KERNEL).is_ok() { + if let Err(error) = kept.push_within_capacity(graph) { + LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + core::mem::forget(error.0); + } + } else { + LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + core::mem::forget(graph); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f8020e8962f84..0222ae4c478c8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -100,6 +100,8 @@ impl ReclaimBatch { struct Service<'a> { render: &'a mut super::render::RenderState, + admission: &'a crate::g17::queue::admission::Pool, + objects: &'a Arc, compute: &'a [Option], buffers: &'a mut BufferIds, init: &'a InitData, @@ -115,7 +117,7 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) + .filter_map(|entry| entry.queue.as_deref()) .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { @@ -137,8 +139,15 @@ impl Host for Service<'_> { self.primary.notify(MSG_CONTROL_NOTIFY) } fn detach(&mut self, context: &Arc) -> Result { + let linked = context.retain_linked_work(self.objects); + let (_, _, slots) = self.render.exit_scope(context); + crate::g17::trace::teardown(5, u64::from(context.id()), slots, u64::from(linked)); self.render .detach_context(context, self.buffers, self.reclaim)?; + // The firmware consumed this context's scheduler release after its kill (if any) + // was acknowledged and its pairs went idle: it no longer names these pairs, so + // their kill hold ends here rather than at the next processor stop. + self.admission.release_killed_slots(slots); // detach_context preflights this slot together with every graph slot. // Retain it before Pending removes its reference under the device lock. self.reclaim.context(context) @@ -173,6 +182,7 @@ impl Firmware { self.queues.render.observe_teardown_releases(&self.init)?; let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; let Registry { + admission, render, compute, buffers, @@ -181,6 +191,8 @@ impl Firmware { } = &mut self.queues; let mut service = Service { render, + admission, + objects: &self.pool, compute, buffers, init: &self.init, @@ -190,10 +202,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 53518a76e91a4..fa2a72a77f060 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -24,6 +24,45 @@ struct Entry { phase: Phase, armed: u64, announce_pending: bool, + render_kill: Option, + /// Since when, and whether reported, a render-only kill has waited for other contexts' + /// compute to pause. + kill_wait: Option<(u64, bool)>, + /// A render-only kill of this context was refused (counted once per context). + kill_refused: bool, +} + +/// The kill of a closed context whose compute side is clean, published so that its render +/// pairs' executed but unstamped commands can be released (see `Pair::firmware_drained`). +/// The scheduler page's killed state, not this record, proves the kill to the pairs. +#[derive(Clone, Copy)] +struct RenderKill { + cookie: u64, + published: u64, + announced: bool, + acknowledged: bool, +} + +impl RenderKill { + fn new(cookie: u64, now: u64) -> Self { + Self { + cookie, + published: now, + announced: false, + acknowledged: false, + } + } + /// Only this kill's exact cookie acknowledges it, once. + fn acknowledge(&mut self, cookie: u64) -> bool { + if cookie == 0 || cookie != self.cookie || self.acknowledged { + return false; + } + self.acknowledged = true; + true + } + fn awaiting(&self, now: u64, window: u64) -> bool { + !self.acknowledged && now.saturating_sub(self.published) < window + } } /// Publication and announcement errors retain the entry; no timeout frees @@ -104,6 +143,9 @@ impl Pending { phase, armed: now, announce_pending: false, + render_kill: None, + kill_wait: None, + kill_refused: false, }; self.entries.push(entry, GFP_KERNEL)?; Ok(()) @@ -233,6 +275,94 @@ impl Pending { Ok(None) } + /// A render-only kill of the closed context at `index` was published. + pub(crate) fn render_kill_published(&self, index: usize) -> bool { + self.entries.get(index).is_some_and(|entry| entry.render_kill.is_some()) + } + + /// Record a published render-only kill. Publication is irrevocable: the record stays + /// with the entry until the context's scheduler release is consumed. + pub(crate) fn record_render_kill(&mut self, index: usize, cookie: u64, now: u64) -> Result { + let entry = self.entries.get_mut(index).ok_or(EINVAL)?; + if entry.render_kill.is_some() { + return Err(EINVAL); + } + entry.render_kill = Some(RenderKill::new(cookie, now)); + Ok(()) + } + + /// Ring the doorbell for published render-only kills not yet announced. A failed + /// doorbell is retried on a later pass and never duplicates the record. + pub(crate) fn announce_render_kills(&mut self, mut notify: impl FnMut() -> Result) { + for entry in self.entries.iter_mut() { + if let Some(kill) = entry.render_kill.as_mut() { + if !kill.announced && !kill.acknowledged && notify().is_ok() { + kill.announced = true; + } + } + } + } + + /// Only the exact cookie of an unacknowledged render-only kill is accepted. + pub(crate) fn acknowledge_render_kill(&mut self, cookie: u64) -> bool { + self.entries + .iter_mut() + .filter_map(|entry| entry.render_kill.as_mut()) + .any(|kill| kill.acknowledge(cookie)) + } + + /// A render-only kill awaits its acknowledgement within the polling window. + pub(crate) fn render_kill_polling(&self, now: u64, window: u64) -> bool { + self.entries + .iter() + .any(|entry| entry.render_kill.is_some_and(|kill| kill.awaiting(now, window))) + } + + /// The render-only kill of the entry at `index` waits; how long, and whether this is the + /// first time it crossed `slow`. + pub(crate) fn note_kill_wait(&mut self, index: usize, now: u64, slow: u64) -> (u64, bool) { + let Some(entry) = self.entries.get_mut(index) else { + return (0, false); + }; + let (since, warned) = entry.kill_wait.get_or_insert((now, false)); + let waited = now.saturating_sub(*since); + let crossed = waited >= slow && !*warned; + *warned |= crossed; + (waited, crossed) + } + + /// The render-only kill was published: how long it waited, if it did. + pub(crate) fn take_kill_wait(&mut self, index: usize, now: u64) -> u64 { + self.entries + .get_mut(index) + .and_then(|entry| entry.kill_wait.take()) + .map_or(0, |(since, _)| now.saturating_sub(since)) + } + + /// A render-only kill of the context at `index` was refused: it no longer waits, and this + /// returns whether it is the first refusal for this context. + pub(crate) fn refuse_render_kill(&mut self, index: usize) -> bool { + let Some(entry) = self.entries.get_mut(index) else { + return false; + }; + entry.kill_wait = None; + !core::mem::replace(&mut entry.kill_refused, true) + } + + /// The context at `index` is not waiting for a render-only kill (any more). + pub(crate) fn clear_kill_wait(&mut self, index: usize) { + if let Some(entry) = self.entries.get_mut(index) { + entry.kill_wait = None; + } + } + + /// How long each waiting render-only kill has waited. + pub(crate) fn kill_waits(&self, now: u64) -> impl Iterator + '_ { + self.entries + .iter() + .filter_map(move |entry| entry.kill_wait.map(|(since, _)| now.saturating_sub(since))) + } + /// Poll for a bounded window at close and again at actual release publication. /// Failed announcements remain retryable on later service even after polling /// expires. Time passage never substitutes for a firmware-consumption witness. @@ -242,3 +372,21 @@ impl Pending { }) } } + +#[kernel::macros::kunit_tests(asahi_g17_render_kill)] +mod render_kill_tests { + use super::RenderKill; + + #[test] + fn only_the_exact_cookie_acknowledges_a_render_kill_once() { + let mut kill = RenderKill::new(7, 1_000); + assert!(kill.awaiting(1_000, 10)); + assert!(!kill.acknowledge(0)); + assert!(!kill.acknowledge(8)); + assert!(kill.awaiting(1_009, 10)); + assert!(!kill.awaiting(1_010, 10)); + assert!(kill.acknowledge(7)); + assert!(!kill.acknowledge(7)); + assert!(!kill.awaiting(1_001, 10)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index d951bd164d6e9..ce003e6b18196 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -18,6 +18,13 @@ pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS pub(crate) const POLL_MS: u32 = 2000; const EXECUTION_RENEWALS: u8 = 4; +/// How long an innocent client's publication waits for a recovery to settle and its +/// spared work to drain, instead of failing with EBUSY: the watchdog defers work behind +/// another VM at most this long before it settles it (`classify`; ray work gets the +/// execution renewals on top). +pub(crate) const RECOVERY_WAIT_MS: u32 = + POLL_MS * (1 + EXECUTION_RENEWALS as u32 + MAX_RENEWALS as u32); + /// A background control/replay poll has no scheduler fence to expire it. /// Its timestamp belongs to the exact transaction and is never renewed by work. pub(crate) fn background_stalled(since: Option, now: u64) -> bool { @@ -53,6 +60,37 @@ pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) true } +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Whether some command retired within [`LIVENESS_NS`] (`since_retired`). +pub(crate) fn retiring(since_retired: Option) -> bool { + since_retired.is_some_and(|since| since < LIVENESS_NS) +} + +/// Renews a published job's deadline while the GPU is `shared`, at most +/// [`SHARED_RENEWALS`] times. It is shared while some command retires +/// ([`retiring`]), or while a long job of another queue runs beside a long job: +/// two ray traversal dispatches of about 6 s each took 10 to 17 s side by side, and +/// neither retires anything until it ends. +pub(crate) fn renew_shared(published: bool, shared: bool, renewals: &mut u8) -> bool { + if !published || !shared || *renewals >= SHARED_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. @@ -215,6 +253,37 @@ mod tests { assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); } #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, retiring(Some(4_900_000_000)), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, true, &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, retiring(Some(LIVENESS_NS)), &mut renewals)); + assert!(!renew_shared(true, retiring(None), &mut renewals)); + assert!(!renew_shared(false, true, &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] + fn a_long_job_beside_another_shares_the_gpu_without_retirements() { + // Two traversal dispatches side by side: nothing retires, yet both progress. + let mut renewals = 0; + let long_job_beside = true; + let shared = retiring(Some(LIVENESS_NS + 1)) || long_job_beside; + for poll in 1..=14u8 { + assert!(renew_shared(true, shared, &mut renewals)); + assert_eq!(renewals, poll); + } + // The thirty-second bound still ends a pair that never finishes. + assert!(!renew_shared(true, true, &mut renewals)); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b2a6cf6e8bfe3..b0f2861382ad7 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -144,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 72238159d0475..f100820a0c75e 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,9 +41,7 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, - memacct, - mmu, + file, memacct, mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -133,9 +131,6 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -410,6 +405,23 @@ impl AsahiObject { } } +/// Populates and pins a new user object through its DMA mapping, before any CPU view exists. +/// +/// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the +/// freshly zeroed pages to the point of coherency, so that no dirty line can later overwrite data +/// written through a write-combined view. The pin keeps reclaim from creating a new cacheable +/// alias. Only used when the GPU asks for it ([`crate::gpu::Gpu::prepare_new_objects`]). +/// +pub(crate) fn prepare_new_object( + gem: &Object, + dev: &kernel::device::Device, +) -> Result { + // The native backend requires a non-coherent DMA device. Use its DMA mapping + // rather than interpreting DMA addresses as CPU physical addresses. + let _sgt = held_sg_table(gem, dev)?; + Ok(()) +} + /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { @@ -419,6 +431,26 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { Ok(u32::from(gem.madvise(i32::from(dontneed)))) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} + /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the @@ -429,7 +461,7 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { /// The DMA mapping only cleans if the device is not marked DMA-coherent (G15's synthesized node /// has no `dma-coherent`). Should it ever be marked so, the pages are cleaned here instead. #[cfg(CONFIG_DRM_ASAHI_G15)] -pub(crate) fn prepare_new_object(gem: &Object) -> Result { +pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { // SAFETY: G15 objects belong to the retained bound platform device; user object // creation occurs only after registration and before a file can outlive its device. let dev: &kernel::device::Device = gem.dev().as_ref().as_ref(); @@ -466,28 +498,7 @@ pub(crate) fn prepare_new_object(gem: &Object) -> Result { Ok(()) } - #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } -/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. -/// Keep ordinary resource allocations and kernel-owned backing on their existing -/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. -pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { - g17 && size >= 256 * 1024 * 1024 -} - -#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] -mod allocation_policy_tests { - use super::*; - - #[test] - fn large_user_backing() { - assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); - assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); - assert!(user_pages_no_reclaim(3_257_113_344, true)); - assert!(!user_pages_no_reclaim(3_257_113_344, false)); - } +extern "C" { + fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index be65af433a633..3da16c10696e3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -35,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -44,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -84,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -258,11 +261,13 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, - #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>>>, + garbage_contexts: Mutex>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. @@ -275,6 +280,11 @@ pub(crate) trait Gpu: Send + Sync { /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -291,7 +301,7 @@ pub(crate) trait Gpu: Send + Sync { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). fn supports_soft_faults(&self) -> bool { - *crate::module_parameters::fault_control.value() == 0xb + crate::module_parameters::fault_control.value() == 0xb } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { @@ -305,14 +315,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); } if self.queue_limits().is_some() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } if self.soft_faults() && self.supports_soft_faults() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -351,11 +361,7 @@ pub(crate) trait Gpu: Send + Sync { fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. fn uat_geometry(&self) -> mmu::UatGeometry; - /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. - fn vm_destroyed(&self) {} - /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS - /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 - /// T8122 runtime admits any client. + /// Admit a client's parameter ABI before creating its VM. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -380,6 +386,7 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( @@ -428,8 +435,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); + /// Register an unused context, with the objects it retains, for invalidation. + fn free_context(&self, ctx: workqueue::RetiredContext); } /// Private generic trait for functions that don't need to escape this module. @@ -839,6 +846,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -854,6 +867,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -881,6 +895,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1112,7 +1127,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1162,20 +1177,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) @@ -1342,6 +1351,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } @@ -1424,6 +1437,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1441,6 +1455,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) @@ -1483,7 +1498,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { + if self.invalidate_context(&ctx.data).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1492,6 +1507,9 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } + // The firmware no longer knows the context's queues, so their structures may go now, + // ahead of the context data they point to. + core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1760,7 +1778,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: KBox>) { + fn free_context(&self, ctx: workqueue::RetiredContext) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 7a4d6f288dc3d..c2e7c06f67b7f 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -7,7 +7,7 @@ use crate::g17::fw::{ self, - initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, + initdata::{PerfConfig, RegionCConfig, RegisterWindow}, }; use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; @@ -62,11 +62,49 @@ pub(crate) const CONFIG: Config = Config { firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { os_version: [26, 6, 2], name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + images: &[ + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, + 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, + 0x10, 0xe0, 0x63, 0x87, 0x6d, 0xcd, + ], + }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, + 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, + 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, + ], + }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, + 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, + 0x37, 0xee, 0x5d, 0x6e, 0x5e, 0x98, + ], + }, + ], variant: &fw::variant::V25G83, }]), - // Measured on the 20-core M4 Pro; other core counts are refused at probe. + // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { cores: Some((2, 20)), + // Admit two clusters of ten slots with eight enabled cores in each cluster. + // Reusing the measured 20-core buffers on it remains experimental. + alternate_core_masks: Some((10, 8)), operand_state: fw::compute::OperandState::TWO_BY_TEN, deflake: fw::render::Deflake::TWO_CLUSTERS, }, @@ -111,7 +149,11 @@ const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> /// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), - instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + instances( + window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), + 4, + 0x200_0000, + ), window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), @@ -120,7 +162,11 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), - instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + instances( + window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), @@ -137,33 +183,23 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), ]; -/// Performance-table configuration. The device tree lists the sixteen operating points in -/// performance-state order, with one voltage per cluster. +/// Inert defaults. The 25G83 builder requires the loader's per-chip performance maps. pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], - states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], - relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], - relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + states_a: [0; 11], + states_b: [0; 11], + relative_a: [0; 11], + relative_b: [0; 11], sram_min_mv: 810, }; /// Operating points the performance tables reference. pub(crate) const OPPS: usize = 16; -const _: () = { - let mut index = 0; - while index < PERF_STATES { - assert!((PERF.states_a[index] as usize) < OPPS); - assert!((PERF.states_b[index] as usize) < OPPS); - index += 1; - } -}; - /// Region C. const REGION_C: RegionCConfig = RegionCConfig { unk_54: 0x0fa0, unk_78: [1, 0, 0], - max_power_mw: v25g83::MAX_POWER_MW, + max_power_mw: 0, unk_9c: 0, unk_a0: 0, unk_7c4: 1500, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 75925713bf10b..ea0f9cd782e81 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -57,6 +57,7 @@ pub(crate) const CONFIG: Config = Config { // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { cores: None, + alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, }, @@ -417,9 +418,10 @@ pub(crate) mod scheduling { pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, + // as many as a pair has command records (the render memory's RECORDS). pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; - pub(crate) const RENDER_DEPTH: u8 = 16; + pub(crate) const RENDER_DEPTH: u8 = 32; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index bcb9fd32c7d25..3b13ca94462f5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -250,7 +250,7 @@ impl gpu::Gpu for Backend { } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, - usc_exec_base: u64) -> Result> { + usc_exec_base: u64, _usage: Arc) -> Result> { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index a60e38bf9addd..1212113af1618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -44,6 +44,9 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, + /// Set before a batch with any of this packet's commands can change firmware-visible state, + /// and never cleared. + published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -79,6 +82,10 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + /// Called before the first firmware-visible change of a batch that includes this packet. + pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Whether the firmware may have seen any of this packet's commands. + pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -233,7 +240,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 5f0ab2b4aea4f..79071cbc434a0 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -71,7 +71,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) + x = in(reg) (u64::from(asid) << 48) ); } } @@ -85,7 +85,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -147,7 +147,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= (exp as u64) << 44; + val |= u64::from(exp) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index 9da09968171e4..12fc90b23d8eb 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -131,7 +131,7 @@ fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bo fn sub(counter: &AtomicU64, bytes: u64) { let prev = counter.fetch_sub(bytes, Ordering::AcqRel); if prev < bytes { - pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); counter.store(0, Ordering::Relaxed); } } @@ -157,12 +157,12 @@ impl Ledger { let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); let limits = limits_for( total, - *module_parameters::mem_limit_pct.value(), - *module_parameters::client_mem_limit_pct.value(), + module_parameters::mem_limit_pct.value(), + module_parameters::client_mem_limit_pct.value(), ); self.set_limits(limits); pr_info!( - "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", limits.device >> 20, limits.client >> 20 ); @@ -300,7 +300,7 @@ impl Charge { /// A user object admitted by `commit`, attributed to `mm` if there is one. pub(crate) fn user(commit: Commit, mm: Option>) -> Self { let bytes = commit.bytes; - let mm = if *module_parameters::oom_attribution.value() != 0 { + let mm = if module_parameters::oom_attribution.value() != 0 { mm } else { None diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index cda3ac8d206bd..437685bfad1de 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -108,8 +108,6 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; @@ -416,6 +414,21 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -1019,6 +1032,7 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1229,56 +1243,88 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, - offset: usize, -} +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); -impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: gem.owned_vmap::<0>()?, - offset, - }) + self.0.owner().size() } - pub(crate) fn write(&self, value: u64) { + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { + return; + } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.mapping.owner().timestamp_lock.lock(); - // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. - let base = self.mapping.addr.cast::(); - let pointer = unsafe { base.add(self.offset) }; + let _guard = self.0.owner().timestamp_lock.lock(); + let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // SAFETY: All eight bytes were checked above; the vmap pins the backing. // A naturally aligned query result becomes available in one complete store. let destination = unsafe { &*pointer.cast::() }; destination.store(value.to_le(), Ordering::Relaxed); } else { for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(self.offset + index).cast::() }; + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; destination.store(byte, Ordering::Relaxed); } } // A CPU-retired query can be consumed by a later GPU command. WRITEBACK // objects use a cacheable vmap, while GPU mappings need not snoop that cache. // Publish the bytes before signalling the completion fence or queue frontier. - if self.mapping.owner().is_writeback() { + if self.0.owner().is_writeback() { crate::pgtable::clean_poc(pointer, size_of::()); } crate::pgtable::poc_barrier(); } } +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct TimestampMapping { + mapping: TimestampVmap, + offset: usize, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1304,16 +1350,22 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)? - .page_table - .translate_iova(address) + self.0.owner.lock_inner().page_table.translate_iova(address) } /// Writes a userspace timestamp through the retained GEM mapping. Aligned results /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1324,7 +1376,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - TimestampMapping::new(gem, start) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1436,7 +1493,7 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); uat_inner.handoff_lock(); @@ -1679,7 +1736,7 @@ impl UatShared { /// Writes back context-table slot `idx` after it changed, where this SoC needs it. fn publish_slot(&self, idx: usize) { if self.quirks.clean_tables { - let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); pgtable::clean_poc(slot, size_of::()); pgtable::poc_barrier(); } @@ -1688,13 +1745,20 @@ impl UatShared { /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. @@ -1882,7 +1946,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1989,10 +2055,7 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -2000,7 +2063,10 @@ fn prefetch_sgt_segments( segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { segments, _lease: sgt.into_lease() }) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) } /// Reuse cached segments without allocating another copy for each bind. The bind lease was @@ -2112,7 +2178,9 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2144,7 +2212,7 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn with_tracked_status(mut self) -> Result { self.status = Some(Arc::pin_init( crate::g17::status::VmStatus::new(), @@ -2167,9 +2235,18 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2239,9 +2316,7 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2256,9 +2331,7 @@ impl Vm { if stride <= block_size { return false; } - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; @@ -2268,12 +2341,12 @@ impl Vm { }) } - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return (false, None); - }; + // Every mapping change holds the VM reservation, so readers need only that one. The full + // exec lock would also lock every external BO, making each submit O(external BOs). + let mut inner = self.gpuvm.lock_inner(); let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2327,7 +2400,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, ) } @@ -2342,7 +2421,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, ) } @@ -2370,20 +2455,27 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2452,21 +2544,26 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock(/*Some(&gem)*/ None, false) + .exec_lock_vm(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? @@ -2535,7 +2632,7 @@ impl Vm { gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; + let dev: &device::Device = device.as_ref().as_ref(); let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm @@ -2556,8 +2653,10 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2628,7 +2727,9 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2732,7 +2833,9 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2763,7 +2866,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2802,7 +2905,8 @@ impl Vm { // The BO's newest sequence is checked under the same lock as binds. If // it is not newer than the close, every mapping belongs to that close // and the per-object list is safe and substantially cheaper to walk. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => { return self.drop_mappings_filtered(gem, sequence); } @@ -2942,7 +3046,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -2965,34 +3069,44 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // A carveout outside System RAM is mapped write-combined; a reserved-memory region - // inside System RAM (mapped by the kernel) through the kernel's own mapping, as - // `map_region` does for the other generations. - let mode = crate::g15::memory_mode(range.start, size as u64)?; - let ram = mode == kernel::bindings::MEMREMAP_WB as u32; - let flags = if ram { - io::mem::MemFlag::WB + let flags = io::mem::MemFlag::WC; + #[cfg(CONFIG_DRM_ASAHI_G15)] + let flags = if reserved_ram { + // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. + if crate::g15::memory_mode(range.start, size as u64)? + == kernel::bindings::MEMREMAP_WB as u32 + { + io::mem::MemFlag::WB + } else { + flags + } } else { - io::mem::MemFlag::WC + flags }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3000,7 +3114,7 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3020,25 +3134,36 @@ impl Uat { geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_with_user_upper(dev, geometry, regions, true) + Self::new_adopted_inner(dev, geometry, regions, true, true) } /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn new_adopted_with_user_upper( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, map_kernel_to_user: bool, ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; + Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn new_adopted_inner( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + reserved_ram: bool, + ) -> Result { + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { @@ -3110,13 +3235,13 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3180,7 +3305,7 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } @@ -3218,6 +3343,16 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { self.inner.check_m4()?; let mut binding = vm.binding.lock(); self.inner.check_m4()?; @@ -3225,7 +3360,7 @@ impl Uat { if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = *module_parameters::robust_isolation.value() != 0; + let isolation = module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3233,7 +3368,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; @@ -3308,9 +3447,9 @@ impl Uat { { return Err(EIO); } - for index in UAT_USER_CTX_START..UAT_NUM_CTX { - if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 { return Err(EIO); } @@ -3325,7 +3464,7 @@ impl Uat { return Err(EINVAL); } { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3411,14 +3550,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -3431,11 +3575,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -3477,7 +3626,7 @@ impl Uat { } /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). pub(crate) fn invalidate_reserved_tables(&self) { if let Some(reserved) = self.inner.reserved.as_ref() { reserved.invalidate(); @@ -3523,7 +3672,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); let shared_quirks = quirks; @@ -3652,7 +3801,10 @@ impl Uat { inner.handoff().init(firmware, quirks.handoff_slot)?; if quirks.clean_tables { - pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); pgtable::poc_barrier(); } @@ -3680,7 +3832,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); pgtable::poc_barrier(); } @@ -3707,7 +3859,7 @@ impl Vm { /// Diagnostic reservation occupancy (including guards), not a placement promise. /// Removal can progress between queries; allocations are excluded by the VM lock. pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = lock_vm(&self.gpuvm, None, false)?; + let inner = self.gpuvm.lock_inner(); let mut next = range.start; let mut used = 0; let mut largest = 0; diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index e222c7cd8e6a9..acce5677b9f92 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -135,7 +135,7 @@ impl Vm { PreparedUserBindOp::Map(map) => { // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is // not locked: the map step may still populate its pages. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let (flags, repeat) = if map.single_page { @@ -185,7 +185,7 @@ impl Vm { } PreparedUserBindOp::Unmap(unmap) => { // SAFETY: See `lock_vm()`. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let mut ctx = diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index 8611221af492a..a63101a813426 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 49c2241bc399f..1df852184b31d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -266,7 +266,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index a66a82d92a583..0c9e6f62b5bbe 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,11 +298,15 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>(); let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -337,10 +341,14 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>>(); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { + if !core::ptr::eq(p.cast::>(), raw) { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -349,7 +357,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner, @@ -400,16 +408,16 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = &*ret.inner as *const T; + let q = core::ptr::from_ref::(&*ret.inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; + unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } @@ -437,7 +445,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -449,7 +457,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } } } @@ -475,8 +483,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) + .field("inner", &format_args!("{:#X?}", self.inner)) + .field("alloc", &format_args!("{:?}", self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index c5a47c24ec928..0f915607cb1f4 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -153,9 +153,8 @@ impl ReservedTables { pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { let start = res.start(); // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = unsafe { - kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? - }; + let map = + unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; self.starts.push(start, GFP_KERNEL)?; self.regions.push(map, GFP_KERNEL)?; Ok(()) @@ -183,6 +182,15 @@ impl ReservedTables { /// Drops the CPU's cached lines of every region, so that later reads see what the firmware /// wrote while booting. Lines the host dirtied are written back first, never discarded. + /// + /// This runs once, after the firmware booted. The host has written table entries by then + /// (fixed kernel objects are mapped before the boot), but the T6040, the only SoC with + /// reserved tables, also sets `UatQuirks::clean_tables`: every host table store is cleaned + /// to the PoC as it is made, so this drops only clean lines and loses no host write. + /// + /// The regions are not invalidated again, which assumes the firmware no longer writes the + /// tables the host walks once it has booted. That is not proven by a trace; the J616s has + /// run this way since its GPU came up without a fault traced to a stale table. pub(crate) fn invalidate(&self) { for map in self.regions.iter() { clean_invalidate_poc(map.ptr(), map.size()); @@ -192,7 +200,9 @@ impl ReservedTables { } /// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the -/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// reserved mapping when the page lies in a reserved region, else through `struct page`. With +/// reserved regions, a page outside them that is not System RAM with a `struct page` (other +/// no-map memory) is refused with `EIO`. /// /// # Safety /// @@ -211,8 +221,14 @@ unsafe fn with_table( // SAFETY: lookup() checked that the whole page is mapped. return f(unsafe { base.add(offset) }); } - // SAFETY: The caller guarantees an allocated page-table page in RAM. - let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + let page = if reserved.is_some() { + // SAFETY: The caller guarantees the page stays allocated; borrow_phys() checks that it + // is System RAM with a struct page. + unsafe { Page::borrow_phys(&phys) }.ok_or(EIO)? + } else { + // SAFETY: The caller guarantees an allocated page-table page in RAM. + unsafe { Page::borrow_phys_unchecked(&phys) } + }; page.with_pointer_into_page(offset, len, f) } @@ -353,7 +369,7 @@ impl Prot { /// The protection bits of a leaf with this protection (no address, table type or non-global /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() } @@ -654,7 +670,7 @@ impl UatPageTable { let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; let entry = |p: *mut u8| -> Result> { - let uptep = p as *const _ as *const Pte; + let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is // aligned so it is safe to deref as an AtomicU64. let upte = unsafe { &*uptep }; @@ -662,7 +678,10 @@ impl UatPageTable { // Allocate if requested if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + mod_pr_debug!( + "UATPageTable::with_pages: alloc PT at {:#x}\n", + pt_page.phys() + ); if clean { // A non-snooping walker must never follow a parent to a // table whose zeroes are still in the CPU caches. @@ -738,7 +757,7 @@ impl UatPageTable { iova ); let leaves = |p: *mut u8| -> Result { - let ptep = p as *const _ as *const Pte; + let ptep = p.cast_const().cast::(); // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; @@ -752,7 +771,13 @@ impl UatPageTable { // SAFETY: Page table addresses are either allocated by us, or // firmware-managed tables in RAM or in a reserved region. unsafe { - with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + with_table( + reserved.as_deref(), + phys, + idx * PTE_SIZE, + count * PTE_SIZE, + leaves, + ) }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -789,7 +814,10 @@ impl UatPageTable { } if let Some((parent_phys, parent_index)) = parent { let oas_mask = self.oas_mask; - let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + let in_reserved = self + .reserved + .as_ref() + .is_some_and(|r| r.contains(parent_phys)); // SAFETY: Only looks the page up; nothing is accessed through it. if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { return Err(EIO); @@ -815,7 +843,15 @@ impl UatPageTable { // its child descriptor is detached. A changed descriptor indicates // an ownership violation; fail closed instead of freeing a table // that may still be reachable. - unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; + unsafe { + with_table( + reserved, + parent_phys, + parent_index * PTE_SIZE, + PTE_SIZE, + detach, + ) + }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 4abdd3e485f72..aa5dc1fba4723 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,7 +67,9 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { return Err(EINVAL); } + if layouts != [0; 2] { + return Err(EINVAL); + } Ok(()) } @@ -151,6 +153,7 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, + usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -191,7 +194,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -263,10 +266,13 @@ impl QueueJob::ver { impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; const MODULE: Option<&'static kernel::ThisModule> = - Some(&crate::THIS_MODULE); + Some(kernel::module::this_module::()); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } + QueueTimeout { + dev: self.dev.clone(), + id: self.id, + } } fn prepare(job: &mut Self) -> Option { @@ -358,7 +364,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - return guard.registration_data_with(|reg_data| { + guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -433,7 +439,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }); + }) } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -462,7 +468,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_NAME: &CStr = c_str!("aurora_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -482,6 +488,7 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -519,12 +526,21 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("asahi_sched"), + c_str!("aurora_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + let gpu_context = Arc::new( + workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, + GFP_KERNEL, + )?; + // Every command of this queue points the firmware at the notifier, and the notifier links + // into the notifier list of the work queues. Keep it as long as they are kept, until the + // context is invalidated, instead of freeing it with the last command. + gpu_context.retain(KBox::new(notifier.clone(), GFP_KERNEL)?)?; + let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -538,15 +554,12 @@ impl Queue::ver { inner: QueueInner::ver { dev: dev.clone(), ualloc, - gpu_context: Arc::new( - workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, - GFP_KERNEL, - )?, - + gpu_context, buffer, notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, + usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), @@ -554,7 +567,7 @@ impl Queue::ver { }; // Rendering structures - let tvb_blocks = *module_parameters::initial_tvb_size.value(); + let tvb_blocks = module_parameters::initial_tvb_size.value(); ret.inner.buffer.ensure_blocks(tvb_blocks)?; @@ -624,8 +637,11 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -636,8 +652,10 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i as usize] = microseq::Attachment { + attachments.list[i] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -756,16 +774,24 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; let payload_size = header.size as usize; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if render + .ts_frag_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if compute + .ts_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } @@ -773,7 +799,7 @@ impl Queue for Queue::ver { } nr_commands += 1; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -865,7 +891,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { + let queue_job = match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -895,7 +921,7 @@ impl Queue for Queue::ver { } } - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 0a4ee8e7563ec..e8b845cba989f 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -17,9 +17,11 @@ use kernel::{ c_str, device::Core, error::{from_err_ptr, to_result}, - devres::Devres, io::{ - mem::IoMem, + mem::{ + DevresIoMem, + IoMem, // + }, Io, IoBase, // }, @@ -564,7 +566,7 @@ impl Resources<'_> { #[derive(Clone)] pub(crate) struct G17Resources { dev: ARef, - sgx: Arc>>, + sgx: Arc>, } impl G17Resources { @@ -707,6 +709,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { @@ -775,7 +782,7 @@ fn gpu_revision( #[cfg(CONFIG_DRM_ASAHI_G16)] #[derive(Clone)] pub(crate) struct G16Resources { - sgx: Arc>>, + sgx: Arc>, } #[cfg(CONFIG_DRM_ASAHI_G16)] diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7d7d785a83133..7132e6de41c09 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,7 +18,7 @@ #include "sysfs.h" -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ +/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -48,8 +50,69 @@ struct asahi_stats_attribute { const struct asahi_stats_snapshot *snapshot; const struct asahi_gpu_memory *memory; bool export_enabled; + asahi_stats_read_fn read; + asahi_stats_suspend_fn suspend; + struct notifier_block pm_notifier; }; +/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ +static const char *asahi_stats_quality(u32 status) +{ + return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; +} + +static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) +{ + struct asahi_qualified_stats snap; + ssize_t n = 0; + + stats->read(stats->snapshot, &snap); + n += sysfs_emit_at(buf, n, "utilisation_status %s\n", + asahi_stats_quality(snap.util_status)); + if (snap.util_status == 1) { + n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); + n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); + n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); + } else { + n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" + "util_renderer_percent unsupported\n" + "util_device_percent unsupported\n"); + } + n += sysfs_emit_at(buf, n, "pstate_status %s\n", + asahi_stats_quality(snap.pstate_status)); + if (snap.pstate_status == 1) + n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); + else + n += sysfs_emit_at(buf, n, "pstate unsupported\n"); + /* No A/B selector, packet duration, measured power or temperature. */ + n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" + "achieved_frequency_hz unsupported\n" + "busy_ns unsupported\n" + "power_mw unsupported\n" + "temperature_raw unsupported\n"); + return n; +} + +static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct asahi_stats_attribute *stats = + container_of(nb, struct asahi_stats_attribute, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + stats->suspend(stats->snapshot, true); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + stats->suspend(stats->snapshot, false); + break; + } + return NOTIFY_OK; +} + static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -60,6 +123,8 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); + if (stats->read) + return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -118,14 +183,18 @@ static ssize_t gpu_memory_show(struct device *dev, * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle) + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || + (!!read != !!suspend)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); + BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); @@ -145,6 +214,16 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; + stats->read = read; + stats->suspend = suspend; + if (suspend) { + stats->pm_notifier.notifier_call = asahi_stats_pm_notify; + ret = register_pm_notifier(&stats->pm_notifier); + if (ret) { + kfree(stats); + return ret; + } + } sysfs_attr_init(&stats->memory_attr.attr); stats->memory_attr.attr.name = "gpu_memory"; @@ -154,6 +233,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->attr); if (ret) { + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -161,6 +242,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->memory_attr); if (ret) { device_remove_file(dev, &stats->attr); + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -217,6 +300,9 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; + /* Both notifier removal and file removal join readers of the Rust Arc. */ + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); @@ -230,4 +316,4 @@ EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index f1873f9be6caa..d43f482903e55 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,10 +3,30 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H +#include + struct device; +/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ +struct asahi_qualified_stats { + u32 util_status; + u32 pstate_status; + u32 tiler; + u32 renderer; + u32 device; + u32 pstate; + u64 util_host_ns; + u64 pstate_host_ns; + u64 util_samples; + u64 pstate_timestamp; +}; + +typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); +typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); + int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle); + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index cee5d52a4e261..7c34f0627b020 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,16 +3,21 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{memacct::Ledger, stats::StatsSnapshot}; +use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; +type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); +type SuspendStats = unsafe extern "C" fn(*const c_void, bool); + extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const StatsSnapshot, + snapshot: *const c_void, export_enabled: c_int, memory: *const Ledger, handle: *mut *mut c_void, + read: Option, + suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } @@ -29,7 +34,11 @@ pub(crate) fn register_idle( // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes // it with READ_ONCE/WRITE_ONCE. let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } } /// Register only after initialization succeeds, keeping the snapshot Arc alive. @@ -39,6 +48,66 @@ pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks(dev, snapshot, export_enabled, None, None) +} + +/// The caller retains the G17 snapshot Arc until unregister joins both sysfs +/// readers and the PM notifier. The callbacks never power the GPU up. +pub(crate) fn register_g17( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks( + dev, + snapshot, + export_enabled, + Some(read_g17), + Some(suspend_g17), + ) +} + +/// Copy the locked qualified snapshot to the C formatter's local output. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot retained by the registration. +/// `out` must be aligned and writable for one Readout for this call. +unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { + // SAFETY: C calls only while registration retains the snapshot Arc, and + // supplies a writable, aligned output for this call. + let snapshot = unsafe { &*snapshot.cast::() }; + let view = snapshot + .g17_reader + .as_ref() + .map_or_else(Readout::default, |reader| reader.read()); + // SAFETY: As above; the C output is initialized in one locked read. + unsafe { out.write(view) }; +} + +/// End the telemetry epoch at either side of system sleep. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot. The caller must join this +/// callback before releasing the registration's owning Arc. +unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { + // SAFETY: The notifier is unregistered and joined before the Arc is dropped. + if let Some(reader) = unsafe { &*snapshot.cast::() } + .g17_reader + .as_ref() + { + reader.suspend(suspended); + } +} + +fn register_with_callbacks( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, + read: Option, + suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -46,10 +115,12 @@ pub(crate) fn register( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot, + snapshot.cast(), c_int::from(export_enabled), &crate::memacct::LEDGER, &mut handle, + read, + suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 976ec9f6bf48b..9f13436b4caf5 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -13,10 +13,20 @@ use kernel::{prelude::*, sync::Arc}; #[cfg(CONFIG_DRM_ASAHI_G16)] use kernel::{uapi, xarray}; +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. /// CPU-retired timestamps retain their backing without consuming global firmware address space. pub(crate) enum Buffer { - Firmware(Arc), + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), #[cfg(CONFIG_DRM_ASAHI_G16)] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. @@ -27,7 +37,11 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) } /// Retain a CPU-written byte range without publishing any firmware address. @@ -46,15 +60,16 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { - Self::Firmware(mapping) => mapping.size(), + Self::Firmware(buffer) => buffer.mapping.size(), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { range, .. } => range.end - range.start, } } + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { - Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -65,10 +80,30 @@ impl Buffer { } } + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. pub(crate) fn firmware_mapping(&self) -> Result> { match self { - Self::Firmware(mapping) => Ok(mapping.clone()), + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { .. } => Err(EINVAL), } diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index a5f9455b23f5f..6e5540caeac3f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,6 +96,13 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } +/// A value that can be populated from an untrusted byte record. +/// +/// # Safety +/// +/// Every bit pattern must be valid for Self. Its byte representation must +/// contain no uninitialized bytes (including padding) when produced by Default, +/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -117,7 +124,9 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + let dst = unsafe { + core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) + }; dst.copy_from_slice(&src[..size]); self.offset = end; @@ -217,17 +226,3 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } - -/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). -/// -/// # Safety -/// -/// - `slot` is a valid pointer to uninitialized memory. -/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to -/// deallocate. -/// - `slot` will not move until it is dropped, i.e. it will be pinned. -#[inline(always)] -pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { - // SAFETY: Per safety requirement. - unsafe { init.__pinned_init(slot) } -} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 9de2e7f0f4f97..4b613b1cc7f7a 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -491,10 +491,23 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, } EXPORT_SYMBOL_GPL(drm_gem_shmem_vunmap_locked); -static int -drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle) +/** + * drm_gem_shmem_create_with_handle - Allocate an object with the given size and + * returns a GEM handle + * @file_priv: DRM file structure to create the dumb buffer for + * @dev: DRM device + * @size: Size of the object to allocate + * @handle: Returns the GEM handle on success + * + * Allocates an shmem GEM buffer using drm_gem_shmem_create() and returns + * a GEM handle to it. + * + * Returns: + * Zero on success, or an error code otherwise. + */ +int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle) { struct drm_gem_shmem_object *shmem; int ret; @@ -513,6 +526,7 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, return ret; } +EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_handle); /* Update madvise status, returns true if not purged, else * false or -errno. @@ -988,7 +1002,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; #ifdef CONFIG_MM_SUBPAGE @@ -1023,7 +1037,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 30791d328550d..8392f9e4a0109 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -168,6 +168,10 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); +int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle); + extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 66396f89cc333..418522205ee6f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -368,6 +368,60 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. */ DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /* + * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports + * them; this kernel never does. They are listed so that the numbers are + * not given to anything else, and userspace reads every kernel's + * features the same way. Bits 25 and 27 are also reserved for that + * runtime. + */ + + /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ + DRM_ASAHI_FEATURE_RENDER_RESOURCES = (1UL) << 17, + + /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2: M5: versioned render storage. */ + DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 = (1UL) << 18, + + /** @DRM_ASAHI_FEATURE_SPARSE_TILES: M5: DRM_ASAHI_BIND_SPARSE_TILE. */ + DRM_ASAHI_FEATURE_SPARSE_TILES = (1UL) << 19, + + /** @DRM_ASAHI_FEATURE_COMPUTE_MATRIX: M5: compute matrix instructions. */ + DRM_ASAHI_FEATURE_COMPUTE_MATRIX = (1UL) << 20, + + /** @DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: M5: task and mesh storage. */ + DRM_ASAHI_FEATURE_TASK_MESH_STORAGE = (1UL) << 21, + + /** @DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: M5: per-render shader writes. */ + DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES = (1UL) << 22, + + /** + * @DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: M5: compressed + * multisample depth. + */ + DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION = (1UL) << 23, + + /** + * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a + * render's tiling half. + */ + DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, + + /** + * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed + * multisample stencil. + */ + DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, + + /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ + DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -391,7 +445,10 @@ enum drm_asahi_feature { * The driver publishes an accepted job's failure before signaling its fence. * Thus userspace must query after a successful DRM syncobj wait/poll before * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. + * an execution-success indication. A job cancelled (ECANCELED) because + * another VM's recovery discarded it, an input did not signal in time, or + * its queue was closed, is not a failure of this VM: its fence carries the + * error, error stays 0, and its output is undefined. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. @@ -646,6 +703,12 @@ enum drm_asahi_bind_flags { * It is intended as a mechanism to accelerate sparse. */ DRM_ASAHI_BIND_SINGLE_PAGE = (1L << 3), + + /** + * @DRM_ASAHI_BIND_SPARSE_TILE: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_SPARSE_TILES). This kernel rejects it. + */ + DRM_ASAHI_BIND_SPARSE_TILE = (1L << 4), }; /** @@ -807,6 +870,24 @@ enum drm_asahi_cmd_type { * Followed by (possibly multiple) @drm_asahi_attachment payloads. */ DRM_ASAHI_SET_COMPUTE_ATTACHMENTS = 4, + + /** + * @DRM_ASAHI_SET_RENDER_RESOURCES: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_RENDER_RESOURCES). This kernel rejects it. + */ + DRM_ASAHI_SET_RENDER_RESOURCES = 5, + + /** + * @DRM_ASAHI_SET_COMPUTE_RESOURCES: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_COMPUTE_RESOURCES). This kernel rejects it. + */ + DRM_ASAHI_SET_COMPUTE_RESOURCES = 6, + + /** + * @DRM_ASAHI_SET_RENDER_RESOURCES_V2: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2). This kernel rejects it. + */ + DRM_ASAHI_SET_RENDER_RESOURCES_V2 = 7, }; /** @@ -901,6 +982,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 9ab7494251e28..56f9f84689804 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -65,9 +67,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -94,13 +99,18 @@ #include #include #include +#include #include #include #include +#include #include #include +#include +#include #include #include +#include #include #include #include @@ -114,6 +124,8 @@ #include #include #include +#include +#include #include #include #include @@ -134,6 +146,7 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; @@ -144,6 +157,7 @@ const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; @@ -153,8 +167,17 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; +const u32 BINDINGS_IIO_COLORTEMP = IIO_COLORTEMP; +const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; +const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; +const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; +const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; +const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; +const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; +const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index e63b62b54a17d..3b76d329688db 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -207,7 +207,8 @@ impl UnregisteredDevice { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::().as_ptr()); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// @@ -251,7 +252,7 @@ impl UnregisteredDevice { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; @@ -300,6 +301,26 @@ impl Device { self.dev.get() } + /// Reports that device execution is unavailable and the recovery method is unknown. + /// + /// May sleep: call in task context outside fence signalling and driver locks. + /// A registration guard keeps the parent bound throughout event delivery; a device + /// still being probed or already unplugged cannot deliver the event. + pub fn report_wedged(&self) -> Result { + // SAFETY: Every device context satisfies the weaker Normal context invariants. + let normal = unsafe { self.assume_ctx::() }; + let _registration = normal.registration_guard().ok_or(ENODEV)?; + let raw = self.as_raw(); + // SAFETY: The referenced device and its registered primary node remain live under + // the registration guard. A probe failure has already been rejected above. + if unsafe { (*raw).primary.is_null() } { + return Err(ENODEV); + } + // SAFETY: The device/node are live, 0 asks DRM to report an unknown recovery method, + // and a NULL task means no userspace task is attributed as guilty. + crate::error::to_result(unsafe { bindings::drm_dev_wedged_event(raw, 0, ptr::null_mut()) }) + } + /// # Safety /// /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 88709a7f9b36d..81f0a90e4a517 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -44,6 +44,11 @@ pub trait DriverFile { fn memory_stats(&self) -> Option { None } + + /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. + /// Engine names must be stable, unique and contain no whitespace or colons. + /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. + fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -136,6 +141,18 @@ impl File { // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the // file, so `postclose_callback()` cannot run concurrently. let file = unsafe { File::::from_raw(raw_file) }; + file.inner().engine_stats(|name, ns| { + // SAFETY: The DRM core owns printer throughout this callback; the format and + // name are NUL-terminated, and the vararg types match %s and %llu. + unsafe { + bindings::drm_printf( + printer, + c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), + name.as_char_ptr(), + ns as core::ffi::c_ulonglong, + ) + }; + }); let Some(stats) = file.inner().memory_stats() else { return; }; @@ -164,7 +181,7 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) }; } From f4b1301d77c938917f81937882c1d731de1507c4 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:26:09 -0500 Subject: [PATCH 048/392] selftests: asahi: account for the paired-header fixture through its runner Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 53f803bddc8e5..97087990539df 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -186,6 +186,9 @@ def mutations(flag, names, base, expect=KILL): def jobs(work, conf, dtc): found = {p.name for p in ASAHI.iterdir() if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} + # This Rust fixture is compiled and executed by the paired-header wrapper. + if 'test-m3-uapi.py' in ASAHI_TESTS: + found.discard('m3-uapi-tests.rs') known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) missing = sorted(found - known) gone = sorted(name for name in known if not (ASAHI / name).is_file()) From 4b07a28daa8348cb232b860874fac0c8bf2513b6 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:33:34 -0500 Subject: [PATCH 049/392] selftests: apple: accept the explicitly registered PMGR fixture Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 00b9d65f68575..540858e834a6f 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -189,7 +189,7 @@ def jobs(work, conf, dtc): if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) missing = sorted(found - known) - gone = sorted(known - found) + gone = sorted(name for name in known if not (ASAHI / name).is_file()) if missing or gone: for name in missing: print(f'not registered in {Path(__file__).name}: tools/asahi/{name}', file=sys.stderr) From 3965338688b11aa16ae04bdf5e5b4ae11357103d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:49:04 -0500 Subject: [PATCH 050/392] ci: run the carried SEP identity-preservation regressions Signed-off-by: Satya Benson --- .github/workflows/sep-source.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index e77ffd8ad7340..114f5c240434e 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -35,6 +35,16 @@ jobs: EOF cc -Wall -Werror -Iinclude/uapi /tmp/sep-abi.c -o /tmp/sep-abi /tmp/sep-abi + - name: Select the identity-test Rust toolchain + run: | + rustup toolchain install 1.85.0 --profile minimal --no-self-update + echo "RUSTUP_TOOLCHAIN=1.85.0" >> "$GITHUB_ENV" + - name: Test saved identity readers + run: | + python3 -m unittest tools/aurora-sep/tests/test_store_open_errors.py -v + python3 -O -m unittest tools/aurora-sep/tests/test_store_open_errors.py -v + python3 -m unittest tools/aurora-sep/tests/test_identity_preservation.py -v + python3 -O -m unittest tools/aurora-sep/tests/test_identity_preservation.py -v - name: Test recovery utility and package sources run: | python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v From a2dbf7ecde3015d447341b1761d78a8bb6ab44f5 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:49:04 -0500 Subject: [PATCH 051/392] ci: run the J613 selected-policy diagnostic regressions Signed-off-by: Satya Benson --- .github/workflows/sep-source.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 114f5c240434e..fe24b285eb6b5 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -45,6 +45,10 @@ jobs: python3 -O -m unittest tools/aurora-sep/tests/test_store_open_errors.py -v python3 -m unittest tools/aurora-sep/tests/test_identity_preservation.py -v python3 -O -m unittest tools/aurora-sep/tests/test_identity_preservation.py -v + - name: Test selected-policy diagnostics + run: | + python3 -m unittest tools/aurora-sep/tests/test_diag_policy.py tools/aurora-sep/tests/test_diag_sensor.py -v + python3 -O -m unittest tools/aurora-sep/tests/test_diag_policy.py tools/aurora-sep/tests/test_diag_sensor.py -v - name: Test recovery utility and package sources run: | python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v From 95f3e2a637284e0ffc8b7a981acbe9bb2a7e3177 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:59:22 -0500 Subject: [PATCH 052/392] drm/asahi: use the 7.3 MMIO mapping transaction and ownership contract Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/mmu/m3.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a813426..e9dc7184d6582 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -56,7 +56,7 @@ impl Vm { return Err(EINVAL); } // SAFETY: As in `map_io`, the exec lock is only used for this mapping. - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; let inner = vm_exec.unique_gpuvm.data(); let payload = KernelMappingInner { owner: self.gpuvm.clone(), @@ -82,18 +82,35 @@ impl Vm { let iova = node.start(); let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); let ret = inner .page_table .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec lock before the node, as in `map_io`. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. From 4abcf6603441027e0fce2798a0fc4151569b8b20 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 22:00:06 -0500 Subject: [PATCH 053/392] drm/asahi: expose adopted upper-root selection to the T8122 backend Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/mmu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..55a9fb7de0c22 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3140,7 +3140,7 @@ impl Uat { /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_M3_T8122))] pub(crate) fn new_adopted_with_user_upper( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, From 9f442fa11953d5b1af1bfb050dbad3806ea4639b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 22:47:00 -0500 Subject: [PATCH 054/392] drm/asahi: adapt T8122 resources and module pins to 7.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use lifetime-aware managed MMIO, the current coherent DMA address accessor and module metadata. Keep Aurora’s identity-selected hwdata block instead of restoring the older diagnostic opt-out. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 24 +++++++---------------- drivers/gpu/drm/asahi/m3/m3_device.rs | 7 +++---- drivers/gpu/drm/asahi/m3/m3_memory.rs | 4 ++-- drivers/gpu/drm/asahi/m3/m3_params.rs | 12 ------------ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 8 ++++---- 5 files changed, 16 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 1893c113cf2a7..983917b3b5686 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -920,27 +920,17 @@ fn build_images( for &(at, value) in hwdata_words { write(&mut hwdata, at, &value.to_le_bytes())?; } - // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 - // words of crate::m3::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + // T8122's exact chip/firmware identity selects this block on Aurora; keep the + // existing constant model instead of restoring omacom's diagnostic opt-out. if crate::m3::t8122_start::is_t8122(soc) && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT) && cfg.gpu_gen == hw::GpuGen::G15 { - if m3_params::t8122_hwdata_418c() { - let a = hwdata - .get_mut(HWDATA_A..HWDATA_A + size_of::()) - .ok_or(EINVAL)?; - t8122_hwdata::write(a).map_err(|()| EINVAL)?; - dev_info!( - dev, - "M3 G15G: HwDataA +0x418c..+0x41db: wrote the 20 words (asahi.t8122_hwdata_418c=0 leaves them 0)\n" - ); - } else { - dev_info!( - dev, - "M3 G15G: HwDataA +0x418c..+0x41db left 0 (asahi.t8122_hwdata_418c=0)\n" - ); - } + let a = hwdata + .get_mut(HWDATA_A..HWDATA_A + size_of::()) + .ok_or(EINVAL)?; + t8122_hwdata::write(a).map_err(|()| EINVAL)?; + dev_info!(dev, "M3 G15G: wrote identity-selected HwDataA +0x418c block\n"); } // On T8122, log and require the two HwDataB words the firmware's power management depends // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index c43017a8933c0..caef9657b4c9f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -7,10 +7,9 @@ use kernel::{ bindings, c_str, device::Core, - devres::Devres, error::to_result, io::{ - mem::IoMem, + mem::DevresIoMem, Io, IoBase, }, @@ -28,8 +27,8 @@ const ASC_CPU_RUN: u32 = 1 << 4; pub(crate) struct Device { dev: ARef, - asc: Devres>, - sgx: Devres>, + asc: DevresIoMem<0x4000>, + sgx: DevresIoMem, firmware: Firmware, core_mask: u64, soc: &'static Soc, diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs index 5e8018a802e70..6ba456e382e57 100644 --- a/drivers/gpu/drm/asahi/m3/m3_memory.rs +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -68,7 +68,7 @@ impl Buffer { // diagnostic; all mappings retire before normal DMA allocation Drop. let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL, kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; - let pa:usize=object.dma_handle().try_into()?; + let pa:usize=object.dma_address().try_into()?; let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?} else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; @@ -100,7 +100,7 @@ impl Buffer { pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { let address=self.gpu_va()?&!0x3fff; match &mut self.object { - Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_address().try_into()?,object.size(),self.gpu_protection), Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), } } diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index 8167c6c3ec05c..85033c88e4192 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -665,18 +665,6 @@ m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number); static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); -/// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) -/// writes the 20 words at HwDataA +0x418c ([`crate::m3::t8122_hwdata`]); 0 leaves the -/// range zero, for an A/B test. Readable in sysfs. -static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::m3::t8122_hwdata::DEFAULT); -m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::m3::t8122_hwdata::parse, 0o444, - Some(get_atomic_param)); - -/// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). -pub(crate) fn t8122_hwdata_418c() -> bool { - crate::m3::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) -} - /// The T8122 start-experiment parameters of this boot, as given. pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw { let get = |p: &AtomicU64| p.load(Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 1a366d7f4b340..f9dd341aa89fb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -991,7 +991,7 @@ static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); fn pin_module() -> bool { // SAFETY: THIS_MODULE is this module, whose code is running; try_module_get fails once // removal has begun instead of reviving the reference count. - unsafe { kernel::bindings::try_module_get(crate::THIS_MODULE.as_ptr()) } + unsafe { kernel::bindings::try_module_get(kernel::module::this_module::().as_ptr()) } } /// Without module unloading there is no removal to race. #[cfg(not(CONFIG_MODULE_UNLOAD))] @@ -1000,7 +1000,7 @@ fn pin_module() -> bool { true } #[cfg(CONFIG_MODULE_UNLOAD)] fn unpin_module() { // SAFETY: called only to drop a reference pin_module took. - unsafe { kernel::bindings::module_put(crate::THIS_MODULE.as_ptr()) }; + unsafe { kernel::bindings::module_put(kernel::module::this_module::().as_ptr()) }; } #[cfg(not(CONFIG_MODULE_UNLOAD))] fn unpin_module() {} @@ -1074,7 +1074,7 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { - unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; + unsafe {kernel::bindings::__module_get(kernel::module::this_module::().as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); core::mem::forget(drm);core::mem::forget(device); @@ -1285,7 +1285,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. - unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + unsafe { kernel::bindings::__module_get(kernel::module::this_module::().as_ptr()) }; return; } // SAFETY: no GPU command is pending and ASC is stopped. Field order From a8be37d13bddd51b009f8d6e69556f5d8af5ff34 Mon Sep 17 00:00:00 2001 From: "Michal S." Date: Fri, 6 Nov 2026 22:11:34 +0000 Subject: [PATCH 055/392] wifi: brcmfmac: select CCMP explicitly for firmware offload On BCM4388, an SAE join that requests CCMP can return a driver-generated association RSNE selecting GCMP-256 instead. wpa_supplicant rejects that cipher because it is outside the profile's allowed set and disconnects locally with PAIRWISE_CIPHER_NOT_VALID. The CCMP arm sets the generic AES_ENABLED flag but leaves the wsec_info algorithm selection and mask at zero. The firmware ABI leaves algorithms outside the mask unchanged, so this does not select CCMP over GCMP-256. Select AES-CCM explicitly and mask AES-GCM256, mirroring the existing GCMP-256 arm. The existing GCMP feature guard keeps this command away from firmware that does not advertise it. Add host controls that execute the production cipher-selection and XTLV encoding functions and observe the firmware command bytes. Check the CCMP mask, existing GCMP and legacy paths, and setup failures. The original code fails the CCMP payload assertion. Hardware verification of the fix is pending. Fixes: 999b7240c323 ("[brcmfmac] Support GCMP cipher suite, used by WPA3.") Assisted-by: LLM Signed-off-by: Michal S. --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 3 + tools/asahi/brcmf-cipher-tests.py | 342 ++++++++++++++++++ tools/asahi/run-host-tests.py | 2 + 3 files changed, 347 insertions(+) create mode 100755 tools/asahi/brcmf-cipher-tests.py diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 078a689320ef2..1cbdc1cdd18ba 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2001,6 +2001,9 @@ brcmf_set_wsec_mode(struct net_device *ndev, break; case WLAN_CIPHER_SUITE_CCMP: pval = AES_ENABLED; + /* SAE offload builds its own RSNE; select CCMP explicitly. */ + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); break; case WLAN_CIPHER_SUITE_AES_CMAC: pval = AES_ENABLED; diff --git a/tools/asahi/brcmf-cipher-tests.py b/tools/asahi/brcmf-cipher-tests.py new file mode 100755 index 0000000000000..0ac22f07b3547 --- /dev/null +++ b/tools/asahi/brcmf-cipher-tests.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Observe brcmfmac's cipher-selection commands at the firmware boundary.""" + +import argparse +import importlib.util +import json +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = Path("drivers/net/wireless/broadcom/brcm80211/brcmfmac") +spec = importlib.util.spec_from_file_location( + "association_controls", ROOT / "tools/asahi/brcmf-assoc-tests.py" +) +association = importlib.util.module_from_spec(spec) +spec.loader.exec_module(association) + +PREFIX = r""" + +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef int32_t s32; +#define BIT(n) (1U << (n)) +#define GFP_KERNEL 0 +#define brcmf_dbg(...) ((void)0) +#define brcmf_err(...) ((void)0) +#define bphy_err(...) ((void)0) +#define WARN(test, ...) assert(!(test)) +#define WARN_ON(test) assert(!(test)) +#define WLAN_CIPHER_SUITE_WEP40 0x000fac01 +#define WLAN_CIPHER_SUITE_TKIP 0x000fac02 +#define WLAN_CIPHER_SUITE_CCMP 0x000fac04 +#define WLAN_CIPHER_SUITE_WEP104 0x000fac05 +#define WLAN_CIPHER_SUITE_AES_CMAC 0x000fac06 +#define WLAN_CIPHER_SUITE_GCMP_256 0x000fac09 +#define BRCMF_FEAT_GCMP 1 +struct brcmf_pub { + bool gcmp; +}; +struct brcmf_if { + struct brcmf_pub *drvr; +}; +struct brcmf_cfg80211_security { + u32 cipher_pairwise, cipher_group; +}; +struct brcmf_cfg80211_profile { + struct brcmf_cfg80211_security sec; +}; +struct net_device { + struct brcmf_if *ifp; + struct brcmf_cfg80211_profile profile; +}; +struct cfg80211_connect_params { + struct { + unsigned int n_ciphers_pairwise; + u32 ciphers_pairwise[1]; + u32 cipher_group; + } crypto; + const u8 *ie; + unsigned int ie_len; + bool privacy; +}; +static unsigned int wsec_calls, algorithm_calls, allocations; +static u32 observed_wsec, observed_algos, observed_mask, firmware_algos; +static int wsec_error, algorithm_error; +static bool allocation_error; +static void *netdev_priv(struct net_device *ndev) +{ + return ndev->ifp; +} +static struct brcmf_cfg80211_profile *ndev_to_prof(struct net_device *ndev) +{ + return &ndev->profile; +} +static bool brcmf_feat_is_enabled(struct brcmf_if *ifp, int feature) +{ + assert(feature == BRCMF_FEAT_GCMP); + return ifp->drvr->gcmp; +} +static bool brcmf_find_wpsie(const u8 *ie, unsigned int len) +{ + assert(!ie && !len); + return false; +} +static void *kzalloc(size_t size, int flags) +{ + assert(flags == GFP_KERNEL); + if (allocation_error) + return NULL; + void *data = calloc(1, size); + assert(data); + allocations++; + return data; +} +static void kfree(void *data) +{ + assert(allocations); + allocations--; + free(data); +} +static u32 read_le32(const u8 *data) +{ + return (u32)data[0] | (u32)data[1] << 8 | (u32)data[2] << 16 | + (u32)data[3] << 24; +} +static void put_unaligned_le16(u16 value, u8 *data) +{ + data[0] = value; + data[1] = value >> 8; +} +static int brcmf_fil_bsscfg_int_set(struct brcmf_if *ifp, const char *name, + u32 value) +{ + (void)ifp; + assert(!strcmp(name, "wsec")); + assert(!wsec_calls && !algorithm_calls); + wsec_calls++; + observed_wsec = value; + return wsec_error; +} +static int brcmf_fil_bsscfg_data_set(struct brcmf_if *ifp, const char *name, + const void *data, u32 len) +{ + const u8 *bytes = data; + (void)ifp; + assert(!strcmp(name, "wsec_info")); + assert(wsec_calls == 1 && !algorithm_calls); + /* Version 1, one BSS_ALGOS (0x106) TLV with two little-endian u32s. */ + assert(len == 16); + assert(bytes[0] == 1 && !bytes[1] && !bytes[2] && bytes[3] == 1); + assert(bytes[4] == 6 && bytes[5] == 1 && bytes[6] == 8 && !bytes[7]); + observed_algos = read_le32(bytes + 8); + observed_mask = read_le32(bytes + 12); + algorithm_calls++; + if (algorithm_error) + return algorithm_error; + /* Firmware ABI: algorithms outside the mask remain unchanged. */ + firmware_algos = (firmware_algos & ~observed_mask) | + (observed_algos & observed_mask); + return 0; +} +""" + +CONTROLS = r""" + +static struct brcmf_pub pub; +static struct brcmf_if ifp = { &pub }; +static struct net_device ndev = { .ifp = &ifp }; +static struct cfg80211_connect_params params; + +static void reset(bool gcmp, u32 pairwise, u32 group) +{ + assert(!allocations); + pub.gcmp = gcmp; + memset(¶ms, 0, sizeof(params)); + params.crypto.n_ciphers_pairwise = 1; + params.crypto.ciphers_pairwise[0] = pairwise; + params.crypto.cipher_group = group; + params.privacy = true; + memset(&ndev.profile, 0, sizeof(ndev.profile)); + observed_wsec = observed_algos = observed_mask = 0; + wsec_calls = algorithm_calls = 0; + wsec_error = algorithm_error = 0; + allocation_error = false; + /* GCMP-256 starts enabled; bit 31 represents an unrelated algorithm. */ + firmware_algos = 0x80010000U; +} + +int main(void) +{ + unsigned int controls = 0; + reset(true, WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_CCMP); + assert(!brcmf_set_wsec_mode(&ndev, ¶ms)); + assert(observed_wsec == 4 && algorithm_calls == 1); + assert(observed_algos == 0x10 && observed_mask == 0x10010); + assert(firmware_algos == 0x80000010U); + assert(ndev.profile.sec.cipher_pairwise == WLAN_CIPHER_SUITE_CCMP); + assert(ndev.profile.sec.cipher_group == WLAN_CIPHER_SUITE_CCMP); + assert(!allocations); + controls++; + + reset(false, WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_CCMP); + assert(!brcmf_set_wsec_mode(&ndev, ¶ms)); + assert(observed_wsec == 4 && !algorithm_calls); + assert(firmware_algos == 0x80010000U); + controls++; + + const u32 groups[] = { WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_GCMP_256 }; + for (unsigned int i = 0; i < 2; i++) { + reset(true, WLAN_CIPHER_SUITE_GCMP_256, groups[i]); + assert(!brcmf_set_wsec_mode(&ndev, ¶ms)); + assert(observed_wsec == 4 && algorithm_calls == 1); + assert(observed_algos == 0x10000 && observed_mask == 0x10010); + assert(firmware_algos == 0x80010000U); + assert(!allocations); + controls++; + } + + const u32 legacy[] = { WLAN_CIPHER_SUITE_TKIP, WLAN_CIPHER_SUITE_WEP40, + WLAN_CIPHER_SUITE_WEP104 }; + const u32 flags[] = { 2, 1, 1 }; + for (unsigned int i = 0; i < 3; i++) { + reset(true, legacy[i], legacy[i]); + assert(!brcmf_set_wsec_mode(&ndev, ¶ms)); + assert(observed_wsec == flags[i] && algorithm_calls == 1); + assert(!observed_algos && !observed_mask); + assert(firmware_algos == 0x80010000U && !allocations); + controls++; + } + + reset(false, WLAN_CIPHER_SUITE_GCMP_256, WLAN_CIPHER_SUITE_CCMP); + assert(brcmf_set_wsec_mode(&ndev, ¶ms) == -EOPNOTSUPP); + assert(!wsec_calls && !algorithm_calls); + controls++; + + reset(true, WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_CCMP); + wsec_error = -EIO; + assert(brcmf_set_wsec_mode(&ndev, ¶ms) == -EIO); + assert(wsec_calls == 1 && !algorithm_calls && !allocations); + assert(!ndev.profile.sec.cipher_pairwise); + controls++; + + reset(true, WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_CCMP); + algorithm_error = -EIO; + assert(brcmf_set_wsec_mode(&ndev, ¶ms) == -EIO); + assert(algorithm_calls == 1 && firmware_algos == 0x80010000U); + assert(!ndev.profile.sec.cipher_pairwise && !allocations); + controls++; + + reset(true, WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_CCMP); + allocation_error = true; + assert(brcmf_set_wsec_mode(&ndev, ¶ms) == -ENOMEM); + assert(!algorithm_calls && !allocations); + assert(!ndev.profile.sec.cipher_pairwise); + controls++; + + printf("%u production cipher-command controls passed\n", controls); + return 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", help="Git source revision to exercise") + parser.add_argument("--out", type=Path, help="retain executable and results") + args = parser.parse_args() + + def read(path): + if args.ref: + return subprocess.check_output( + ["git", "show", f"{args.ref}:{path}"], cwd=ROOT, text=True + ) + return (ROOT / path).read_text() + + cfg = read(DRIVER / "cfg80211.c") + types = read(DRIVER / "fwil_types.h") + xtlv = read(DRIVER / "xtlv.h") + wifi = read(DRIVER.parent / "include/brcmu_wifi.h") + constants = "\n".join( + line + for source in (cfg, types, wifi) + for line in source.splitlines() + if re.match( + r"#define\s+(KEY_ALGO_MASK\(|WL_WSEC_INFO_BSS_(?:BASE|ALGOS)\s|" + r"BRCMF_WSEC_INFO_VER\s|(?:AES|TKIP|WEP)_ENABLED\s|" + r"CRYPTO_ALGO_AES_(?:CCM|GCM256)\s)", + line, + ) + ) + parts = [PREFIX, constants] + for name in ("brcmf_wsec_info_tlv", "brcmf_wsec_info"): + parts.append(association.structure(types, name)) + parts.append(association.structure(xtlv, "brcmf_xtlv")) + start = xtlv.index("enum brcmf_xtlv_option {") + parts.append(xtlv[start : xtlv.index("};", start) + 2]) + parts.append( + association.function(read(DRIVER / "xtlv.c"), "brcmf_xtlv_pack_header") + ) + for name in ("brcmf_set_wsec_info_algos", "brcmf_set_wsec_mode"): + parts.append(association.function(cfg, name)) + code = "\n".join(parts) + CONTROLS + + def run(out): + out.mkdir(parents=True, exist_ok=True) + source = out / "controls.c" + source.write_text(code) + executable = out / "controls" + subprocess.run( + [ + "clang", + "-std=gnu11", + "-O1", + "-g", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-unused-variable", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=all", + str(source), + "-o", + str(executable), + ], + check=True, + ) + result = subprocess.run([str(executable)], capture_output=True, text=True) + receipt = { + "exit": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "ref": args.ref, + "limits": ( + "Production command encoding at an observing host boundary; " + "no firmware or hardware test." + ), + } + (out / "results.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(result.stdout or result.stderr, end="") + return result.returncode != 0 + + if args.out: + return run(args.out) + with tempfile.TemporaryDirectory(prefix="brcmf-cipher-") as out: + return run(Path(out)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 95444268423a3..cbeca96b33ca6 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -85,6 +85,8 @@ def mutations(flag, names, base, expect=KILL): 'brcmf-assoc-tests.py': [(out('--output'), PASS)] + mutations('--mutation', ('short-ie', 'header-length', 'skb-capacity', 'cmplt-header'), out('--output')), 'brcmf-bss-tests.py': [(out(), PASS)], + 'brcmf-cipher-tests.py': [(out(), PASS)] + + [(out() + ['--ref', '33bec4016c466985bfbb5ff4c2089fcab8242dcf'], KILL)], # The old revision reports a failed join against the last associated BSS. 'brcmf-join-bssid-tests.py': [(out(), PASS)] + mutations('--mutation', ('stale-target', 'zero-named', 'no-call'), out()), From 72f8eecf708bb43e3409cade6fbbdcb7b3ff292a Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sat, 7 Nov 2026 04:21:48 -0500 Subject: [PATCH 056/392] arm64: idle: T6050 idles in WFI even with idle=nop m1n1 appends "idle=nop arm64.nowfxt" to the command line on every chip whose Apple system registers are locked (M4 and later), for kernels that do not preserve the state WFI loses there. It appends them after the user's arguments, so an idle= on the command line cannot override it. This kernel preserves that state (apple_wfi_quirk_init(), and WFxT is forced off for M4 and later in idreg-override), so on T6050 idle=nop only makes every idle CPU spin: cpuidle's two WFI states call cpu_do_idle(), which does nothing with idle=nop. Treat T6050 (MIDR parts 0x064 and 0x065) like T8142 and keep WFI as the idle method. On the 16-inch M5 Pro (J716s) the system then draws 9.8 W at idle on the charger instead of 16.4 W (SMC total system power), and a 56-minute idle soak with an FP consistency check on all 18 CPUs and four s2idle steps up to a real suspend/resume passed. On the 14-inch M5 Pro (J714s), betizzel measured 5.6 W against 15.3 W, and a per-CPU register check passed 250,000 1 ms sleeps on each of its 15 CPUs. Add apple_keep_idle_nop to keep idle=nop for comparisons and as a fallback that needs no new kernel. Link: https://github.com/betizzel/omarchy-m5/commit/a5df9e4df61b60dd6195e13a2a3a043d76b92c0f Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- .../admin-guide/kernel-parameters.txt | 7 +++++ arch/arm64/kernel/idle.c | 27 ++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9526a1b399fcd..2e6fa8b49752e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -552,6 +552,13 @@ Kernel parameters apm= [APM] Advanced Power Management See header of arch/x86/kernel/apm_32.c. + apple_keep_idle_nop + [ARM64,EARLY] On T6050 and T8142, keep idle=nop. + These cores idle in WFI by default, and an idle=nop + that the boot loader appends is ignored, because the + kernel preserves the registers WFI loses there. With + this flag idle=nop spins the idle loop as elsewhere. + apparmor= [APPARMOR] Disable or enable AppArmor at boot time Format: { "0" | "1" } See security/apparmor/Kconfig help text diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 267677adc220a..0d76030f78205 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -50,6 +50,18 @@ early_param("idle", setup_idle); * yet, so it is a plain flag. */ static bool apple_wfi_loses_regs __ro_after_init; +static bool apple_keep_idle_nop __initdata; + +/* + * The boot loader appends idle=nop after the user's arguments, so it cannot + * be overridden on the command line. This flag keeps it anyway. + */ +static int __init setup_apple_keep_idle_nop(char *arg) +{ + apple_keep_idle_nop = true; + return 0; +} +early_param("apple_keep_idle_nop", setup_apple_keep_idle_nop); void __init apple_wfi_quirk_init(void) { @@ -60,14 +72,17 @@ void __init apple_wfi_quirk_init(void) pr_info("Apple: preserving registers across WFI\n"); } /* - * aura passes idle=nop on M5 for kernels without this quirk. Spinning - * idle keeps all ten cores busy (cpuidle's WFI state spins too), and - * the SoC sits near 90 C and throttles; WFI is safe here. + * aura (J813) and m1n1 (T6050) pass idle=nop on M4 and later for + * kernels without this quirk. Spinning idle keeps every core busy + * (cpuidle's WFI states spin too): on T8142 the SoC sits near 90 C + * and throttles, on T6050 the system draws about 10 W more at idle. + * WFI is safe here. */ - if (idle == ARM64_IDLE_NOP && - (model == MIDR_APPLE_T8142_E || model == MIDR_APPLE_T8142_P)) { + if (idle == ARM64_IDLE_NOP && !apple_keep_idle_nop && + (model == MIDR_APPLE_T8142_E || model == MIDR_APPLE_T8142_P || + model == MIDR_APPLE_T6050_064 || model == MIDR_APPLE_T6050_065)) { idle = ARM64_IDLE_WFI; - pr_info("Apple: T8142 idles in WFI, ignoring idle=nop\n"); + pr_info("Apple: idling in WFI, ignoring idle=nop\n"); } } From 6ca4d4fa5a339eb761ff967ed6b8858f720844a2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 7 Nov 2026 12:23:14 -0500 Subject: [PATCH 057/392] drm/asahi: initialize the M3 backend slot in the M5 and G18 probes The M5 and G18 probes came in with integration/aurora after this series gave AsahiPlatformDriverData an m3 field, so a kernel with CONFIG_DRM_ASAHI_M5 or CONFIG_DRM_ASAHI_G18 no longer builds. Neither probe starts an M3 backend: set the field to None. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index fd5bc2157a26b..24b9ac9ae2bea 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -535,6 +535,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, #[cfg(CONFIG_DRM_ASAHI_G16)] g16: None, raw_dev: core::ptr::null_mut(), @@ -629,6 +630,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, #[cfg(CONFIG_DRM_ASAHI_G16)] g16: None, raw_dev: core::ptr::null_mut(), From 57af6957a795967cb1425c657862f1fbd69b81b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:16:10 +1100 Subject: [PATCH 058/392] nfc: apple-stockholm: add an opt-in read-only Secure Element probe The controller fronts an embedded Secure Element whose card manager is not ours. Add a way to look at it without being able to change it. With se_probe=1 (default off, load time only) the driver creates, in debugfs/apple-stockholm//: nfcee power up, CORE_RESET/CORE_INIT, NFCEE discovery, park se_probe the same, then, if an NFCEE takes ISO 7816 APDUs directly, a logical connection to it and a fixed table of SELECT by name and GET DATA (CPLC, card data, key information), then park apdu_allowlist the command table, without touching the hardware Commands come only from the compiled-in table. Each one, including the GET RESPONSE and corrected-Le retries the probe builds, passes stockholm_se_apdu_allowed() right before it is sent: CLA 00 SELECT by DF name, CLA 80 GET DATA of tags 9F7F, 0066 and 00E0, GET RESPONSE, nothing else. No authentication, content or key management, life-cycle or store command can be sent; a failed GlobalPlatform authentication can lock a card for good. An NFCEE that the probe enables is disabled again. The files are 0400 and need CAP_SYS_ADMIN. The report is wiped on release because CPLC carries the chip serial number. The receive handler now also queues a copy of every packet while a probe runs, so notifications (NFCEE discovery, connection credits) and data packets reach the probe; outside a run nothing changes. On the J700 (firmware 37010160) the controller reports one NFCEE, 0x01: the HCI network, disabled, HCI access only. No NFCEE takes APDUs directly, so se_probe stops after discovery and nothing reaches the Secure Element. Reaching applets there would need HCI pipes, which change controller state that macOS shares; this does not do that. KUnit: the table passes the allowlist; INITIALIZE UPDATE, EXTERNAL AUTHENTICATE, INSTALL, DELETE, LOAD, PUT KEY, SET STATUS, STORE DATA, PUT DATA, VERIFY, CHANGE REFERENCE DATA, GET STATUS, MANAGE CHANNEL and GET CHALLENGE are refused, as are SELECT and GET DATA variants with other classes, P1/P2, tags or lengths; NFCEE protocol parsing including the J700 notification. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/nfc/apple-stockholm-test.c | 122 +++- drivers/nfc/apple-stockholm.c | 886 ++++++++++++++++++++++++++++- 2 files changed, 999 insertions(+), 9 deletions(-) diff --git a/drivers/nfc/apple-stockholm-test.c b/drivers/nfc/apple-stockholm-test.c index 3e0c9644f9668..2274daeff5708 100644 --- a/drivers/nfc/apple-stockholm-test.c +++ b/drivers/nfc/apple-stockholm-test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * KUnit tests for the Apple Stockholm NCI identity parsing + * KUnit tests for the Apple Stockholm NCI identity parsing and the read-only + * Secure Element probe allowlist * * The responses are the ones the J616s and the J700 controllers returned. * @@ -105,12 +106,131 @@ static void stockholm_test_init_errors(struct kunit *test) KUNIT_EXPECT_EQ(test, stockholm_parse_init_rsp(rsp, sizeof(rsp), &id), -EIO); } +/* Every command the probe sends passes the allowlist, unchanged. */ +static void stockholm_test_se_table(struct kunit *test) +{ + const struct stockholm_se_apdu *cmd; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(stockholm_se_apdus); i++) { + cmd = &stockholm_se_apdus[i]; + KUNIT_EXPECT_TRUE_MSG(test, + stockholm_se_apdu_allowed(cmd->apdu, + cmd->len), + "%s", cmd->name); + KUNIT_EXPECT_LE(test, cmd->len, sizeof(cmd->apdu)); + } +} + +/* + * GlobalPlatform authentication, content and key management, and anything + * else that writes or could lock the card, is refused. + */ +static void stockholm_test_se_refused(struct kunit *test) +{ + static const u8 refused[][13] = { + /* INITIALIZE UPDATE, EXTERNAL AUTHENTICATE */ + { 0x80, 0x50, 0x00, 0x00, 0x08, 1, 2, 3, 4, 5, 6, 7, 8 }, + { 0x84, 0x82, 0x00, 0x00, 0x08, 1, 2, 3, 4, 5, 6, 7, 8 }, + /* INSTALL, DELETE, LOAD, PUT KEY, SET STATUS, STORE DATA */ + { 0x80, 0xe6, 0x0c, 0x00, 0x05, 1, 2, 3, 4, 5 }, + { 0x80, 0xe4, 0x00, 0x00, 0x05, 0x4f, 3, 1, 2, 3 }, + { 0x80, 0xe8, 0x00, 0x00, 0x05, 1, 2, 3, 4, 5 }, + { 0x80, 0xd8, 0x00, 0x81, 0x05, 1, 2, 3, 4, 5 }, + { 0x80, 0xf0, 0x80, 0x07, 0x05, 1, 2, 3, 4, 5 }, + { 0x80, 0xe2, 0x80, 0x00, 0x05, 1, 2, 3, 4, 5 }, + /* PUT DATA, VERIFY PIN, CHANGE REFERENCE DATA, GET STATUS */ + { 0x80, 0xda, 0x00, 0x66, 0x05, 1, 2, 3, 4, 5 }, + { 0x00, 0x20, 0x00, 0x80, 0x05, 1, 2, 3, 4, 5 }, + { 0x00, 0x24, 0x00, 0x80, 0x05, 1, 2, 3, 4, 5 }, + { 0x80, 0xf2, 0x80, 0x00, 0x05, 0x4f, 0x00, 0x00, 0x00, 0x00 }, + /* MANAGE CHANNEL, GET CHALLENGE */ + { 0x00, 0x70, 0x00, 0x00, 0x01, 0x00 }, + { 0x00, 0x84, 0x00, 0x00, 0x08 }, + }; + static const u8 lens[] = { 13, 13, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 6, 5 }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(refused); i++) + KUNIT_EXPECT_FALSE_MSG(test, + stockholm_se_apdu_allowed(refused[i], + lens[i]), + "INS %02x", refused[i][1]); +} + +/* The allowed instructions only in their exact read-only form. */ +static void stockholm_test_se_variants(struct kunit *test) +{ + /* SELECT with a logical channel, secure messaging, by file ID, next */ + static const u8 sel_chan[] = { 0x01, 0xa4, 0x04, 0x00, 0x00 }; + static const u8 sel_sm[] = { 0x0c, 0xa4, 0x04, 0x00, 0x00 }; + static const u8 sel_fid[] = { 0x00, 0xa4, 0x00, 0x00, 0x00 }; + static const u8 sel_next[] = { 0x00, 0xa4, 0x04, 0x02, 0x00 }; + /* SELECT whose Lc does not match the length, or names too short */ + static const u8 sel_lc[] = { 0x00, 0xa4, 0x04, 0x00, 0x07, 0xa0, 0x00, + 0x00, 0x00, 0x03, 0x10, 0x10 }; + static const u8 sel_short[] = { 0x00, 0xa4, 0x04, 0x00, 0x02, 0xa0, + 0x00, 0x00 }; + /* GET DATA of another tag, with secure messaging, with data */ + static const u8 gd_tag[] = { 0x80, 0xca, 0x00, 0x45, 0x00 }; + static const u8 gd_sm[] = { 0x84, 0xca, 0x9f, 0x7f, 0x00 }; + static const u8 gd_data[] = { 0x80, 0xca, 0x9f, 0x7f, 0x01, 0x00, 0x00 }; + /* GET RESPONSE with parameters */ + static const u8 gr_p1[] = { 0x00, 0xc0, 0x01, 0x00, 0x10 }; + static const u8 ok_gr[] = { 0x00, 0xc0, 0x00, 0x00, 0x10 }; + + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_chan, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_sm, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_fid, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_next, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_lc, + sizeof(sel_lc))); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(sel_short, + sizeof(sel_short))); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(gd_tag, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(gd_sm, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(gd_data, + sizeof(gd_data))); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(gr_p1, 5)); + KUNIT_EXPECT_FALSE(test, stockholm_se_apdu_allowed(ok_gr, 4)); + KUNIT_EXPECT_TRUE(test, stockholm_se_apdu_allowed(ok_gr, 5)); +} + +/* NFCEE_DISCOVER_NTF protocol lists, including truncated ones. */ +static void stockholm_test_se_nfcee(struct kunit *test) +{ + static const u8 apdu_hci[] = { 0xc0, 0x01, 0x02, 0x00, 0x01, 0x00 }; + static const u8 hci_only[] = { 0x01, 0x00, 0x01, 0x01, 0x00 }; + static const u8 cut[] = { 0xc0, 0x01, 0x03, 0x01 }; + /* What the J700 reported: the HCI network, disabled, HCI access only. */ + static const u8 j700[] = { 0x01, 0x01, 0x01, 0x01, 0x00 }; + + const u8 apdu = NCI_NFCEE_INTERFACE_APDU; + const u8 hci = NCI_NFCEE_INTERFACE_HCI_ACCESS; + + KUNIT_EXPECT_TRUE(test, + stockholm_se_nfcee_has(apdu_hci, sizeof(apdu_hci), apdu)); + KUNIT_EXPECT_FALSE(test, + stockholm_se_nfcee_has(hci_only, sizeof(hci_only), apdu)); + KUNIT_EXPECT_TRUE(test, + stockholm_se_nfcee_has(hci_only, sizeof(hci_only), hci)); + KUNIT_EXPECT_FALSE(test, stockholm_se_nfcee_has(cut, sizeof(cut), apdu)); + KUNIT_EXPECT_FALSE(test, stockholm_se_nfcee_has(cut, 2, hci)); + KUNIT_EXPECT_FALSE(test, stockholm_se_nfcee_has(j700, sizeof(j700), apdu)); + KUNIT_EXPECT_TRUE(test, stockholm_se_nfcee_has(j700, sizeof(j700), hci)); +} + static struct kunit_case stockholm_test_cases[] = { KUNIT_CASE(stockholm_test_reset), KUNIT_CASE(stockholm_test_reset_errors), KUNIT_CASE(stockholm_test_init_macs), KUNIT_CASE(stockholm_test_init_rf_list), KUNIT_CASE(stockholm_test_init_errors), + KUNIT_CASE(stockholm_test_se_table), + KUNIT_CASE(stockholm_test_se_refused), + KUNIT_CASE(stockholm_test_se_variants), + KUNIT_CASE(stockholm_test_se_nfcee), {} }; diff --git a/drivers/nfc/apple-stockholm.c b/drivers/nfc/apple-stockholm.c index d4810655bfc81..58e46c7343b9e 100644 --- a/drivers/nfc/apple-stockholm.c +++ b/drivers/nfc/apple-stockholm.c @@ -10,19 +10,31 @@ * present, reads the controller identity once and then holds the controller * in its deepest power state (VEN low) for the life of the system. * + * With the se_probe module parameter set, debugfs also offers a read-only + * look at the Secure Element: NFCEE discovery and a fixed table of SELECT + * and GET DATA commands. There is no path for any other command; see + * stockholm_se_apdu_allowed(). + * * Copyright The Aurora Silicon Contributors */ +#include #include +#include #include #include +#include #include #include #include #include #include +#include +#include +#include #include #include +#include #include @@ -39,6 +51,24 @@ #define STOCKHOLM_INIT_RSP_TAIL 6 #define STOCKHOLM_INIT_RSP_MANUF_LEN 5 +/* + * While a Secure Element probe runs, the receive handler also queues a copy + * of every packet (notifications and data as well as responses) here. + */ +#define STOCKHOLM_TAP_SLOTS 16 + +struct stockholm_tap_pkt { + u16 len; + u8 data[NCI_CTRL_HDR_SIZE + NCI_MAX_PAYLOAD_SIZE]; +}; + +struct stockholm_tap { + unsigned int head; + unsigned int count; + unsigned int dropped; + struct stockholm_tap_pkt pkt[STOCKHOLM_TAP_SLOTS]; +}; + /* Transport parameters of one controller model. */ struct stockholm_model { u16 tx_reg; /* non-incrementing NCI transmit FIFO */ @@ -79,6 +109,13 @@ struct stockholm { u8 rx_len; u8 rx_payload[NCI_MAX_PAYLOAD_SIZE]; u8 pkt[NCI_CTRL_HDR_SIZE + NCI_MAX_PAYLOAD_SIZE]; + + /* Read-only Secure Element probe (se_probe=1 only). */ + struct mutex se_lock; /* one probe run at a time */ + spinlock_t tap_lock; /* protects tap */ + wait_queue_head_t tap_wq; + struct stockholm_tap *tap; /* packet copies while a run is active */ + struct dentry *debugfs; }; static int stockholm_fifo_read(struct stockholm *st, u8 *buf, size_t len) @@ -153,6 +190,27 @@ static int stockholm_park(struct stockholm *st) return gpiod_direction_output(st->enable_gpio, 0); } +static void stockholm_tap_push(struct stockholm *st, const u8 *pkt, u16 len) +{ + struct stockholm_tap *tap; + + spin_lock(&st->tap_lock); + tap = st->tap; + if (tap && tap->count < STOCKHOLM_TAP_SLOTS) { + struct stockholm_tap_pkt *slot; + + slot = &tap->pkt[(tap->head + tap->count) % STOCKHOLM_TAP_SLOTS]; + slot->len = len; + memcpy(slot->data, pkt, len); + tap->count++; + } else if (tap) { + tap->dropped++; + } + spin_unlock(&st->tap_lock); + if (tap) + wake_up(&st->tap_wq); +} + static irqreturn_t stockholm_data_irq(int irq, void *data) { struct stockholm *st = data; @@ -173,6 +231,8 @@ static irqreturn_t stockholm_data_irq(int irq, void *data) return IRQ_HANDLED; } + stockholm_tap_push(st, hdr, NCI_CTRL_HDR_SIZE + plen); + /* Ignore the packet boundary flag; match MT, GID and OID. */ if (st->rx_waiting && (hdr[0] & 0xef) == st->rx_hdr[0] && (hdr[1] & 0x3f) == st->rx_hdr[1]) { @@ -227,7 +287,7 @@ static irqreturn_t stockholm_error_irq(int irq, void *data) static int stockholm_nci_cmd(struct stockholm *st, u16 opcode, const u8 *payload, u8 plen) { - u8 cmd[NCI_CTRL_HDR_SIZE + 2] = {}; + u8 cmd[NCI_CTRL_HDR_SIZE + 8] = {}; unsigned long left; int ret; @@ -314,24 +374,33 @@ static int stockholm_parse_init_rsp(const u8 *rsp, int len, return 0; } -/* CORE_RESET and CORE_INIT, then log what the controller reports. */ -static int stockholm_identify(struct stockholm *st) +/* CORE_RESET and CORE_INIT. */ +static int stockholm_read_ident(struct stockholm *st, + struct stockholm_ident *id) { u8 reset_type = NCI_RESET_TYPE_RESET_CONFIG; - struct stockholm_ident id = {}; int len, ret; len = stockholm_nci_cmd(st, NCI_OP_CORE_RESET_CMD, &reset_type, 1); if (len < 0) return len; - ret = stockholm_parse_reset_rsp(st->rx_payload, len, &id); + ret = stockholm_parse_reset_rsp(st->rx_payload, len, id); if (ret) return ret; len = stockholm_nci_cmd(st, NCI_OP_CORE_INIT_CMD, NULL, 0); if (len < 0) return len; - ret = stockholm_parse_init_rsp(st->rx_payload, len, &id); + return stockholm_parse_init_rsp(st->rx_payload, len, id); +} + +/* CORE_RESET and CORE_INIT, then log what the controller reports. */ +static int stockholm_identify(struct stockholm *st) +{ + struct stockholm_ident id = {}; + int ret; + + ret = stockholm_read_ident(st, &id); if (ret) return ret; @@ -386,6 +455,781 @@ static int stockholm_request_irq(struct stockholm *st, const char *name, return irq; } +/* + * Read-only Secure Element probe + * + * The Secure Element sits behind the controller as an NFC Execution + * Environment (NFCEE). With se_probe=1 the driver offers two debugfs files + * that power the controller up, look, and park it again: + * + * nfcee NFCEE discovery only: which execution environments the + * controller reports and which protocols each one speaks. + * se_probe the same, then, if one of them takes ISO 7816 APDUs directly, + * a logical connection to it and the fixed command table below. + * + * Only SELECT by name, GET DATA of three public tags and GET RESPONSE ever + * reach the Secure Element. Commands come from the compiled-in table, never + * from userspace, and stockholm_se_apdu_allowed() checks every command again + * right before it is sent. Nothing here authenticates, installs, deletes, + * stores, sets keys or changes life-cycle state: failed GlobalPlatform + * authentication attempts can lock a card for good. + * + * An NFCEE that the probe enables is disabled again afterwards, so the + * controller is left in the mode it was found in. + */ +static bool se_probe; +module_param(se_probe, bool, 0444); +MODULE_PARM_DESC(se_probe, + "Offer a debugfs read-only Secure Element probe (default: off)"); + +#define STOCKHOLM_SE_NFCEE_MAX 8 +#define STOCKHOLM_SE_RX_MAX 1024 +#define STOCKHOLM_SE_REPORT_SIZE (32 * 1024) +#define STOCKHOLM_SE_DISCOVER_MS 1500 +#define STOCKHOLM_SE_APDU_MS 3000 +#define STOCKHOLM_SE_CREDIT_MS 1000 +#define STOCKHOLM_SE_MAX_GET_RESPONSE 4 + +/* NCI 1.x NFCEE status in NFCEE_DISCOVER_NTF. */ +#define STOCKHOLM_NFCEE_ENABLED 0x00 +#define STOCKHOLM_NFCEE_DISABLED 0x01 + +/* NCI 1.x: an initial credit count of 0xff turns data flow control off. */ +#define STOCKHOLM_NCI_NO_FLOW_CONTROL 0xff + +struct stockholm_se_apdu { + const char *name; + u8 len; + u8 apdu[24]; +}; + +#define SE_SELECT(_name, ...) \ + { .name = _name, \ + .len = 6 + sizeof((u8[]){ __VA_ARGS__ }), \ + .apdu = { 0x00, 0xa4, 0x04, 0x00, \ + sizeof((u8[]){ __VA_ARGS__ }), __VA_ARGS__, 0x00 } } + +#define SE_GET_DATA(_name, _p1, _p2) \ + { .name = _name, .len = 5, .apdu = { 0x80, 0xca, _p1, _p2, 0x00 } } + +/* + * The complete list of commands the probe sends, in order. GET DATA goes to + * the application selected last, so it follows the card manager SELECT. + */ +static const struct stockholm_se_apdu stockholm_se_apdus[] = { + /* Card manager (Issuer Security Domain): SELECT with no name. */ + { .name = "isd-default", .len = 5, + .apdu = { 0x00, 0xa4, 0x04, 0x00, 0x00 } }, + SE_GET_DATA("cplc", 0x9f, 0x7f), + SE_GET_DATA("card-data", 0x00, 0x66), + SE_GET_DATA("key-info", 0x00, 0xe0), + SE_SELECT("isd-gp", 0xa0, 0x00, 0x00, 0x01, 0x51, 0x00, 0x00, 0x00), + SE_SELECT("isd-op", 0xa0, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00), + SE_SELECT("gp-crs", 0xa0, 0x00, 0x00, 0x01, 0x51, 0x43, 0x52, 0x53, + 0x00), + /* "2PAY.SYS.DDF01" and "1PAY.SYS.DDF01" */ + SE_SELECT("ppse", 0x32, 0x50, 0x41, 0x59, 0x2e, 0x53, 0x59, 0x53, + 0x2e, 0x44, 0x44, 0x46, 0x30, 0x31), + SE_SELECT("pse", 0x31, 0x50, 0x41, 0x59, 0x2e, 0x53, 0x59, 0x53, + 0x2e, 0x44, 0x44, 0x46, 0x30, 0x31), + SE_SELECT("visa", 0xa0, 0x00, 0x00, 0x00, 0x03, 0x10, 0x10), + SE_SELECT("mastercard", 0xa0, 0x00, 0x00, 0x00, 0x04, 0x10, 0x10), + SE_SELECT("maestro", 0xa0, 0x00, 0x00, 0x00, 0x04, 0x30, 0x60), + SE_SELECT("amex", 0xa0, 0x00, 0x00, 0x00, 0x25, 0x01), + SE_SELECT("discover", 0xa0, 0x00, 0x00, 0x01, 0x52, 0x30, 0x10), + SE_SELECT("jcb", 0xa0, 0x00, 0x00, 0x00, 0x65, 0x10, 0x10), + SE_SELECT("unionpay", 0xa0, 0x00, 0x00, 0x03, 0x33, 0x01, 0x01), + SE_SELECT("interac", 0xa0, 0x00, 0x00, 0x02, 0x77, 0x10, 0x10), + SE_SELECT("fido-u2f", 0xa0, 0x00, 0x00, 0x06, 0x47, 0x2f, 0x00, 0x01), + SE_SELECT("piv", 0xa0, 0x00, 0x00, 0x03, 0x08, 0x00, 0x00, 0x10, + 0x00, 0x01, 0x00), + SE_SELECT("openpgp", 0xd2, 0x76, 0x00, 0x01, 0x24, 0x01), + SE_SELECT("ndef", 0xd2, 0x76, 0x00, 0x00, 0x85, 0x01, 0x01), + /* Car Connectivity Consortium digital key framework, "CCCDKFv1" */ + SE_SELECT("ccc-digital-key", 0xa0, 0x00, 0x00, 0x08, 0x09, 0x43, 0x43, + 0x43, 0x44, 0x4b, 0x46, 0x76, 0x31), +}; + +/* + * The allowlist. Every command is checked here right before it reaches the + * Secure Element, including the GET RESPONSE and corrected-length retries + * the probe builds itself: + * + * 00 A4 04 00 [Lc name 00] SELECT by DF name, first occurrence + * 80 CA 9F 7F 00 GET DATA: Card Production Life Cycle + * 80 CA 00 66 00 GET DATA: card data + * 80 CA 00 E0 00 GET DATA: key information template + * 00 C0 00 00 Le GET RESPONSE + * + * Only the basic logical channel, no secure messaging, no chaining. + */ +static bool stockholm_se_apdu_allowed(const u8 *apdu, size_t len) +{ + u16 tag; + + if (len < 5) + return false; + + switch (apdu[1]) { + case 0xa4: /* SELECT */ + if (apdu[0] != 0x00 || apdu[2] != 0x04 || apdu[3] != 0x00) + return false; + if (len == 5) + return true; + return apdu[4] >= 5 && apdu[4] <= 16 && + len == 6 + (size_t)apdu[4]; + case 0xca: /* GET DATA */ + if (apdu[0] != 0x80 || len != 5) + return false; + tag = get_unaligned_be16(&apdu[2]); + return tag == 0x9f7f || tag == 0x0066 || tag == 0x00e0; + case 0xc0: /* GET RESPONSE */ + return apdu[0] == 0x00 && apdu[2] == 0x00 && apdu[3] == 0x00 && + len == 5; + default: + return false; + } +} + +struct stockholm_se_run { + struct stockholm *st; + char *text; + size_t len; + + struct stockholm_tap_pkt pkt; + + int nfcee_count; + u8 nfcee_ntf_len[STOCKHOLM_SE_NFCEE_MAX]; + u8 nfcee_ntf[STOCKHOLM_SE_NFCEE_MAX][NCI_MAX_PAYLOAD_SIZE]; + + u8 conn_id; + bool conn_open; + bool flow_control; + u8 max_payload; + unsigned int credits; + + bool rx_done; + bool rx_overflow; + size_t rx_len; + u8 rx[STOCKHOLM_SE_RX_MAX]; +}; + +static __printf(2, 3) void stockholm_se_print(struct stockholm_se_run *run, + const char *fmt, ...) +{ + va_list args; + int n; + + if (run->len >= STOCKHOLM_SE_REPORT_SIZE - 1) + return; + va_start(args, fmt); + n = vscnprintf(run->text + run->len, STOCKHOLM_SE_REPORT_SIZE - run->len, + fmt, args); + va_end(args); + run->len += n; +} + +static void stockholm_se_print_hex(struct stockholm_se_run *run, + const u8 *buf, size_t len) +{ + size_t chunk; + + while (len) { + chunk = min_t(size_t, len, 64); + stockholm_se_print(run, "%*phN", (int)chunk, buf); + buf += chunk; + len -= chunk; + } +} + +static bool stockholm_tap_pending(struct stockholm *st) +{ + bool pending; + + spin_lock(&st->tap_lock); + pending = st->tap && st->tap->count; + spin_unlock(&st->tap_lock); + + return pending; +} + +/* Take the oldest queued packet, waiting until the deadline for one. */ +static int stockholm_tap_pop(struct stockholm *st, struct stockholm_tap_pkt *out, + unsigned long deadline) +{ + struct stockholm_tap *tap; + long left; + + for (;;) { + spin_lock(&st->tap_lock); + tap = st->tap; + if (tap->count) { + *out = tap->pkt[tap->head]; + tap->head = (tap->head + 1) % STOCKHOLM_TAP_SLOTS; + tap->count--; + spin_unlock(&st->tap_lock); + return 0; + } + spin_unlock(&st->tap_lock); + + left = (long)(deadline - jiffies); + if (left <= 0) + return -ETIMEDOUT; + wait_event_timeout(st->tap_wq, stockholm_tap_pending(st), left); + } +} + +/* Account for one received packet: notifications and connection data. */ +static void stockholm_se_handle(struct stockholm_se_run *run) +{ + const u8 *pkt = run->pkt.data; + u8 plen = run->pkt.len - NCI_CTRL_HDR_SIZE; + const u8 *payload = pkt + NCI_CTRL_HDR_SIZE; + u16 op; + int i; + + switch (nci_mt(pkt)) { + case NCI_MT_DATA_PKT: + if (!run->conn_open || nci_conn_id(pkt) != run->conn_id) { + stockholm_se_print(run, "drop data conn=%u len=%u\n", + nci_conn_id(pkt), plen); + return; + } + if (run->rx_len + plen > sizeof(run->rx)) { + run->rx_overflow = true; + plen = sizeof(run->rx) - run->rx_len; + } + memcpy(run->rx + run->rx_len, payload, plen); + run->rx_len += plen; + if (!nci_pbf(pkt)) + run->rx_done = true; + return; + + case NCI_MT_NTF_PKT: + op = nci_opcode_pack(pkt[0] & 0x0f, pkt[1] & 0x3f); + break; + + default: + /* Command responses are handled by stockholm_nci_cmd(). */ + return; + } + + if (op == NCI_OP_CORE_CONN_CREDITS_NTF) { + /* Entry count, then connection ID and credits per entry. */ + for (i = 0; plen && i < payload[0] && 2 + 2 * i < plen; i++) + if (payload[1 + 2 * i] == run->conn_id) + run->credits += payload[2 + 2 * i]; + return; + } + + if (op == NCI_OP_NFCEE_DISCOVER_NTF && + run->nfcee_count < STOCKHOLM_SE_NFCEE_MAX) { + memcpy(run->nfcee_ntf[run->nfcee_count], payload, plen); + run->nfcee_ntf_len[run->nfcee_count] = plen; + run->nfcee_count++; + } + + stockholm_se_print(run, "ntf gid=%u oid=%02x data=", pkt[0] & 0x0f, + pkt[1] & 0x3f); + stockholm_se_print_hex(run, payload, plen); + stockholm_se_print(run, "\n"); +} + +enum stockholm_se_wait { + SE_WAIT_NFCEE, + SE_WAIT_DATA, + SE_WAIT_CREDIT, +}; + +static bool stockholm_se_done(struct stockholm_se_run *run, + enum stockholm_se_wait what, int nfcee_want) +{ + switch (what) { + case SE_WAIT_NFCEE: + return run->nfcee_count >= nfcee_want; + case SE_WAIT_DATA: + return run->rx_done; + case SE_WAIT_CREDIT: + return !run->flow_control || run->credits > 0; + } + return true; +} + +/* Process queued packets until the condition holds or the time is up. */ +static int stockholm_se_wait(struct stockholm_se_run *run, + enum stockholm_se_wait what, int nfcee_want, + unsigned int ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(ms); + int ret; + + while (!stockholm_se_done(run, what, nfcee_want)) { + ret = stockholm_tap_pop(run->st, &run->pkt, deadline); + if (ret) + return ret; + stockholm_se_handle(run); + } + + return 0; +} + +/* Drain whatever the controller sent before the next step. */ +static void stockholm_se_drain(struct stockholm_se_run *run) +{ + while (!stockholm_tap_pop(run->st, &run->pkt, jiffies)) + stockholm_se_handle(run); +} + +/* One NCI command whose response starts with a status byte. */ +static int stockholm_se_cmd(struct stockholm_se_run *run, const char *step, + u16 opcode, const u8 *payload, u8 plen) +{ + struct stockholm *st = run->st; + int len; + + len = stockholm_nci_cmd(st, opcode, payload, plen); + stockholm_se_print(run, "cmd %s ", step); + if (len < 0) { + stockholm_se_print(run, "error=%d\n", len); + return len; + } + stockholm_se_print(run, "rsp="); + stockholm_se_print_hex(run, st->rx_payload, len); + stockholm_se_print(run, "\n"); + if (len < 1) + return -EIO; + if (st->rx_payload[0] != NCI_STATUS_OK) + return -EREMOTEIO; + + return len; +} + +/* Send one command APDU on the open connection and collect the reply. */ +static int stockholm_se_transceive(struct stockholm_se_run *run, + const u8 *apdu, size_t len) +{ + struct stockholm *st = run->st; + u8 pkt[NCI_DATA_HDR_SIZE + sizeof(stockholm_se_apdus[0].apdu)]; + int ret; + + /* The allowlist gate: nothing else ever reaches the Secure Element. */ + if (!stockholm_se_apdu_allowed(apdu, len) || len > sizeof(pkt) - 3) { + dev_err(&st->sdev->dev, "refusing APDU %*ph\n", + (int)min_t(size_t, len, 8), apdu); + return -EPERM; + } + if (len > run->max_payload) + return -EMSGSIZE; + + ret = stockholm_se_wait(run, SE_WAIT_CREDIT, 0, STOCKHOLM_SE_CREDIT_MS); + if (ret) + return ret; + if (run->flow_control) + run->credits--; + + pkt[0] = run->conn_id; /* data packet, last segment */ + pkt[1] = 0; + pkt[2] = len; + memcpy(pkt + NCI_DATA_HDR_SIZE, apdu, len); + + run->rx_done = false; + run->rx_overflow = false; + run->rx_len = 0; + + mutex_lock(&st->io_lock); + ret = stockholm_fifo_write(st, pkt, NCI_DATA_HDR_SIZE + len); + mutex_unlock(&st->io_lock); + if (ret) + return ret; + + ret = stockholm_se_wait(run, SE_WAIT_DATA, 0, STOCKHOLM_SE_APDU_MS); + if (ret) + return ret; + if (run->rx_overflow || run->rx_len < 2) + return -EPROTO; + + return 0; +} + +/* + * One table entry: send it, follow 61xx with GET RESPONSE and 6Cxx with the + * same command and the corrected length, and print the whole response. + */ +static int stockholm_se_exchange(struct stockholm_se_run *run, + const struct stockholm_se_apdu *cmd) +{ + u8 apdu[sizeof(cmd->apdu)]; + u8 *resp; + size_t resp_len = 0; + u8 sw1 = 0, sw2 = 0; + int ret, i; + + resp = kmalloc(STOCKHOLM_SE_RX_MAX, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + memcpy(apdu, cmd->apdu, cmd->len); + ret = stockholm_se_transceive(run, apdu, cmd->len); + if (!ret && run->rx[run->rx_len - 2] == 0x6c) { + /* Wrong Le: repeat the same command with the length given. */ + apdu[cmd->len - 1] = run->rx[run->rx_len - 1]; + ret = stockholm_se_transceive(run, apdu, cmd->len); + } + + for (i = 0; !ret; i++) { + size_t data = run->rx_len - 2; + + sw1 = run->rx[run->rx_len - 2]; + sw2 = run->rx[run->rx_len - 1]; + data = min_t(size_t, data, STOCKHOLM_SE_RX_MAX - resp_len); + memcpy(resp + resp_len, run->rx, data); + resp_len += data; + + if (sw1 != 0x61 || i >= STOCKHOLM_SE_MAX_GET_RESPONSE) + break; + + /* More data waiting: GET RESPONSE. */ + apdu[0] = 0x00; + apdu[1] = 0xc0; + apdu[2] = 0x00; + apdu[3] = 0x00; + apdu[4] = sw2; + ret = stockholm_se_transceive(run, apdu, 5); + } + + stockholm_se_print(run, "apdu name=%s cmd=", cmd->name); + stockholm_se_print_hex(run, cmd->apdu, cmd->len); + if (ret) { + stockholm_se_print(run, " error=%d\n", ret); + } else { + stockholm_se_print(run, " sw=%02x%02x resp=", sw1, sw2); + stockholm_se_print_hex(run, resp, resp_len); + stockholm_se_print(run, "\n"); + } + kfree(resp); + + return ret; +} + +/* NCI 1.x NFCEE_DISCOVER_NTF: ID, status, protocol count and list, TLVs. */ +static bool stockholm_se_nfcee_has(const u8 *ntf, u8 len, u8 protocol) +{ + int i; + + if (len < 3) + return false; + for (i = 0; i < ntf[2] && 3 + i < len; i++) + if (ntf[3 + i] == protocol) + return true; + + return false; +} + +static int stockholm_se_discover(struct stockholm_se_run *run) +{ + struct stockholm *st = run->st; + u8 action = NCI_NFCEE_DISCOVERY_ACTION_ENABLE; + int ret, want, wait; + + ret = stockholm_se_cmd(run, "nfcee-discover", + NCI_OP_NFCEE_DISCOVER_CMD, &action, 1); + if (ret < 0) + return ret; + want = ret >= 2 ? st->rx_payload[1] : 0; + + wait = stockholm_se_wait(run, SE_WAIT_NFCEE, want, + STOCKHOLM_SE_DISCOVER_MS); + stockholm_se_print(run, "nfcee expected=%d reported=%d%s\n", want, + run->nfcee_count, wait ? " (timed out)" : ""); + + action = NCI_NFCEE_DISCOVERY_ACTION_DISABLE; + ret = stockholm_se_cmd(run, "nfcee-discover-stop", + NCI_OP_NFCEE_DISCOVER_CMD, &action, 1); + + return ret < 0 ? ret : 0; +} + +static int stockholm_se_apdus_run(struct stockholm_se_run *run) +{ + struct stockholm *st = run->st; + u8 mode[2], conn[6], close; + bool enabled = false; + const u8 *ntf = NULL; + unsigned int i; + int ret, err; + u8 id; + + for (i = 0; i < run->nfcee_count; i++) { + if (stockholm_se_nfcee_has(run->nfcee_ntf[i], + run->nfcee_ntf_len[i], + NCI_NFCEE_INTERFACE_APDU)) { + ntf = run->nfcee_ntf[i]; + break; + } + } + if (!ntf) { + stockholm_se_print(run, "apdu-nfcee none\n"); + return 0; + } + + id = ntf[0]; + stockholm_se_print(run, "apdu-nfcee id=%02x status=%02x\n", id, ntf[1]); + if (ntf[1] != STOCKHOLM_NFCEE_ENABLED && + ntf[1] != STOCKHOLM_NFCEE_DISABLED) + return -ENODEV; + + if (ntf[1] == STOCKHOLM_NFCEE_DISABLED) { + mode[0] = id; + mode[1] = NCI_NFCEE_ENABLE; + ret = stockholm_se_cmd(run, "nfcee-enable", + NCI_OP_NFCEE_MODE_SET_CMD, mode, 2); + if (ret < 0) + return ret; + enabled = true; + stockholm_se_drain(run); + } + + conn[0] = NCI_DESTINATION_NFCEE; + conn[1] = 1; + conn[2] = NCI_DESTINATION_SPECIFIC_PARAM_NFCEE_TYPE; + conn[3] = 2; + conn[4] = id; + conn[5] = NCI_NFCEE_INTERFACE_APDU; + ret = stockholm_se_cmd(run, "conn-create", NCI_OP_CORE_CONN_CREATE_CMD, + conn, sizeof(conn)); + if (ret >= 4) { + /* status, max data payload, initial credits, connection ID */ + run->max_payload = st->rx_payload[1]; + run->flow_control = st->rx_payload[2] != STOCKHOLM_NCI_NO_FLOW_CONTROL; + run->credits = run->flow_control ? st->rx_payload[2] : 0; + run->conn_id = st->rx_payload[3] & 0x0f; + run->conn_open = true; + stockholm_se_print(run, + "conn id=%u max-payload=%u credits=%u%s\n", + run->conn_id, run->max_payload, run->credits, + run->flow_control ? "" : " (no flow control)"); + + ret = 0; + for (i = 0; i < ARRAY_SIZE(stockholm_se_apdus); i++) { + err = stockholm_se_exchange(run, &stockholm_se_apdus[i]); + /* A dead link fails every later command the same way. */ + if (err == -ETIMEDOUT || err == -EPERM) { + ret = err; + break; + } + } + + close = run->conn_id; + err = stockholm_se_cmd(run, "conn-close", + NCI_OP_CORE_CONN_CLOSE_CMD, &close, 1); + run->conn_open = false; + if (err < 0 && !ret) + ret = err; + stockholm_se_drain(run); + } else if (ret >= 0) { + ret = -EPROTO; + } + + if (enabled) { + mode[0] = id; + mode[1] = NCI_NFCEE_DISABLE; + err = stockholm_se_cmd(run, "nfcee-restore-disabled", + NCI_OP_NFCEE_MODE_SET_CMD, mode, 2); + if (err < 0 && !ret) + ret = err; + stockholm_se_drain(run); + } + + return ret; +} + +/* Power up, identify, look, park. Returns the text report. */ +static void stockholm_se_run(struct stockholm_se_run *run, bool apdus) +{ + struct stockholm *st = run->st; + struct stockholm_ident id = {}; + struct stockholm_tap *tap; + int ret, err; + + tap = kzalloc_obj(*tap); + if (!tap) { + stockholm_se_print(run, "result error=%d\n", -ENOMEM); + return; + } + + stockholm_se_print(run, "stockholm-se-probe 1 mode=%s\n", + apdus ? "apdu" : "nfcee"); + + spin_lock(&st->tap_lock); + st->tap = tap; + spin_unlock(&st->tap_lock); + + ret = stockholm_power_on(st); + if (!ret) + ret = stockholm_check_follower(st); + if (ret) + goto park; + + enable_irq(st->error_irq); + enable_irq(st->data_irq); + + ret = stockholm_read_ident(st, &id); + if (!ret) { + stockholm_se_print(run, + "ident nci=%u.%u manufacturer=%02x firmware=%08x rf-interfaces=%u\n", + id.nci_version >> 4, id.nci_version & 0xf, + id.manufacturer, id.firmware, + id.rf_interfaces); + stockholm_se_drain(run); + ret = stockholm_se_discover(run); + } + if (!ret && apdus) + ret = stockholm_se_apdus_run(run); + + stockholm_se_drain(run); + disable_irq(st->data_irq); + disable_irq(st->error_irq); + +park: + err = stockholm_park(st); + if (err && !ret) + ret = err; + + spin_lock(&st->tap_lock); + if (tap->dropped) + stockholm_se_print(run, "tap dropped=%u\n", tap->dropped); + st->tap = NULL; + spin_unlock(&st->tap_lock); + kfree(tap); + + stockholm_se_print(run, "result %s=%d parked=%s\n", + ret ? "error" : "ok", ret, err ? "no" : "yes"); +} + +struct stockholm_se_report { + size_t len; + char text[]; +}; + +static int stockholm_se_open_common(struct inode *inode, struct file *file, + bool apdus) +{ + struct stockholm *st = inode->i_private; + struct stockholm_se_report *report; + struct stockholm_se_run *run; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + run = kzalloc_obj(*run); + report = kvzalloc(struct_size(report, text, STOCKHOLM_SE_REPORT_SIZE), + GFP_KERNEL); + if (!run || !report) { + kfree(run); + kvfree(report); + return -ENOMEM; + } + run->st = st; + run->text = report->text; + + mutex_lock(&st->se_lock); + stockholm_se_run(run, apdus); + mutex_unlock(&st->se_lock); + + report->len = run->len; + kfree_sensitive(run); + file->private_data = report; + + return nonseekable_open(inode, file); +} + +static int stockholm_se_nfcee_open(struct inode *inode, struct file *file) +{ + return stockholm_se_open_common(inode, file, false); +} + +static int stockholm_se_probe_open(struct inode *inode, struct file *file) +{ + return stockholm_se_open_common(inode, file, true); +} + +static ssize_t stockholm_se_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct stockholm_se_report *report = file->private_data; + + return simple_read_from_buffer(buf, count, ppos, report->text, + report->len); +} + +static int stockholm_se_release(struct inode *inode, struct file *file) +{ + struct stockholm_se_report *report = file->private_data; + + /* The report holds card production data: wipe it. */ + kvfree_sensitive(report, + struct_size(report, text, STOCKHOLM_SE_REPORT_SIZE)); + return 0; +} + +static const struct file_operations stockholm_se_nfcee_fops = { + .owner = THIS_MODULE, + .open = stockholm_se_nfcee_open, + .read = stockholm_se_read, + .release = stockholm_se_release, +}; + +static const struct file_operations stockholm_se_probe_fops = { + .owner = THIS_MODULE, + .open = stockholm_se_probe_open, + .read = stockholm_se_read, + .release = stockholm_se_release, +}; + +/* The command table, one APDU per line, without touching the hardware. */ +static int stockholm_se_allowlist_show(struct seq_file *s, void *unused) +{ + const struct stockholm_se_apdu *cmd; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(stockholm_se_apdus); i++) { + cmd = &stockholm_se_apdus[i]; + seq_printf(s, "%s %*phN\n", cmd->name, cmd->len, cmd->apdu); + } + seq_puts(s, "get-response 00c00000xx\n"); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(stockholm_se_allowlist); + +static struct dentry *stockholm_debugfs_root; + +static void stockholm_se_debugfs_remove(void *data) +{ + struct stockholm *st = data; + + debugfs_remove_recursive(st->debugfs); +} + +static int stockholm_se_debugfs_init(struct stockholm *st) +{ + struct device *dev = &st->sdev->dev; + + if (!se_probe || IS_ERR_OR_NULL(stockholm_debugfs_root)) + return 0; + + st->debugfs = debugfs_create_dir(dev_name(dev), stockholm_debugfs_root); + debugfs_create_file("nfcee", 0400, st->debugfs, st, + &stockholm_se_nfcee_fops); + debugfs_create_file("se_probe", 0400, st->debugfs, st, + &stockholm_se_probe_fops); + debugfs_create_file("apdu_allowlist", 0400, st->debugfs, NULL, + &stockholm_se_allowlist_fops); + dev_info(dev, "read-only Secure Element probe enabled in debugfs\n"); + + return devm_add_action_or_reset(dev, stockholm_se_debugfs_remove, st); +} + static int stockholm_probe(struct spmi_device *sdev) { struct device *dev = &sdev->dev; @@ -402,6 +1246,9 @@ static int stockholm_probe(struct spmi_device *sdev) return -ENODEV; mutex_init(&st->io_lock); init_completion(&st->rx_done); + mutex_init(&st->se_lock); + spin_lock_init(&st->tap_lock); + init_waitqueue_head(&st->tap_wq); st->enable_gpio = devm_gpiod_get(dev, "enable", GPIOD_ASIS); if (IS_ERR(st->enable_gpio)) @@ -436,8 +1283,10 @@ static int stockholm_probe(struct spmi_device *sdev) err = stockholm_park(st); if (err) return dev_err_probe(dev, err, "failed to park VEN low\n"); + if (ret) + return ret; - return ret; + return stockholm_se_debugfs_init(st); } static const struct of_device_id stockholm_of_match[] = { @@ -453,7 +1302,28 @@ static struct spmi_driver stockholm_driver = { .of_match_table = stockholm_of_match, }, }; -module_spmi_driver(stockholm_driver); + +static int __init stockholm_init(void) +{ + int ret; + + if (se_probe) + stockholm_debugfs_root = debugfs_create_dir("apple-stockholm", + NULL); + ret = spmi_driver_register(&stockholm_driver); + if (ret) + debugfs_remove(stockholm_debugfs_root); + + return ret; +} +module_init(stockholm_init); + +static void __exit stockholm_exit(void) +{ + spmi_driver_unregister(&stockholm_driver); + debugfs_remove(stockholm_debugfs_root); +} +module_exit(stockholm_exit); #if IS_ENABLED(CONFIG_NFC_APPLE_STOCKHOLM_KUNIT_TEST) #include "apple-stockholm-test.c" From 8a515bbbcd47335012f0b6349e78c1bc00dea5e3 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Sat, 7 Nov 2026 19:58:16 -0300 Subject: [PATCH 059/392] arm64: ctxr-execmem: keep an M4-generation boot CPU at EL2 from its first entry On a J616s (T6040) booted from a kcbz 0 boot object, Linux died before earlycon=efifb printed a line, through U-Boot's booti and through GRUB alike, with the image in the CTXR window. aura 1.0.0b7, U-Boot e1ef8b63 and GRUB all ran. The only kernel code before earlycon that is new under a window is the boot CPU's trip through EL1, which 46c5ae7cd01 judged safe because CTXR_D_CTL_EL1 reads 0 on the T8140's boot CPU. The T6040's boot CPU, cpu@10100, also reads 0 there, so that value does not tell a confined core from a free one. Take the direct VHE entry on the boot CPU's first entry again when it is an Apple core with a part number from 0x50 that implements VHE, as 73b99c17e605 and a57207316f8b did, without reading ctxr_stay_el2. Secondaries and resume keep reading the flag. This skips the boot CPU's EL1 phase on raw boots of those chips too, and with it the VHE overrides (kvm-arm.mode=nvhe/protected, id_aa64mmfr1.vh=0), which 46c5ae7cd01 kept: this early the boot CPU can't tell whether it is in a window. Tested on a J616s with aura 1.0.0b7's Mach-O Stage 1 and U-Boot booti: the window at 0x10006c00000 (128 MiB), 14 CPUs started at EL2, boot verify 968 executable mappings and 0 violations (ctxr_exec_enforce=0), and the Apple TSO cpucap detected. The same kernel without this commit died before earlycon on that chain. On the raw Stage 1 it also brings up all 14 CPUs at EL2. Fixes: 46c5ae7cd01b ("arm64: ctxr-execmem: keep the boot CPU on the usual entry path") Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- Documentation/arch/arm64/ctxr-execmem.rst | 15 +++++++---- arch/arm64/kernel/head.S | 31 +++++++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/Documentation/arch/arm64/ctxr-execmem.rst b/Documentation/arch/arm64/ctxr-execmem.rst index 28f580fc1de2e..4eb104ef0c2c4 100644 --- a/Documentation/arch/arm64/ctxr-execmem.rst +++ b/Documentation/arch/arm64/ctxr-execmem.rst @@ -80,11 +80,16 @@ How it works * If the window's pool is exhausted the allocation fails, exactly as a hardware CTXR fault would prevent executing from outside the window; there is no buddy fallback. -* Code in the window can't execute at EL1 on a confined core. The boot CPU isn't - confined and enters as it always has, through EL1 and ``finalise_el2``. Once it - has reserved a window at EL2, it sets a flag that later entries read with the - MMU off, and those switch to VHE in ``init_kernel_el`` without running at EL1. - With no window, every core enters as it always has. +* Code in the window can't execute at EL1 on a confined core. On an Apple core + with a part number from 0x50 that implements VHE, the boot CPU's first entry + switches to VHE in ``init_kernel_el`` without running at EL1, with or without a + window: the J616s (T6040) boot CPU doesn't survive the trip through EL1 inside a + window, and the kernel can't tell this early whether it is in one. Such a boot + CPU stays in VHE whatever ``kvm-arm.mode`` and ``id_aa64mmfr1.vh`` say. Any + other boot CPU enters as it always has, through EL1 and ``finalise_el2``. Once + the boot CPU has reserved a window at EL2, it sets a flag that later entries + read with the MMU off, and those switch to VHE the same way. With no window, + later entries go as they always have. Userspace ========= diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 53e1c9af9fd22..a596a8d347183 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -314,16 +314,37 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) /* * A core confined to a CTXR window can't run the window's code at EL1, * so once a window is active it switches to VHE here instead of going - * through EL1 and finalise_el2. Only a secondary or a resuming CPU is - * ever confined (x22 == 1): the boot CPU's first entry (x22 == 0) is - * not, and it runs before ctxr_stay_el2 is set and cleaned to the PoC, - * so it must not read the flag at all. + * through EL1 and finalise_el2. Only a secondary or a resuming CPU + * reads the flag (x22 == 1): the boot CPU's first entry (x22 == 0) + * runs before ctxr_stay_el2 is set and cleaned to the PoC, so it must + * not read it at all. */ - cbz x22, .Lctxr_no_direct_vhe + cbnz x22, .Lctxr_read_flag + /* + * The boot CPU of a J616s (T6040) booted from a kcbz 0 boot object + * dies before earlycon on the trip through EL1, while the T8140's + * boot CPU survives it. Keep Apple cores from the M4 generation on + * (part numbers from 0x50) with VHE at EL2 from the boot CPU's first + * entry, as 73b99c17e605 did, without reading the flag. + */ + mrs x0, midr_el1 + ubfx x1, x0, #MIDR_IMPLEMENTOR_SHIFT, #8 + cmp x1, #ARM_CPU_IMP_APPLE + b.ne .Lctxr_no_direct_vhe + ubfx x0, x0, #MIDR_PARTNUM_SHIFT, #12 + cmp x0, #0x50 + b.lo .Lctxr_no_direct_vhe + /* A virtual EL2 without VHE, as nested KVM may give a guest. */ + mrs x0, id_aa64mmfr1_el1 + ubfx x0, x0, #ID_AA64MMFR1_EL1_VH_SHIFT, #4 + cbz x0, .Lctxr_no_direct_vhe + b .Lctxr_direct_vhe +.Lctxr_read_flag: adr_l x0, ctxr_stay_el2 ldr w0, [x0] cbz w0, .Lctxr_no_direct_vhe +.Lctxr_direct_vhe: finalise_el2_state mov_q x0, HCR_HOST_VHE_FLAGS From 54485c922f72eeb4dfcad88f144117c70ef45f61 Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:06:01 +0100 Subject: [PATCH 060/392] soc: apple: t8132: describe the CIO PMGR in the platform data Every T8132 tree carries pmgr_cio (t8132-usb4.dtsi). The platform data describes only pmgr and pmgr_mini, so t8132_pmgr_policy() cannot resolve pmgr_cio and fails the whole handoff: t8132-platform-handoff: cannot resolve PMGR policy for /soc/power-management@380780000: -1 Since the policy runs on the qualified firmware of every M4 Mac, a tree that leaves its domain status to the policy then has none: on the J604 65 domains fail closed with "missing M4 power-domain policy", and the NVMe mailbox, serial, IOMMUs and PMP never probe. The J713 tree reaches the same error but states every domain itself. Describe pmgr_cio in the data. It has no power-state domains, so the policy is unchanged and an undescribed PMGR still fails closed. On the J604 the handoff then reports "supplied 0 platform nodes, 140 PMGR policy properties" and the system boots to the login screen. Fixes: ce5aebdff7ce ("arm64: dts: apple: t8132, j713: describe the USB4 host routers") Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- drivers/soc/apple/t8132-platform-data.dts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/soc/apple/t8132-platform-data.dts b/drivers/soc/apple/t8132-platform-data.dts index 52d97f73b81b3..13edfc1b94b44 100644 --- a/drivers/soc/apple/t8132-platform-data.dts +++ b/drivers/soc/apple/t8132-platform-data.dts @@ -17,6 +17,13 @@ reg = <0x3 0x80700000 0 0x14000>; }; + /* Describes no domain; resolves the CIO PMGR the SoC tree carries. */ + pmgr_cio: power-management@380780000 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x80780000 0 0x4000>; + }; + pmgr_mini: power-management@388280000 { #address-cells = <1>; #size-cells = <1>; From e0815b436112837cc46bf67fc6d3655f65107991 Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:06:02 +0100 Subject: [PATCH 061/392] arm64: dts: apple: t8132-j604: drop the boot framebuffer's power domains The boot framebuffer (t8132-jxxx.dtsi) lists ps_disp_fe and ps_dptx_phy. The M4 PMGR policy disables ps_dptx_phy, so simpledrm waits for it until the deferred probe timeout: simple-framebuffer 103e0126880.framebuffer: [drm] pm_domain_attach_by_id(1) failed: -110 and the J604 shows no console for the first ~16 s. Delete the property, as the J713 tree does (t8132-j713-j715.dtsi). Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8132-j604.dts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts index 8c86b23bf92ef..09a2730f1c442 100644 --- a/arch/arm64/boot/dts/apple/t8132-j604.dts +++ b/arch/arm64/boot/dts/apple/t8132-j604.dts @@ -80,6 +80,14 @@ apple,panel-height-mm = <196>; }; +/* + * The M4 PMGR policy disables ps_dptx_phy, which the shared boot + * framebuffer lists; as on the J713, the loader keeps the display on. + */ +&framebuffer0 { + /delete-property/ power-domains; +}; + &sound_jack { compatible = "apple,j604-macaudio-jack", "apple,macaudio"; model = "MacBook Pro J604 Jack"; From 98463e69f8f22c81c4b8904d31d77d227406f46a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:33:12 +1100 Subject: [PATCH 062/392] arm64: kexec: start the crash kernel from the boot CPU under a CTXR window A CTXR window confines every core but the boot CPU to code inside the window, so kexec was refused outright and crashkernel= was ignored. A crash kernel only needs one CPU, and the boot CPU is not confined. Reserve the crash kernel as usual (the window is already reserved, so it lands outside it) and allow loading a crash image. When the panic is handled on a confined core, that core offers the crash kernel to CPU 0, which waits for it in its stop handler instead of parking, and parks itself. If CPU 0 never acknowledged the stop or does not take the offer within a second, the offer is withdrawn and the panic restarts the machine as before. Ordinary kexec stays refused. The crash kernel finds the same aurora,ctxr-exec-window in its device tree. It runs from ordinary memory on the boot CPU, so it keeps the secondaries where they are and leaves the window alone. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/include/asm/kexec.h | 6 ++ arch/arm64/kernel/machine_kexec.c | 109 +++++++++++++++++++++++++++++- arch/arm64/kernel/smp.c | 3 + arch/arm64/mm/ctxr_execmem.c | 11 ++- arch/arm64/mm/init.c | 13 +--- 5 files changed, 128 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 892e5bebda957..3f4bac22eb7d7 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -104,6 +104,12 @@ int machine_kexec_post_load(struct kimage *image); #define machine_kexec_post_load machine_kexec_post_load #endif +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM) +void arm64_crash_handoff_wait(void); +#else +static inline void arm64_crash_handoff_wait(void) {} +#endif + #define ARCH_HAS_KIMAGE_ARCH struct kimage_arch { diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 9c4a407222ea1..b8e6b6c8aa653 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -6,11 +6,13 @@ * Copyright (C) Huawei Futurewei Technologies. */ +#include #include #include #include #include #include +#include #include #include #include @@ -56,10 +58,12 @@ void machine_kexec_cleanup(struct kimage *kimage) int machine_kexec_prepare(struct kimage *kimage) { /* - * The relocation code runs from an ordinary page, outside the window, - * and a crash dump can start on a core enforcing it. + * A new kernel runs from ordinary memory, outside the window, so the + * confined secondaries could never run it. Only a crash kernel can + * start: it runs on the boot CPU alone, which the window doesn't + * confine (see crash_handoff()). */ - if (execmem_confine_active()) { + if (execmem_confine_active() && kimage->type != KEXEC_TYPE_CRASH) { pr_err("Can't kexec: the kernel's code is confined to a CTXR window.\n"); return -EOPNOTSUPP; } @@ -165,6 +169,100 @@ int machine_kexec_post_load(struct kimage *kimage) return 0; } +#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_ARM64_CTXR_EXECMEM) +/* + * Under a CTXR window every core but the boot CPU fetches code only from the + * window, and the crash kernel sits in ordinary memory. A crash handled on a + * confined core therefore offers the jump to CPU 0, which waits for it in its + * stop handler. The state makes taking the offer and withdrawing it after a + * timeout exclusive, so exactly one of the two CPUs goes on. + */ +enum { + CRASH_HANDOFF_IDLE, + CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_TAKEN, + CRASH_HANDOFF_WITHDRAWN, +}; + +static int crash_handoff_state; +static struct kimage *crash_handoff_image; + +static bool crash_needs_handoff(void) +{ + return execmem_confine_active() && smp_processor_id() != 0; +} + +/* + * Called by every CPU told to stop, with interrupts masked. On CPU 0, with a + * window and a crash kernel loaded, wait for a confined core to hand the crash + * kernel over instead of parking. Waiting is parking until that happens. + */ +void arm64_crash_handoff_wait(void) +{ + if (!execmem_confine_active() || smp_processor_id() != 0 || + !kexec_crash_loaded()) + return; + + for (;;) { + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_TAKEN) == CRASH_HANDOFF_OFFERED) + machine_kexec(READ_ONCE(crash_handoff_image)); + wfe(); + } +} +NOKPROBE_SYMBOL(arm64_crash_handoff_wait); + +/* + * Offer the crash kernel to CPU 0 and park. Returns only when CPU 0 can't take + * it: it never acknowledged the stop, or didn't answer within a second. The + * panic then goes on to its normal restart. + */ +static void crash_handoff(struct kimage *kimage) +{ + unsigned int cpu = smp_processor_id(); + unsigned long timeout = USEC_PER_SEC; + + if (cpu_online(0)) { + pr_err("CPU 0 did not stop, so the crash kernel can't start from confined CPU %u\n", + cpu); + return; + } + + pr_info("CPU %u is confined to the CTXR window, CPU 0 starts the crash kernel\n", + cpu); + WRITE_ONCE(crash_handoff_image, kimage); + /* CPU 0 then sees no other CPU online, as after any crash stop. */ + set_cpu_online(cpu, false); + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_IDLE, + CRASH_HANDOFF_OFFERED) != CRASH_HANDOFF_IDLE) + cpu_park_loop(); + dsb(ish); + sev(); + + while (timeout--) { + if (READ_ONCE(crash_handoff_state) == CRASH_HANDOFF_TAKEN) + cpu_park_loop(); + udelay(1); + } + + if (cmpxchg(&crash_handoff_state, CRASH_HANDOFF_OFFERED, + CRASH_HANDOFF_WITHDRAWN) == CRASH_HANDOFF_TAKEN) + cpu_park_loop(); + + set_cpu_online(cpu, true); + pr_err("CPU 0 did not take the crash kernel\n"); +} +#else +static bool crash_needs_handoff(void) +{ + return false; +} + +static void crash_handoff(struct kimage *kimage) +{ +} +#endif + /** * machine_kexec - Do the kexec reboot. * @@ -175,6 +273,11 @@ void machine_kexec(struct kimage *kimage) bool in_kexec_crash = (kimage == kexec_crash_image); bool stuck_cpus = cpus_are_stuck_in_kernel(); + if (in_kexec_crash && crash_needs_handoff()) { + crash_handoff(kimage); + return; + } + /* * New cpus may have become stuck_in_kernel after we loaded the image. */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 9cd9bc7e40812..78d9b1857e147 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -924,6 +924,9 @@ void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash) sdei_mask_local_cpu(); + /* Under a CTXR window, CPU 0 may have to start the crash kernel. */ + arm64_crash_handoff_wait(); + if (crash && die_on_crash) __cpu_try_die(cpu); diff --git a/arch/arm64/mm/ctxr_execmem.c b/arch/arm64/mm/ctxr_execmem.c index 58f95322bd5d8..7851030724dca 100644 --- a/arch/arm64/mm/ctxr_execmem.c +++ b/arch/arm64/mm/ctxr_execmem.c @@ -22,6 +22,7 @@ #define pr_fmt(fmt) "ctxr-execmem: " fmt +#include #include #include #include @@ -255,7 +256,15 @@ void __init ctxr_exec_reserve(void) * core. So a command-line window can only replace the device-tree one * when it is identical; a different one (a sub-window, say) is ignored. */ - if (dt < 0) { + if (is_kdump_kernel() && (dt || cmd)) { + /* + * A crash kernel runs on the boot CPU only, from ordinary memory. + * The confined secondaries stay parked in the crashed kernel, + * whose window holds that kernel's code, not this one's. + */ + ctxr_hold_cpus = true; + pr_info("crash kernel: running on the boot CPU, outside the window\n"); + } else if (dt < 0) { ctxr_hold_secondaries("malformed aurora,ctxr-exec-window"); } else if (dt || cmd) { if (!is_kernel_in_hyp_mode()) { diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index e7f6751b718e3..bfb02a5524780 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -109,17 +109,10 @@ static void __init arch_reserve_crashkernel(void) return; /* - * A CTXR window refuses kexec (the relocation code runs outside it), so - * a crash kernel can never run. Don't reserve and waste its memory. + * Under a CTXR window the crash kernel runs on the boot CPU, which the + * window doesn't confine. The window is already reserved, so the crash + * kernel lands outside it. */ - if (ctxr_exec_window_reserved()) { - if (parse_crashkernel(boot_command_line, memblock_phys_mem_size(), - &crash_size, &crash_base, &low_size, - &cma_size, &high) == 0 && crash_size) - pr_warn("ctxr-execmem: ignoring crashkernel=, kexec is unavailable under a window\n"); - return; - } - ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), &crash_size, &crash_base, &low_size, &cma_size, &high); From 123384d69453821cbdc7879aace0474cbdda9bdc Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:48:29 +1100 Subject: [PATCH 063/392] nvme: apple: disable a controller handed over enabled on pre-M4 ANS On M1/M2-class ANS the coprocessor keeps running when the kernel is replaced without a shutdown, as when a crashed kernel starts the crash kernel. The new kernel finds CPU_CONTROL.RUN set and only wakes RTKit, but the NVMe controller is still enabled on the old kernel's queues. The admin and NVMMU queue registers may only be written while CC.EN is clear, so the controller keeps using the old queues and the new admin commands time out. On the first reset of a controller this kernel did not start, read CC from the hardware and disable the controller before programming the queues, as the post-M4 hand-over path already does. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/nvme/host/apple.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 732d7b63be3d4..150c8079fcfb9 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1621,6 +1621,7 @@ static void apple_nvme_reset_work(struct work_struct *work) struct apple_nvme *anv = container_of(work, struct apple_nvme, ctrl.reset_work); enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); + bool inherited = !anv->owns_rtkit; if (anv->hw->needs_ioq_registers) { mutex_lock(&anv->disable_lock); @@ -1772,6 +1773,24 @@ static void apple_nvme_reset_work(struct work_struct *work) dev_dbg(anv->dev, "ANS booted successfully."); + /* + * Pre-M4 ANS keeps running across a kexec, and a crashed kernel hands + * it to the crash kernel with the controller still enabled on its own + * queues. Woken this way, it would ignore the queue registers below, + * which may only change while CC.EN is clear. Disable it first. + */ + if (inherited && !anv->hw->needs_ioq_registers) { + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC); + if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) { + dev_info(anv->dev, + "controller handed over enabled, disabling it\n"); + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (ret) + goto out; + } + } + /* * Limit the max command size to prevent iod->sg allocations going * over a single page. From 95e9c52a0875ef41c167529993a85d69a9355e67 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:24:07 -0800 Subject: [PATCH 064/392] dt-bindings: pci: apple,pcie: Add T6050 hosts adopted from the boot loader Add apple,t6050-pcie, a T6050 (M5 Ultra) PCIe host whose root ports the boot loader powered, trained and left running. Each such port is marked apple,link-trained-by-bootloader and named by its port, phy and intr2axi register ranges and its port interrupt; the host's local MSI map starts at apple,msi-vector-base. The OS adopts the trained state and never resets, retrains or changes the port's power: reset-gpios, pwren-gpios and power-domains are not allowed on it. Also name nonposted-mmio, which these hosts take. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 55 +++++++++++++++++-- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 245231ecfc3a9..5823cc0ff837a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -98,11 +98,11 @@ properties: reg: minItems: 3 - maxItems: 11 + maxItems: 14 reg-names: minItems: 3 - maxItems: 11 + maxItems: 14 ranges: minItems: 2 @@ -128,8 +128,7 @@ properties: apple,msi-vector-base: $ref: /schemas/types.yaml#/definitions/uint32 description: - First vector of this PCIe-C controller's slice of the MSI window it - shares with the internal PCIe controller. + First global vector targeted by this controller's local MSI map. iommu-map: true iommu-map-mask: true @@ -234,6 +233,48 @@ allOf: - apple,msi-vector-base - $ref: /schemas/pci/pci-host-bridge.yaml# - $ref: /schemas/interrupt-controller/msi-controller.yaml# + - if: + properties: + compatible: + contains: + const: apple,t6050-pcie + then: + properties: + reg: + minItems: 5 + maxItems: 14 + reg-names: + description: + config and rc, then port, phy and intr2axi for each + bootloader-trained root port 00:.0. + minItems: 5 + maxItems: 14 + items: + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + interrupts: + maxItems: 4 + interrupt-names: + items: + pattern: "^port[0-3]$" + apple,msi-vector-base: + const: 64 + power-domains: false + required: + - interrupt-names + - apple,msi-vector-base + - iommu-map + patternProperties: + "^pci@": + properties: + reset-gpios: false + pwren-gpios: false + required: + - apple,link-trained-by-bootloader + else: + patternProperties: + "^pci@": + properties: + apple,link-trained-by-bootloader: false - if: properties: compatible: @@ -390,10 +431,10 @@ allOf: properties: reg: minItems: 5 - maxItems: 11 + maxItems: 14 reg-names: minItems: 5 - maxItems: 11 + maxItems: 14 items: enum: [config, rc, port0, phy0, intr2axi0, port1, phy1, intr2axi1, port2, phy2, intr2axi2, port3, phy3, @@ -420,6 +461,8 @@ allOf: maxItems: 3 else: properties: + reg: + maxItems: 10 reg-names: minItems: 3 items: From 500d016017203e44579be3068b4a542c6cde6eb1 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:25:06 -0800 Subject: [PATCH 065/392] PCI: apple: Adopt T6050 root ports the boot loader trained On the M5 Ultra (T6050, J775D) running Linux on the hardware, the boot loader brings up the internal PCIe host's ports itself: it powers the shared PHY, trains each port's link and leaves it up, with the endpoint idle. Port 2 carries the onboard RTL8127 Ethernet controller and port 3 the ASM3142 xHCI behind the rear USB-A ports. Repeating the reset and training sequence from Linux is not needed and not known to be safe on this SoC. Add apple,t6050-pcie, a host whose root ports marked apple,link-trained-by-bootloader are adopted as they are: no reset, no tunable replay, no clock or power change and no retraining. Every resource comes from the DT by the port's index (port, phy, intr2axi, interrupt port, the iommu-map entry for the endpoint's RID). Before the first PCIe register write the driver checks that the link is up and not retraining at Gen3 x1, that the boot loader numbered the root port's buses (primary 0, one secondary), that the endpoint on it is device 0 function 0 with DMA and MSI off and a well-formed capability list, and that the port's RID table holds no valid entry. A port that fails a check is left alone and the others are still adopted. Then the endpoint's RID is routed to its SID, the host's MSI map is set up from apple,msi-vector-base, and only the adopted root ports and the endpoint behind each are enumerated. The links cannot be recovered by a user reset or removal, since the boot loader owns them. No other SoC sets adopt_link. Testing: on J775D, every boot adopts port 2 (RTL8127, DHCP and SSH over it on every boot) and port 3 (ASM3142, a USB keyboard and mouse driving the desktop session). Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 283 ++++++++++++++++++++++++++-- 1 file changed, 270 insertions(+), 13 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3173186c85209..447e820d6349d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -207,6 +207,7 @@ struct hw_info { u32 pwren_on_ms; bool sid_indexed_rid2sid; bool retain_clocks; + bool adopt_link; bool root_bus_only; bool tunneled; bool tunnel_intr2axi; @@ -281,6 +282,24 @@ static const struct hw_info t8152_hw = { .pipe_rst = true, }; +/* + * Bare-metal T6050 adoption only. Firmware owns power, PHY, reset and training. + * The 64-entry route table is never probed with test writes. + */ +static const struct hw_info t6050_hw = { + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 64, + .retain_clocks = true, + .adopt_link = true, +}; + +#define T6050_ROUTE(rid, sid) (PORT_RID2SID_VALID | \ + ((sid) << PORT_RID2SID_SID_SHIFT) | (rid)) + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -441,6 +460,10 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; + bool adopted_route; + void __iomem *intr2axi; /* adopt_link: this port's */ + u16 adopted_rid; /* adopt_link: secondary bus << 8 */ + u32 adopted_sid; }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -752,9 +775,20 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 val = 0; - /* FIXME: consider moving each interrupt under each port */ - port->irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), - port->idx); + if (pcie->hw->adopt_link) { + char name[16]; + int irq; + + snprintf(name, sizeof(name), "port%d", port->idx); + irq = of_irq_get_byname(pcie->dev->of_node, name); + + if (irq <= 0) + return irq ?: -ENXIO; + port->irq = irq; + } else { + /* FIXME: consider moving each interrupt under each port */ + port->irq = irq_of_parse_and_map(pcie->dev->of_node, port->idx); + } if (!port->irq) return -ENXIO; @@ -767,6 +801,9 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) return -ENOMEM; } + if (pcie->hw->adopt_link) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + /* Disable all interrupts */ apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); @@ -800,7 +837,22 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } + if (pcie->hw->adopt_link) { + if (apple_pcie_port_readl(port, pcie->hw->port_msiaddr) != DOORBELL_ADDR || + apple_pcie_port_readl(port, pcie->hw->port_msiaddr_hi)) + return -EIO; + for (int i = 0; i < pcie->nvecs; i++) + if (apple_pcie_port_readl(port, pcie->hw->port_msimap + 4 * i) != + (PORT_MSIMAP_ENABLE | (pcie->msi_vector_base + i))) + return -EIO; + /* Enable command is not readable; +0x84 is only a completion read. */ + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + } apple_pcie_port_writel(port, val | PORT_MSICFG_EN, PORT_MSICFG); + if (pcie->hw->adopt_link && + !(apple_pcie_port_readl(port, PORT_MSICFG) & PORT_MSICFG_EN)) + return -EIO; return 0; } @@ -816,6 +868,8 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + if (port->pcie->hw->adopt_link) + WRITE_ONCE(port->link_failed, true); /* A later link-up alone does not prove downstream state survived. */ if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { WRITE_ONCE(port->link_failed, true); @@ -1494,9 +1548,18 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) apple_pcie_port_disable_irq(port); /* Busy managed admission has not acquired or programmed this IRQ block. */ - if (port->base && (!port->pcie->managed_sids || port->domain)) + if (port->base && + (port->domain || + (!port->pcie->managed_sids && !port->pcie->hw->adopt_link))) apple_pcie_port_writel(port, ~0, PORT_INTMSK); + if (port->pcie->hw->adopt_link && port->domain) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + if (port->adopted_route) { + apple_pcie_rid2sid_write(port, port->adopted_sid, 0); + port->adopted_route = false; + } + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); @@ -1580,6 +1643,134 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * Qualified J775D handoff of one bootloader-trained port: its root port + * 00:.0 and the single endpoint on its secondary bus, with DMA and + * endpoint MSI disabled. Every resource comes from the DT by the port's index + * (port, phy, intr2axi, interrupt port, the iommu-map entry for the + * endpoint's RID). All checks precede the first PCIe register write. + */ +static int apple_pcie_adopt_link(struct apple_pcie_port *port) +{ + static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; + struct apple_pcie *pcie = port->pcie; + struct platform_device *pdev = to_platform_device(pcie->dev); + struct of_phandle_args iommu = {}; + void __iomem *rp, *ep; + char name[16]; + u32 value, bus; + u8 cap; + bool msi = false; + int ret, i; + + if (pcie->cfg->busr.start != 0 || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "config") || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc")) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + snprintf(name, sizeof(name), names[i], port->idx); + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) + return -EINVAL; + } + + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN) || + !(readl(pcie->base + CORE_RC_STAT) & CORE_RC_STAT_READY) || + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY) || + (apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP || + !(apple_pcie_port_readl(port, PORT_LTSSMCTL) & PORT_LTSSMCTL_START) || + (apple_pcie_port_readl(port, PORT_APPCLK) & 0x101) != 0x101 || + (apple_pcie_port_readl(port, pcie->hw->port_perst) & 0x10001) != 1 || + (readl(port->phy + PHY_LANE_CFG) & 0x1000061f) != 0x60f) + return -ENOLINK; + + /* The bootloader numbered the root port's buses: primary 0, one secondary. */ + rp = pcie->cfg->win + (PCI_DEVFN(port->idx, 0) << 12); + value = readl(rp + PCI_PRIMARY_BUS); + bus = (value >> 8) & 0xff; + if (readl(rp) != 0x100c106b || (value & 0xff) != 0 || !bus || + ((value >> 16) & 0xff) != bus || bus > pcie->cfg->busr.end || + (readw(rp + PCI_BRIDGE_CONTROL) & PCI_BRIDGE_CTL_BUS_RESET)) + return -ENODEV; + ep = pcie->cfg->win + (bus << 20); + value = readl(ep); + if (value == 0xffffffff || !(value & 0xffff) || + (readb(ep + PCI_HEADER_TYPE) & 0x7f) != PCI_HEADER_TYPE_NORMAL || + (readw(ep + PCI_COMMAND) & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | + PCI_COMMAND_MASTER))) + return -ENODEV; + + port->adopted_rid = bus << 8; + ret = of_map_iommu_id(pcie->dev->of_node, port->adopted_rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + port->adopted_sid = iommu.args[0]; + of_node_put(iommu.np); + if (ret) + return ret; + + snprintf(name, sizeof(name), "intr2axi%d", port->idx); + port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->intr2axi)) + return PTR_ERR(port->intr2axi); + + /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ + cap = readb(rp + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + if (readb(rp + cap + PCI_CAP_LIST_ID) == PCI_CAP_ID_EXP) + break; + cap = readb(rp + cap + PCI_CAP_LIST_NEXT); + } + if (!cap || i == 48) + return -EINVAL; + value = readw(rp + cap + PCI_EXP_LNKSTA); + if ((value & (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != + (PCI_EXP_LNKSTA_CLS_8_0GB | (1 << PCI_EXP_LNKSTA_NLW_SHIFT))) + return -ENOLINK; + + /* Refuse an active endpoint or malformed capability chain. */ + cap = readb(ep + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + value = readw(ep + cap + PCI_MSI_FLAGS); + switch (readb(ep + cap + PCI_CAP_LIST_ID)) { + case PCI_CAP_ID_MSI: + if (value & PCI_MSI_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + case PCI_CAP_ID_MSIX: + if (value & PCI_MSIX_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + } + cap = readb(ep + cap + PCI_CAP_LIST_NEXT); + } + if (cap || !msi) + return -EINVAL; + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + if (apple_pcie_port_readl(port, port_rid2sid_offset(port, i)) & + PORT_RID2SID_VALID) + return -EBUSY; + + /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ + value = T6050_ROUTE(port->adopted_rid, port->adopted_sid); + port->adopted_route = true; + if (apple_pcie_rid2sid_write(port, port->adopted_sid, value) != value) + return -EIO; + dev_info(pcie->dev, "adopted port %d, RID %04x -> SID %u, slot %u\n", + port->idx, port->adopted_rid, port->adopted_sid, port->adopted_sid); + return 0; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -1866,6 +2057,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } } + } else if (pcie->hw->adopt_link) { + ret = apple_pcie_adopt_link(port); + if (ret) + goto err_teardown; } else { if (pcie->hw->retain_clocks) rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); @@ -1907,9 +2102,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; - if (pcie->managed_sids) { + if (pcie->managed_sids || pcie->hw->adopt_link) goto rid2sid_ready; - } /* Reset all RID/SID mappings, and check for RAZ/WI registers */ for (i = 0; i < pcie->hw->max_rid2sid; i++) { @@ -1934,6 +2128,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, unsigned long timeout, left = 0; int attempt, attempts = 1; + if (pcie->hw->adopt_link) { + ret = -ENOLINK; + goto err_teardown; + } if (pcie->hw->tunneled) { if (pcie->kernel_init) { apple_pcie_tunnel_apply_tunable(port->base, @@ -2080,6 +2278,17 @@ static int apple_msi_init(struct apple_pcie *pcie) return ret; } + if (pcie->hw->adopt_link) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + if (ret || pcie->msi_vector_base != 64 || + (pcie->nvecs != 32 && pcie->nvecs != 64) || + args.args_count != 4 || args.args[0] != 0 || args.args[1] != 0 || + args.args[2] != 1888 || args.args[3] != IRQ_TYPE_EDGE_RISING || + DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + /* * The controller-wide ADT count (512) is not a per-host allocation. * Internal PCIe owns vectors 0..31. Each PCIe-C bridge owns a distinct @@ -2186,7 +2395,7 @@ static int apple_msi_init(struct apple_pcie *pcie) invalid_slice: of_node_put(args.np); return dev_err_probe(pcie->dev, -EINVAL, - "invalid or missing PCIe-C MSI slice\n"); + "invalid or missing MSI slice\n"); } static void apple_pcie_cleanup(void *data) @@ -2387,6 +2596,11 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; + if (pcie->hw->adopt_link) + return rid == port->adopted_rid && sid == port->adopted_sid && + !READ_ONCE(port->link_failed) && + apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)) == + T6050_ROUTE(rid, sid) ? 0 : -ENODEV; mutex_lock(&port->pcie->lock); if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { @@ -2453,7 +2667,7 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci int idx; port = apple_pcie_get_port(pdev); - if (!port) + if (!port || port->pcie->hw->adopt_link) return; mutex_lock(&port->pcie->lock); @@ -2494,9 +2708,13 @@ static int apple_pcie_init(struct pci_config_window *cfg) ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); - return ret; + /* Adopted ports are independent: one refused leaves the others. */ + if (!pcie->hw->adopt_link) + return ret; } } + if (pcie->hw->adopt_link && list_empty(&pcie->ports)) + return -ENODEV; /* The bus scan follows this callback. */ apple_pcie_tunnel_wait_ready(pcie); @@ -2516,6 +2734,27 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); struct apple_pcie_port *port; + if (pcie->hw->adopt_link) { + /* Only adopted ports' root ports, and the one endpoint behind each. */ + list_for_each_entry(port, &pcie->ports, entry) { + if (bus->number == 0) { + if (devfn == PCI_DEVFN(port->idx, 0)) + return pci_ecam_map_bus(bus, devfn, where); + continue; + } + if (bus->number != port->adopted_rid >> 8) + continue; + if (devfn || READ_ONCE(port->link_failed)) + return NULL; + if ((apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { + WRITE_ONCE(port->link_failed, true); + return NULL; + } + return pci_ecam_map_bus(bus, devfn, where); + } + return NULL; + } if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); @@ -3089,7 +3328,22 @@ static int apple_pcie_probe(struct platform_device *pdev) * reset-gpios provider here rejects the valid synthesized port before * the tunneled setup path can consume that handoff. */ - if (!hw->tunneled) { + if (hw->adopt_link) { + if (!of_get_available_child_count(dev->of_node) || + of_property_present(dev->of_node, "power-domains")) + return -EINVAL; + for_each_available_child_of_node_scoped(dev->of_node, child) { + u32 reg; + + /* A root port 00:.0 on bus 0, n a GP port */ + if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || + !(reg >> 11) || (reg >> 11) > 3 || + !of_property_read_bool(child, "apple,link-trained-by-bootloader") || + of_property_present(child, "reset-gpios") || + of_property_present(child, "pwren-gpios")) + return -EINVAL; + } + } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_probe_port(of_port); @@ -3132,15 +3386,15 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; } pcie->piodma_supplier = piodma_supplier; - /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ - if (piodma_supplier || pcie->managed_sids) { + /* Firmware-owned links cannot be recovered by user reset/removal. */ + if (piodma_supplier || pcie->managed_sids || hw->adopt_link) { bridge->no_user_remove = true; bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_dbg(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); + dev_dbg(dev, "firmware-owned host: automatic AER/DPC recovery disabled\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -3920,6 +4174,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) */ if (pcie->managed_sids) return pm_suspend_target_state == PM_SUSPEND_TO_IDLE ? 0 : -EBUSY; + if (pcie->hw->adopt_link) + return -EBUSY; if (!pcie->hw->tunneled) return 0; pcie->reset_on_resume = false; @@ -4062,6 +4318,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8152-pcie", .data = &t8152_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, + { .compatible = "apple,t6050-pcie", .data = &t6050_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } }; From 9522dc08d514a8c46a501c2f08c7054a0facefb6 Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:16 -0400 Subject: [PATCH 066/392] PCI: Add pci_trylock_rescan_remove() A driver's work item that rescans or resets part of a bus takes the rescan/remove lock. When the device is being removed, the remover holds that lock while it cancels the work, so a worker that blocks on the lock deadlocks against its own cancellation. Add pci_trylock_rescan_remove(), which takes the rescan/remove lock only if it is free. A worker that cannot get it returns and retries later, and the remover's cancellation completes. Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/probe.c | 7 +++++++ include/linux/pci.h | 1 + 2 files changed, 8 insertions(+) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 27008e2ea5afc..d625dd5e778a8 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3517,6 +3517,13 @@ void pci_lock_rescan_remove(void) } EXPORT_SYMBOL_GPL(pci_lock_rescan_remove); +/* Work cancelled by removal must not wait on the remover's own lock. */ +bool pci_trylock_rescan_remove(void) +{ + return mutex_trylock(&pci_rescan_remove_lock); +} +EXPORT_SYMBOL_GPL(pci_trylock_rescan_remove); + void pci_unlock_rescan_remove(void) { mutex_unlock(&pci_rescan_remove_lock); diff --git a/include/linux/pci.h b/include/linux/pci.h index 5cec6d838b3f7..c0d76d93a78f5 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1559,6 +1559,7 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev); /* Functions for PCI Hotplug drivers to use */ unsigned int pci_rescan_bus(struct pci_bus *bus); void pci_lock_rescan_remove(void); +bool pci_trylock_rescan_remove(void); void pci_unlock_rescan_remove(void); /* Vital Product Data routines */ From 847c7493f8709bca983b86cbbe824c9726c9ce53 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 20:45:53 -0500 Subject: [PATCH 067/392] PCI: apple: Add the T6050 port reset, enable, configure and train sequences An adopted T6050 port is left as the boot loader trained it, but an endpoint that has to come back on a retrained link (the N1 on port 0, after its ROM boot) needs the host to take the port down and bring it up again. Add the sequences for that, on the port's own register blocks (port, phy and intr2axi) and the root port's config space: - Disable: wait for the link to leave L1 and go idle, assert the port's PERST# through its handshake, then stop the lane clocks and the port's application clock. - Enable: the port reset table, which also clears the RID2SID and MSI map tables, the port's tunables, then the application clock and the PERST# release handshake, until the port is ready and its link idle. - Configure: with the root complex's read-only registers opened, its tunables and the per-rate shadow tunables, then a Gen3 x1 link (target speed, link width and capability) and a directed speed change; Intr2AXI enabled. - Train: the lane's reference clocks, the endpoint's PERST# released through its pad, the LTSSM started, then the 100 ms wait after link up. The tunables are the lists the boot loader copies onto the root port node, groups of (offset, 4, mask, value); a list it did not copy is skipped. The root-port reset that runs the sequences follows; until then they are marked __maybe_unused. Co-developed-by: Dj Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 340 ++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 447e820d6349d..9048fe5f0c4d2 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -186,6 +186,20 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_MSIMAP_ENABLE BIT(31) #define PORT_MSIMAP_TARGET GENMASK(7, 0) +/* T6050: the root port's port logic, laid out as DesignWare's */ +#define RC_T6050_PORT_LINK_CTRL 0x710 +#define RC_T6050_LINK_MODE GENMASK(21, 16) +#define RC_T6050_PORT_DEBUG0 0x728 +#define RC_T6050_LTSSM_STATE GENMASK(5, 0) +#define RC_T6050_LTSSM_L1_IDLE 0x14 +#define RC_T6050_LINK_WIDTH_SPEED_CTRL 0x80c +#define RC_T6050_LINK_WIDTH GENMASK(12, 8) +#define RC_T6050_SPEED_CHANGE BIT(17) +#define RC_T6050_GEN3_RELATED 0x890 +#define RC_T6050_RATE_SHADOW_SEL GENMASK(25, 24) +#define RC_T6050_MISC_CONTROL_1 0x8bc +#define RC_T6050_DBI_RO_WR_EN BIT(0) + /* * The doorbell address must be in the bottom 4GB because the base register * is only 32 bits wide. Exclude it from the IOVA range and share the same @@ -464,6 +478,7 @@ struct apple_pcie_port { void __iomem *intr2axi; /* adopt_link: this port's */ u16 adopted_rid; /* adopt_link: secondary bus << 8 */ u32 adopted_sid; + void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -1771,6 +1786,331 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) return 0; } +/* + * T6050 port bring-up, for an adopted port whose endpoint must come back on a + * retrained link: port disable, the port reset table, port enable, root complex + * configuration for Gen3 x1, then the lane PHY, PERST# and the LTSSM. The port's + * registers are its port, phy and intr2axi blocks; the root complex's + * are the root port's config space. The boot loader copies the port's tunables + * onto its node. + */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001001ff }, + { 0x82c, 0x00010000 }, +}; + +/* Written after the RID2SID and MSI map tables are cleared */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs_late[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static void apple_pcie_t6050_rmw(void __iomem *addr, u32 clear, u32 set) +{ + writel((readl(addr) & ~clear) | set, addr); +} + +static int apple_pcie_t6050_rc_rmw(struct pci_dev *rp, int where, u32 clear, u32 set) +{ + u32 value; + int ret; + + ret = pci_read_config_dword(rp, where, &value); + if (!ret) + ret = pci_write_config_dword(rp, where, (value & ~clear) | set); + return pcibios_err_to_errno(ret); +} + +static int apple_pcie_t6050_wait(struct apple_pcie_port *port, void __iomem *base, + u32 offset, u32 mask, u32 value, unsigned int timeout_us) +{ + u32 stat; + int ret; + + ret = readl_poll_timeout(base + offset, stat, (stat & mask) == value, 1, timeout_us); + if (ret) + dev_err(port->pcie->dev, "port %d: %#x = %#x, waiting for %#x/%#x\n", + port->idx, offset, stat, value, mask); + return ret; +} + +/* + * A tunable list the boot loader copies onto the root port node, groups of + * (offset, 4, mask, value): for the port's registers, or with @rp for the root + * port's config space. A list it did not copy is skipped. + */ +static int apple_pcie_t6050_tune(struct apple_pcie_port *port, struct pci_dev *rp, + const char *name) +{ + u32 cells[64]; + int count, ret, i; + + if (!of_property_present(port->np, name)) + return 0; + count = of_property_count_u32_elems(port->np, name); + if (count <= 0 || count > ARRAY_SIZE(cells) || count % 4) + return -EINVAL; + ret = of_property_read_u32_array(port->np, name, cells, count); + if (ret) + return ret; + for (i = 0; i < count; i += 4) + if (cells[i + 1] != 4 || (cells[i] & 3) || + cells[i] >= (rp ? PCI_CFG_SPACE_EXP_SIZE : SZ_16K)) + return -EINVAL; + + for (i = 0; i < count; i += 4) { + if (!rp) { + apple_pcie_t6050_rmw(port->base + cells[i], cells[i + 2], cells[i + 3]); + continue; + } + ret = apple_pcie_t6050_rc_rmw(rp, cells[i], cells[i + 2], cells[i + 3]); + if (ret) + return ret; + } + return 0; +} + +/* PERST# through its pad: a GPIO output (function bits cleared) at the level asked for */ +static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) +{ + u32 old, value; + + old = readl(port->perst); + if (old & BIT(21)) + return -EIO; + value = (old & ~0x27f) | 0x202 | !asserted; + /* The input enable (bit 9) goes on after the rest */ + writel((value & ~0x200) | (old & 0x200), port->perst); + if (!(old & 0x200)) + writel(value, port->perst); + return (readl(port->perst) ^ value) & 0x27f ? -EIO : 0; +} + +/* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ +static int apple_pcie_t6050_port_disable(struct apple_pcie_port *port, struct pci_dev *rp) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + void __iomem *lane = port->phy + PHY_LANE_CFG; + u32 stat; + int ret, err; + + if (!(readl(port->base + PORT_STATUS) & BIT(2))) { + ret = apple_pcie_t6050_wait(port, port->base, 0x820, BIT(2), BIT(2), 10000); + if (ret) + return ret; + } + ret = readl_poll_timeout_atomic(port->base + 0x820, stat, + !(stat & GENMASK(8, 0)) || !(stat & GENMASK(23, 16)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + 0x13c, 0, BIT(8)); + + /* The link out of L1 */ + ret = read_poll_timeout_atomic(pci_read_config_dword, err, + err || FIELD_GET(RC_T6050_LTSSM_STATE, stat) != + RC_T6050_LTSSM_L1_IDLE, + 5, 5000, false, rp, RC_T6050_PORT_DEBUG0, &stat); + if (ret || err) + return ret ?: pcibios_err_to_errno(err); + + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(24)); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(16), BIT(16), 2000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(16)); + udelay(1); + apple_pcie_t6050_rmw(perst, PORT_PERST_OFF, 0); + udelay(1); + apple_pcie_t6050_rmw(perst, BIT(24), 0); + apple_pcie_t6050_rmw(port->base + 0x13c, BIT(8), 0); + + apple_pcie_t6050_rmw(lane, BIT(31), 0); + apple_pcie_t6050_rmw(lane, BIT(10), 0); + apple_pcie_t6050_rmw(lane, BIT(9), 0); + apple_pcie_t6050_rmw(lane, 0, BIT(4)); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK0REQ, 0); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK1REQ, 0); + + writel(~0, port->base + PORT_INTSTAT); + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, PORT_APPCLK_EN, 0); + return apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, 0, + 250000); +} + +/* The port reset table, then the port clocked and out of reset, ready to train */ +static int apple_pcie_t6050_port_enable(struct apple_pcie_port *port) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + u32 stat; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs); i++) + writel(apple_pcie_t6050_reset_regs[i].value, + port->base + apple_pcie_t6050_reset_regs[i].offset); + for (i = 0; i < port->pcie->hw->max_rid2sid; i++) + writel(0, port->base + port_rid2sid_offset(port, i)); + for (i = 0; i < 256; i++) + writel(0, port->base + PORT_T602X_MSIMAP + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs_late); i++) + writel(apple_pcie_t6050_reset_regs_late[i].value, + port->base + apple_pcie_t6050_reset_regs_late[i].offset); + + if (!(readl(perst) & BIT(16))) + return -EIO; + ret = apple_pcie_t6050_tune(port, NULL, "apple,apcie-config-tunables"); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_EN); + + ret = readl_poll_timeout_atomic(port->base + 0xa8, stat, + (stat & (BIT(16) | BIT(0))) != (BIT(16) | BIT(0)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, PORT_PERST_OFF); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 20000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, BIT(16), 0); + ret = apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, + PORT_STATUS_READY, 250000); + if (ret) + return ret; + + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_CGDIS); + return apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_BUSY, 0, + 250000); +} + +/* Root complex tunables and a Gen3 x1 link, with the read-only registers opened for it */ +static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port, struct pci_dev *rp) +{ + static const char * const shadow[] = { + "apple,pcie-rc-gen3-shadow-tunables", + "apple,pcie-rc-gen4-shadow-tunables", + "apple,pcie-rc-gen5-shadow-tunables", + }; + static const u32 shadow_sel[] = { 0, BIT(24), 0 }; + int cap = pci_pcie_cap(rp); + u32 value; + u16 ctl2; + int ret, err, i; + + if (!cap) + return -EIO; + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, 0, RC_T6050_DBI_RO_WR_EN); + if (ret) + return ret; + ret = apple_pcie_t6050_tune(port, rp, "apple,pcie-rc-tunables"); + if (ret) + goto lock; + for (i = 0; i < ARRAY_SIZE(shadow); i++) { + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_GEN3_RELATED, + RC_T6050_RATE_SHADOW_SEL, shadow_sel[i]) ?: + apple_pcie_t6050_tune(port, rp, shadow[i]); + if (ret) + goto lock; + } + ret = apple_pcie_t6050_rc_rmw(rp, 0x194, 0, BIT(0)); + if (ret) + goto lock; + + ret = pcibios_err_to_errno(pci_read_config_dword(rp, cap + PCI_EXP_LNKCAP2, &value)); + if (ret) + goto lock; + if (!(value & PCI_EXP_LNKCAP2_SLS_8_0GB)) { + ret = -EIO; + goto lock; + } + ret = pcibios_err_to_errno(pci_read_config_word(rp, cap + PCI_EXP_LNKCTL2, &ctl2)) ?: + pcibios_err_to_errno(pci_write_config_word(rp, cap + PCI_EXP_LNKCTL2, + (ctl2 & ~PCI_EXP_LNKCTL2_TLS) | + PCI_EXP_LNKCTL2_TLS_8_0GT)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_PORT_LINK_CTRL, RC_T6050_LINK_MODE, + FIELD_PREP(RC_T6050_LINK_MODE, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, RC_T6050_LINK_WIDTH, + FIELD_PREP(RC_T6050_LINK_WIDTH, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW, + FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1)); +lock: + err = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, RC_T6050_DBI_RO_WR_EN, 0); + ret = ret ?: err; + if (ret) + return ret; + + ret = apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCTL, 0, PCI_EXP_LNKCTL_LBMIE) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, 0, RC_T6050_SPEED_CHANGE); + if (ret) + return ret; + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + return 0; +} + +/* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ +static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port *port, + struct pci_dev *rp) +{ + return apple_pcie_t6050_port_disable(port, rp) ?: + apple_pcie_t6050_port_enable(port) ?: + apple_pcie_t6050_port_configure(port, rp); +} + +/* Lane clocks, PERST# released, then the LTSSM to link up */ +static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *port) +{ + void __iomem *lane = port->phy + PHY_LANE_CFG; + int ret; + + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK0REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK0ACK, + PHY_LANE_CFG_REFCLK0ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK1REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK1ACK, + PHY_LANE_CFG_REFCLK1ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, BIT(4), 0); + udelay(1); + apple_pcie_t6050_rmw(lane, 0, BIT(9)); + apple_pcie_t6050_rmw(lane, 0, BIT(10)); + + /* Tperst-clk is at least 100 us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 110); + ret = apple_pcie_t6050_perst(port, false); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_LTSSMCTL, 0, PORT_LTSSMCTL_START); + ret = apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_UP, + PORT_LINKSTS_UP, 500000); + if (ret) + return ret; + + /* Wait for 100ms after the link trains (PCIe r5.0, 6.6.1) */ + msleep(100); + return 0; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) From b8c92012dfe330a0bb5c494043855ced1c608079 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 068/392] PCI: apple: Adopt T6050 port 0's multi-function N1 and reset its root port The N1 Wi-Fi/Bluetooth sits on apcie0 port 0, which the bootloader now trains like ports 2 and 3. Two things the adoption assumed do not hold for it: - It is multi-function. In its ROM personality only function 0 answers; after its firmware starts, Control, Alpha and Beta (0-2) do. Each function's RID is routed to its own DART stream from the iommu-map (slot == SID) when it is enabled, instead of the one route set at adoption; config accesses on the adopted bus reach every function of device 0; teardown clears every slot it routed. Port 0 is accepted. - Its ROM links at Gen1. Adoption takes an x1 link at Gen1 to Gen3 (the bootloader's target), not only Gen3; the endpoint picks the speed. - After the ROM boot it must come back on a retrained link. The host provides reset_root_port for an adopted port that names its PERST# pad (perst): PME_Turn_Off to L2, PERST#, then the T6050 port disable, reset, enable, configure and train sequences, with the routes and the MSI map rewritten in between. Tunables the boot loader copied onto the root port node are applied; absent ones are skipped. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 9 +- drivers/pci/controller/pcie-apple.c | 218 ++++++++++++++---- 2 files changed, 184 insertions(+), 43 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 5823cc0ff837a..2446facb91d3a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -242,15 +242,16 @@ allOf: properties: reg: minItems: 5 - maxItems: 14 + maxItems: 18 reg-names: description: config and rc, then port, phy and intr2axi for each - bootloader-trained root port 00:.0. + bootloader-trained root port 00:.0, and perst (its PERST# + pad) for a port whose endpoint needs a root-port reset. minItems: 5 - maxItems: 14 + maxItems: 18 items: - pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + pattern: "^(config|rc|(port|phy|intr2axi|perst)[0-3])$" interrupts: maxItems: 4 interrupt-names: diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 9048fe5f0c4d2..4ebf2208e9ab0 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -474,14 +474,14 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; - bool adopted_route; void __iomem *intr2axi; /* adopt_link: this port's */ - u16 adopted_rid; /* adopt_link: secondary bus << 8 */ - u32 adopted_sid; + u8 adopted_bus; /* adopt_link: the secondary bus */ + unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); +static struct apple_pcie *apple_pcie_lookup(struct device *dev); static int apple_pcie_radio_check_cold(struct apple_pcie_port *port); static void rmw_set(u32 set, void __iomem *addr) @@ -1570,9 +1570,12 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) if (port->pcie->hw->adopt_link && port->domain) apple_pcie_port_writel(port, 0, PORT_MSICFG); - if (port->adopted_route) { - apple_pcie_rid2sid_write(port, port->adopted_sid, 0); - port->adopted_route = false; + if (port->adopted_slots) { + int slot; + + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + apple_pcie_rid2sid_write(port, slot, 0); + port->adopted_slots = 0; } if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) @@ -1658,9 +1661,60 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * The DART stream an adopted port's function `rid` maps to: one T6050 DART stream, nonzero + * and below the RID2SID table size (slot == SID), from the host's iommu-map. + */ +static int apple_pcie_adopted_sid(struct apple_pcie_port *port, u32 rid, u32 *sid) +{ + struct apple_pcie *pcie = port->pcie; + struct of_phandle_args iommu = {}; + int ret; + + if (rid >> 8 != port->adopted_bus) { + /* The PCI core numbers root ports' buses in devfn order: the bootloader must too */ + dev_err(pcie->dev, "port %d: RID %04x is off bus %02x, which the bootloader numbered\n", + port->idx, rid, port->adopted_bus); + return -ENODEV; + } + ret = of_map_iommu_id(pcie->dev->of_node, rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + iommu.args[0] < BITS_PER_LONG && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + *sid = iommu.args[0]; + of_node_put(iommu.np); + return ret; +} + +/* Route an adopted port's function `rid` to its stream, or confirm the route is in place. */ +static int apple_pcie_adopted_route(struct apple_pcie_port *port, u32 rid) +{ + u32 sid, value, cur; + int ret = apple_pcie_adopted_sid(port, rid, &sid); + + if (ret) + return ret; + value = T6050_ROUTE(rid, sid); + cur = apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)); + if (cur == value && test_bit(sid, &port->adopted_slots)) + return 0; + if (cur & PORT_RID2SID_VALID) + return -EBUSY; + set_bit(sid, &port->adopted_slots); + if (apple_pcie_rid2sid_write(port, sid, value) != value) + return -EIO; + dev_info(port->pcie->dev, "port %d: RID %04x -> SID %u, slot %u\n", + port->idx, rid, sid, sid); + return 0; +} + /* * Qualified J775D handoff of one bootloader-trained port: its root port - * 00:.0 and the single endpoint on its secondary bus, with DMA and + * 00:.0 and the endpoint on its secondary bus (function 0 routed here; a + * multi-function endpoint's other functions when they are enabled), with DMA and * endpoint MSI disabled. Every resource comes from the DT by the port's index * (port, phy, intr2axi, interrupt port, the iommu-map entry for the * endpoint's RID). All checks precede the first PCIe register write. @@ -1670,7 +1724,6 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; struct apple_pcie *pcie = port->pcie; struct platform_device *pdev = to_platform_device(pcie->dev); - struct of_phandle_args iommu = {}; void __iomem *rp, *ep; char name[16]; u32 value, bus; @@ -1715,15 +1768,8 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) PCI_COMMAND_MASTER))) return -ENODEV; - port->adopted_rid = bus << 8; - ret = of_map_iommu_id(pcie->dev->of_node, port->adopted_rid, &iommu); - if (ret) - return ret; - ret = iommu.args_count == 1 && iommu.args[0] && - iommu.args[0] < pcie->hw->max_rid2sid && - of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; - port->adopted_sid = iommu.args[0]; - of_node_put(iommu.np); + port->adopted_bus = bus; + ret = apple_pcie_adopted_sid(port, bus << 8, &value); if (ret) return ret; @@ -1731,6 +1777,13 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); if (IS_ERR(port->intr2axi)) return PTR_ERR(port->intr2axi); + /* Only a port whose endpoint needs a root-port reset names its PERST# pad */ + snprintf(name, sizeof(name), "perst%d", port->idx); + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) { + port->perst = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->perst)) + return PTR_ERR(port->perst); + } /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ cap = readb(rp + PCI_CAPABILITY_LIST); @@ -1743,9 +1796,14 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) } if (!cap || i == 48) return -EINVAL; + /* + * x1 at up to the Gen3 target the bootloader set. The endpoint picks the speed: the N1's + * ROM personality links at Gen1 and comes back at Gen3 after its root-port reset. + */ value = readw(rp + cap + PCI_EXP_LNKSTA); - if ((value & (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != - (PCI_EXP_LNKSTA_CLS_8_0GB | (1 << PCI_EXP_LNKSTA_NLW_SHIFT))) + if ((value & (PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != (1 << PCI_EXP_LNKSTA_NLW_SHIFT) || + (value & PCI_EXP_LNKSTA_CLS) < PCI_EXP_LNKSTA_CLS_2_5GB || + (value & PCI_EXP_LNKSTA_CLS) > PCI_EXP_LNKSTA_CLS_8_0GB) return -ENOLINK; /* Refuse an active endpoint or malformed capability chain. */ @@ -1777,13 +1835,10 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) return -EBUSY; /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ - value = T6050_ROUTE(port->adopted_rid, port->adopted_sid); - port->adopted_route = true; - if (apple_pcie_rid2sid_write(port, port->adopted_sid, value) != value) - return -EIO; - dev_info(pcie->dev, "adopted port %d, RID %04x -> SID %u, slot %u\n", - port->idx, port->adopted_rid, port->adopted_sid, port->adopted_sid); - return 0; + ret = apple_pcie_adopted_route(port, bus << 8); + if (!ret) + dev_info(pcie->dev, "adopted port %d, endpoint %02x:00\n", port->idx, bus); + return ret; } /* @@ -2066,8 +2121,7 @@ static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port, struct } /* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ -static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port *port, - struct pci_dev *rp) +static int apple_pcie_t6050_port_restart(struct apple_pcie_port *port, struct pci_dev *rp) { return apple_pcie_t6050_port_disable(port, rp) ?: apple_pcie_t6050_port_enable(port) ?: @@ -2075,7 +2129,7 @@ static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port * } /* Lane clocks, PERST# released, then the LTSSM to link up */ -static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *port) +static int apple_pcie_t6050_port_train(struct apple_pcie_port *port) { void __iomem *lane = port->phy + PHY_LANE_CFG; int ret; @@ -2111,6 +2165,87 @@ static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *po return 0; } +/* + * Root-port reset of an adopted T6050 port, for an endpoint that must come back on a + * retrained link (the N1 after its ROM boot): PME_Turn_Off to L2, PERST# asserted, the + * port disabled, reset, enabled and configured, its routes and MSI map rewritten, then the + * lane PHY, PERST# release and LTSSM to a Gen3 x1 link. The caller has quiesced the + * endpoint and saved the root port's state; the PCI core restores it. + */ +static int apple_pcie_t6050_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *rp) +{ + struct pci_config_window *cfg = rp->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port = NULL, *p; + u32 routes[BITS_PER_LONG], value; + int ret, slot, i; + + if (!pcie || !pcie->hw->adopt_link || !pci_is_root_bus(rp->bus)) + return -EOPNOTSUPP; + list_for_each_entry(p, &pcie->ports, entry) + if (p->idx == PCI_SLOT(rp->devfn)) + port = p; + if (!port || !port->perst || !port->intr2axi) + return -EOPNOTSUPP; + + mutex_lock(&pcie->lock); + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + routes[slot] = apple_pcie_port_readl(port, port_rid2sid_offset(port, slot)); + + /* Bridge decoding off and the secondary bus unnumbered while the link is down */ + ret = pcibios_err_to_errno(pci_write_config_word(rp, PCI_COMMAND, 0)) ?: + pcibios_err_to_errno(pci_read_config_dword(rp, PCI_PRIMARY_BUS, &value)) ?: + pcibios_err_to_errno(pci_write_config_dword(rp, PCI_PRIMARY_BUS, + value & 0xff0000ff)); + if (ret) + goto out; + /* PME_Turn_Off, then L2 */ + apple_pcie_port_writel(port, apple_pcie_port_readl(port, PORT_APPCLK) | 0x100, PORT_APPCLK); + apple_pcie_port_writel(port, 0x11, 0x88); + ret = apple_pcie_t6050_wait(port, port->base, 0x88, 1, 0, 500000) ?: + apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, BIT(6), BIT(6), 500000); + if (ret) + goto out; + WRITE_ONCE(port->link_failed, true); + ret = apple_pcie_t6050_perst(port, true); + if (ret) + goto out; + writel(readl(port->phy) & ~BIT(31), port->phy); + ret = apple_pcie_t6050_port_restart(port, rp); + if (ret) + goto out; + + /* The reset table cleared the routes and the MSI setup: put back what Linux set */ + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + if (apple_pcie_rid2sid_write(port, slot, routes[slot]) != routes[slot]) + ret = -EIO; + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); + for (i = 0; i < pcie->nvecs; i++) + apple_pcie_port_writel(port, FIELD_PREP(PORT_MSIMAP_TARGET, pcie->msi_vector_base + i) | + PORT_MSIMAP_ENABLE, pcie->hw->port_msimap + 4 * i); + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + apple_pcie_port_writel(port, PORT_MSICFG_EN, PORT_MSICFG); + if (ret) + goto out; + + ret = apple_pcie_t6050_port_train(port); + if (ret) + goto out; + /* Link interrupts stay masked after the reset table: the link is checked on access */ + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + WRITE_ONCE(port->link_failed, false); + dev_info(pcie->dev, "port %d: link retrained after a root-port reset\n", port->idx); +out: + mutex_unlock(&pcie->lock); + if (ret) + dev_err(pcie->dev, "port %d: root-port reset stopped: %d\n", port->idx, ret); + return ret; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -2936,11 +3071,14 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; - if (pcie->hw->adopt_link) - return rid == port->adopted_rid && sid == port->adopted_sid && - !READ_ONCE(port->link_failed) && - apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)) == - T6050_ROUTE(rid, sid) ? 0 : -ENODEV; + if (pcie->hw->adopt_link) { + if (READ_ONCE(port->link_failed)) + return -ENODEV; + mutex_lock(&pcie->lock); + err = apple_pcie_adopted_route(port, rid); + mutex_unlock(&pcie->lock); + return err; + } mutex_lock(&port->pcie->lock); if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { @@ -3075,16 +3213,16 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie_port *port; if (pcie->hw->adopt_link) { - /* Only adopted ports' root ports, and the one endpoint behind each. */ + /* Only adopted ports' root ports, and the endpoint behind each. */ list_for_each_entry(port, &pcie->ports, entry) { if (bus->number == 0) { if (devfn == PCI_DEVFN(port->idx, 0)) return pci_ecam_map_bus(bus, devfn, where); continue; } - if (bus->number != port->adopted_rid >> 8) + if (bus->number != port->adopted_bus) continue; - if (devfn || READ_ONCE(port->link_failed)) + if (PCI_SLOT(devfn) || READ_ONCE(port->link_failed)) return NULL; if ((apple_pcie_port_readl(port, PORT_LINKSTS) & (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { @@ -3675,9 +3813,9 @@ static int apple_pcie_probe(struct platform_device *pdev) for_each_available_child_of_node_scoped(dev->of_node, child) { u32 reg; - /* A root port 00:.0 on bus 0, n a GP port */ + /* A root port 00:.0 on bus 0, n a GP port (0-3) */ if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || - !(reg >> 11) || (reg >> 11) > 3 || + (reg >> 11) > 3 || !of_property_read_bool(child, "apple,link-trained-by-bootloader") || of_property_present(child, "reset-gpios") || of_property_present(child, "pwren-gpios")) @@ -3800,6 +3938,8 @@ static int apple_pcie_probe(struct platform_device *pdev) } } + if (hw->adopt_link) + bridge->reset_root_port = apple_pcie_t6050_reset_root_port; ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); if (ret) { apple_pcie_cleanup(pcie); From 2b7607519da94a9a599cc6b2cece6c3098167479 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 23:53:26 -0800 Subject: [PATCH 069/392] PCI: apple: Drive T6050 port 0's PERST# through the pin controller The adopted port 0 named its PERST# pad as a 4-byte reg of the host (perst0, one word inside the pin controller's block) and wrote it raw. With the pin controller in the device tree the two nodes claim the same MMIO word, and whichever probes second fails: when pinctrl wins, the host's port setup stops with -EBUSY and no port is adopted, so the Ethernet and USB behind ports 2 and 3 are gone with it. One owner for the pad: the pin controller. The port node carries reset-gpios, as every other Apple host's port nodes do, and the host takes the line as the bootloader left it (GPIOD_ASIS: probe never touches a trained link's reset) and drives it only in the root-port reset, as a GPIO output at the logical level the reset asks for. The raw pad write set the same things (function bits cleared, mode output, the data bit) plus the pad's input enable, which the pin controller leaves as the bootloader set it. Probe asks for the line before any port is set up, so a host whose pin controller is not up yet defers instead of adopting nothing; an adopted port without reset-gpios has no reset, as before (reset_root_port answers -EOPNOTSUPP). The perst reg, its binding entry and the raw pad write go; the binding allows reset-gpios on an adopted port again. Probe names the port whose line it cannot have, as the probe of the other hosts' ports does: a line the pin controller reserves (gpio-reserved-ranges) is refused with -EINVAL, which the host's probe otherwise returned with no word of which port or property. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 15 +++--- drivers/pci/controller/pcie-apple.c | 49 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 2446facb91d3a..64f0a2d43e9aa 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -242,16 +242,15 @@ allOf: properties: reg: minItems: 5 - maxItems: 18 + maxItems: 14 reg-names: description: config and rc, then port, phy and intr2axi for each - bootloader-trained root port 00:.0, and perst (its PERST# - pad) for a port whose endpoint needs a root-port reset. + bootloader-trained root port 00:.0. minItems: 5 - maxItems: 18 + maxItems: 14 items: - pattern: "^(config|rc|(port|phy|intr2axi|perst)[0-3])$" + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" interrupts: maxItems: 4 interrupt-names: @@ -267,7 +266,11 @@ allOf: patternProperties: "^pci@": properties: - reset-gpios: false + reset-gpios: + description: + The PERST# line of a port whose endpoint needs a root-port + reset after the bootloader trained the link. Left as the + bootloader set it until that reset. pwren-gpios: false required: - apple,link-trained-by-bootloader diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 4ebf2208e9ab0..349e008bb0154 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -458,7 +458,7 @@ struct apple_pcie_port { struct device_node *np; void __iomem *base; void __iomem *phy; - /* PERST# of a conventional port with a single reset line. */ + /* PERST# of a conventional or adopted port with a single reset line. */ struct gpio_desc *perst; struct irq_domain *domain; struct list_head entry; @@ -477,7 +477,6 @@ struct apple_pcie_port { void __iomem *intr2axi; /* adopt_link: this port's */ u8 adopted_bus; /* adopt_link: the secondary bus */ unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ - void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -1777,12 +1776,18 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); if (IS_ERR(port->intr2axi)) return PTR_ERR(port->intr2axi); - /* Only a port whose endpoint needs a root-port reset names its PERST# pad */ - snprintf(name, sizeof(name), "perst%d", port->idx); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) { - port->perst = devm_platform_ioremap_resource_byname(pdev, name); - if (IS_ERR(port->perst)) + /* + * Only a port whose endpoint needs a root-port reset names its PERST# line + * (reset-gpios, on the port node as on every other Apple host). The pin + * controller owns the pad; the line is taken as the bootloader left it, + * released, and driven only by the reset. + */ + port->perst = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), "reset", + GPIOD_ASIS, "PERST#"); + if (IS_ERR(port->perst)) { + if (PTR_ERR(port->perst) != -ENOENT) return PTR_ERR(port->perst); + port->perst = NULL; } /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ @@ -1940,20 +1945,14 @@ static int apple_pcie_t6050_tune(struct apple_pcie_port *port, struct pci_dev *r return 0; } -/* PERST# through its pad: a GPIO output (function bits cleared) at the level asked for */ +/* + * PERST# through the pin controller: the pad as a GPIO output (its function bits + * cleared, mode output) at the line's logical level, which the DT's + * GPIO_ACTIVE_LOW turns into the pad level. + */ static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) { - u32 old, value; - - old = readl(port->perst); - if (old & BIT(21)) - return -EIO; - value = (old & ~0x27f) | 0x202 | !asserted; - /* The input enable (bit 9) goes on after the rest */ - writel((value & ~0x200) | (old & 0x200), port->perst); - if (!(old & 0x200)) - writel(value, port->perst); - return (readl(port->perst) ^ value) & 0x27f ? -EIO : 0; + return gpiod_direction_output(port->perst, asserted); } /* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ @@ -3817,9 +3816,19 @@ static int apple_pcie_probe(struct platform_device *pdev) if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || (reg >> 11) > 3 || !of_property_read_bool(child, "apple,link-trained-by-bootloader") || - of_property_present(child, "reset-gpios") || of_property_present(child, "pwren-gpios")) return -EINVAL; + /* + * A port whose endpoint needs a root-port reset names its PERST# + * line: probe waits for the line's controller before any port is + * touched (a failed adoption would otherwise swallow the deferral). + */ + if (of_property_present(child, "reset-gpios")) { + ret = apple_pcie_probe_port(child); + if (ret) + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", + child); + } } } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ From 911da8c5008a05f69cffb1cc1169a598c9138117 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 01:06:09 -0500 Subject: [PATCH 070/392] selftests: apple: Preserve multiline PCIe macros in policy tests The PCIe-C policy test extracts driver macro definitions to compile its production-code fixture. Taking only the first physical line truncates T6050_ROUTE and joins the following directive to its trailing backslash, so the fixture fails to compile after the T6050 host support is added. Copy continuation lines along with their define. The production fixture and both negative controls now pass. Developed with Claude Code. Signed-off-by: Erik Bourget --- tools/asahi/pciec-policy-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/pciec-policy-tests.py b/tools/asahi/pciec-policy-tests.py index 825caefac0912..60442d60b8410 100644 --- a/tools/asahi/pciec-policy-tests.py +++ b/tools/asahi/pciec-policy-tests.py @@ -144,7 +144,7 @@ def main(): args = parser.parse_args() pci = (ROOT / 'drivers/pci/controller/pcie-apple.c').read_text() acio = (ROOT / 'drivers/thunderbolt/apple.c').read_text() - defines = '\n'.join(re.findall(r'^#define .*$', pci, re.M)) + defines = '\n'.join(re.findall(r'^#define (?:[^\n]*\\\n)*[^\n]*$', pci, re.M)) info = pci[pci.index('struct hw_info {'):pci.index('static const struct of_device_id apple_pcie_of_match[];')] matches = re.search(r'static const struct of_device_id apple_pcie_of_match\[\] = \{.*?\n\};', pci, re.S)[0] production = '\n'.join((defines, info, matches, From 1aea0561f10facd395ef7e93db969481e88fe06f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:15:11 +1100 Subject: [PATCH 071/392] kexec: let a crash kernel start while devices keep DMA to retained memory The retained-memory interlock refuses every load and every kexec while a device still DMAs to memory that cannot be quiesced. That protects a new kernel that would hand such memory out again. A crash kernel never does: it runs in the crash kernel reservation, which memblock set aside at boot before any driver could give memory to a device, and it only reads the old kernel's memory. On the MacBook Neo the radio and display keep such DMA for the whole boot, so the interlock made kdump impossible there. Let crash images load and run while blockers are held, unless the crash kernel has CMA-backed memory (crashkernel=,cma), which is lent to the page allocator at run time. Ordinary kexec stays refused. The panic log names the first blocker when the crash kernel starts anyway. Update the KUnit cases to the new crash policy. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- kernel/crash_core.c | 3 +++ kernel/kexec.c | 3 ++- kernel/kexec_block_test.c | 17 ++++++++++++----- kernel/kexec_file.c | 3 ++- kernel/kexec_internal.h | 15 ++++++++++++++- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 3a9892c93eb58..e36e12bb18c43 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -142,6 +142,9 @@ void __noclone __crash_kexec(struct pt_regs *regs) } else { struct pt_regs fixed_regs; + if (kexec_blocked()) + pr_warn("crash kernel starts while devices keep DMA to retained memory; first reason: %s\n", + kexec_block_reason()); crash_setup_regs(&fixed_regs, regs); crash_save_vmcoreinfo(); machine_crash_shutdown(&fixed_regs); diff --git a/kernel/kexec.c b/kernel/kexec.c index 34c9e8a79ba49..41d3145062194 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -102,7 +102,8 @@ int do_kexec_load(unsigned long entry, unsigned long nr_segments, */ if (!kexec_trylock()) return -EBUSY; - if (nr_segments && kexec_blocked()) { + if (nr_segments && ((flags & KEXEC_ON_CRASH) ? kexec_crash_blocked() : + kexec_blocked())) { pr_warn_ratelimited("kexec load refused by active interlock; first reason: %s\n", kexec_block_reason()); ret = -EOPNOTSUPP; diff --git a/kernel/kexec_block_test.c b/kernel/kexec_block_test.c index 495e37612fe0c..cc9146bd5ab8f 100644 --- a/kernel/kexec_block_test.c +++ b/kernel/kexec_block_test.c @@ -139,12 +139,17 @@ static void kexec_block_legacy_load_test(struct kunit *test) { /* Bad alignment also makes a missing guard fail safely in validation. */ struct kexec_segment segment = { .mem = 1, .memsz = PAGE_SIZE }; + int ret; kexec_test_acquire(test); kexec_test_require_empty_images(test); KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, 0), -EOPNOTSUPP); - KUNIT_EXPECT_EQ(test, do_kexec_load(0, 1, &segment, KEXEC_ON_CRASH), - -EOPNOTSUPP); + /* A crash kernel only passes the guard without CMA crash memory. */ + ret = do_kexec_load(0, 1, &segment, KEXEC_ON_CRASH); + if (crashk_cma_cnt) + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + else + KUNIT_EXPECT_EQ(test, ret, -EADDRNOTAVAIL); /* Unload an empty slot through the actual loader while still blocked. */ KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, 0), 0); KUNIT_EXPECT_EQ(test, do_kexec_load(0, 0, NULL, KEXEC_ON_CRASH), 0); @@ -166,9 +171,10 @@ static void kexec_block_file_load_test(struct kunit *test) if (ret == -EPERM) kunit_skip(test, "file loads disabled by permissions or attempt limit"); KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + /* Past the guard, the invalid fd stops a crash load before any image. */ KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_ON_CRASH), - -EOPNOTSUPP); + crashk_cma_cnt ? -EOPNOTSUPP : -EBADF); KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, KEXEC_FILE_UNLOAD), 0); KUNIT_EXPECT_EQ(test, do_kexec_file_load(-1, -1, 0, NULL, @@ -188,7 +194,8 @@ static void kexec_block_crash_policy_test(struct kunit *test) loaded_allowed = kexec_crash_image_allowed(true); kexec_unlock(); KUNIT_EXPECT_FALSE(test, empty_allowed); - KUNIT_EXPECT_FALSE(test, loaded_allowed); + /* Retained memory never overlaps a memblock crash reservation. */ + KUNIT_EXPECT_EQ(test, loaded_allowed, !crashk_cma_cnt); kexec_test_release(test); KUNIT_ASSERT_TRUE(test, kexec_trylock()); /* A real guard may have appeared since this case's initial skip check. */ @@ -197,7 +204,7 @@ static void kexec_block_crash_policy_test(struct kunit *test) loaded_allowed = kexec_crash_image_allowed(true); kexec_unlock(); KUNIT_EXPECT_FALSE(test, empty_allowed); - KUNIT_EXPECT_EQ(test, loaded_allowed, !blocked); + KUNIT_EXPECT_EQ(test, loaded_allowed, !(blocked && crashk_cma_cnt)); } static struct kunit_case kexec_block_test_cases[] = { diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 815588ee6d68b..c7c782fbbfb9a 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -382,7 +382,8 @@ int do_kexec_file_load(int kernel_fd, int initrd_fd, unsigned long cmdline_len, if (!kexec_trylock()) return -EBUSY; - if (!(flags & KEXEC_FILE_UNLOAD) && kexec_blocked()) { + if (!(flags & KEXEC_FILE_UNLOAD) && + ((flags & KEXEC_FILE_ON_CRASH) ? kexec_crash_blocked() : kexec_blocked())) { pr_warn_ratelimited("kexec file load refused by active interlock; first reason: %s\n", kexec_block_reason()); kexec_unlock(); diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index a42b95f65a43f..7f02aa1fd8302 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -26,10 +26,23 @@ const char *notrace kexec_block_reason(void); #ifdef CONFIG_KEXEC_BLOCK_KUNIT_TEST bool kexec_test_load_limits_unlimited(void); #endif +/* + * A blocker keeps a new kernel from reusing memory that a device still reaches + * by DMA. A crash kernel reuses none of it: it runs in the crash kernel + * reservation, which memblock set aside at boot, before any driver could give + * memory to a device. Only CMA-backed crash memory (crashkernel=,cma) is lent + * to the page allocator while the system runs, so blockers still stop a crash + * kernel that has some. + */ +static __always_inline bool notrace kexec_crash_blocked(void) +{ + return kexec_blocked() && crashk_cma_cnt; +} + /* Caller holds the existing NMI-safe kexec lock; no image is dereferenced. */ static __always_inline bool kexec_crash_image_allowed(bool image_loaded) { - return image_loaded && !kexec_blocked(); + return image_loaded && !kexec_crash_blocked(); } #ifdef CONFIG_KEXEC From b6425fc7d83139b5ba710554789cfd5b8795df38 Mon Sep 17 00:00:00 2001 From: Sean Dolbec Date: Sat, 7 Nov 2026 15:06:08 -0500 Subject: [PATCH 072/392] arm64: dts: apple: t8132-j773g: Keep the PMC power state on Once the PMP runs on the M4 Mac mini (J773g), its firmware accesses the PMC block at 0x382800000. Linux treats its PMGR power state, ps_pmc, as an ordinary domain without consumers. J713 disables the domain in its board PMGR policy; J773g has no such policy yet. Mark ps_pmc always-on for J773g. Linux then keeps the domain on. Pavel Shago's J773g tree ended up with the same property in his commit of 2026-10-19, which predates ours. He sets it for all T8132 boards and reports a DCP SError on HDMI unplug. His observation and explanation have not been reproduced here. On one J773g (10-core GPU, system firmware 27.0, OS firmware 26.6.2), using a custom m1n1 stage 2 (build/commit unavailable) and Omarchy (Arch, release unavailable), v38 and v40 differed by this property: v38 had a PMP firmware crash and v40 logged PMP started without it. Across several configurations, 12 logs without the property showed the crash and 11 with it showed PMP started without the crash. Two further boots with PMC enabled left the board dead about 22 s after entry; no kernel log was kept for v34/v35. Attribution to ps_pmc rests on v42/v43 passing with the property, not a matched pair. v41 also changed the PMP register window and still crashed without the property. The driver never logged that an always-on domain was off at boot, which supports an inference that ps_pmc was on at handoff. These boots used pd_ignore_unused and clk_ignore_unused. A power-off was not observed; whether Linux turns the domain off without the property and causes the firmware fault remains inferred, not observed. These boots used our own J773g device tree on an older kernel. This tree's t8132-j773g.dts with this patch has not been booted on hardware. Link: https://github.com/shgpavel/linux-asahi/commit/172d11a1691fd2d8df28520517ee58de5307297e Assisted-by: LLM checkpatch Signed-off-by: Sean Dolbec --- arch/arm64/boot/dts/apple/t8132-j773g.dts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts index 89d2514815914..c7878c5a1ca31 100644 --- a/arch/arm64/boot/dts/apple/t8132-j773g.dts +++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts @@ -43,6 +43,15 @@ power-domains = <&ps_dispext0_fe>, <&ps_dptx_phy>; }; +/* + * Without this, the PMP firmware faults in the PMC block (0x382800000) + * shortly after it starts on this board. J713 keeps ps_pmc out of Linux + * in its PMGR policy; this board has none yet. + */ +&ps_pmc { + apple,always-on; +}; + /* * The AOP audio component always runs a microphone stream, and this Mac * has no microphones (ADT iop-audio-controller has no audio-hp) and no From 8f905ae00b1b3612bb6d6c7b5084d1e02d276c4a Mon Sep 17 00:00:00 2001 From: Sean Dolbec Date: Sat, 7 Nov 2026 15:35:45 -0500 Subject: [PATCH 073/392] phy: apple: atc: use the common USB2 host field for a fixed hub T8132's ordinary Type-C ports use its alternate USB2 host-mode field. The fixed-hub path on the M4 Mac mini instead uses the common host field that the other supported SoCs use. Select the field from the fixed-hub topology. Ordinary T8132 Type-C ports keep their current field, while a T8132 PHY connected to a fixed hub takes the common field. SoCs without an alternate field are unchanged. Tested on one J773g with the two front USB-C ports, which sit behind an on-board USB2 hub on ATC2. With this field, together with the hub child and PHY-to-controller graph link in the board tree, the hub enumerates at 480 Mb/s, a keyboard delivers key events, and unplug/replug succeeds. That boot proves the combination works, but does not isolate whether this field is necessary. The change is derived from the U1 bring-up patch and narrowed to the fixed-hub path. Ordinary T8132 Type-C ports are unchanged. Assisted-by: LLM checkpatch Signed-off-by: Sean Dolbec --- drivers/phy/apple/atc.c | 25 +++++++++++++------ .../selftests/drivers/gpu/apple-dcp/common.py | 9 +++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 6ed4bc2295d7b..33507b68de52e 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -925,6 +925,19 @@ static bool atcphy_usb2_only(const struct apple_atcphy *atcphy) return atcphy->hw->usb2_only && !atcphy->t8132_ss; } +/* + * The USB2 PHY's host-mode field. T8132's ordinary Type-C ports use bits 6:4, + * while its fixed-hub path uses bits 14:12 (plus the VBUSDET force bits 1:0 + * that this driver programs separately). + */ +static u32 atcphy_usb2_sig_host(const struct apple_atcphy *atcphy) +{ + if (atcphy->hw->usb2phy_sig_host && + !atcphy->fixed_usb2) + return atcphy->hw->usb2phy_sig_host; + return USB2PHY_SIG_HOST; +} + static const struct { const struct atcphy_mode_configuration normal; const struct atcphy_mode_configuration swapped; @@ -4261,12 +4274,10 @@ static int atcphy_usb2_set_mode(struct phy *phy, enum phy_mode mode, int submode switch (mode) { case PHY_MODE_USB_HOST: - set32(atcphy->regs.usb2phy + USB2PHY_SIG, - atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy_usb2_sig_host(atcphy)); break; case PHY_MODE_USB_DEVICE: - clear32(atcphy->regs.usb2phy + USB2PHY_SIG, - atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + clear32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy_usb2_sig_host(atcphy)); break; default: return -EINVAL; @@ -4310,8 +4321,7 @@ static int atcphy_usb2_init(struct phy *phy) * powered off as well the whole block is off; bring it back in the mode * the Type-C mux last asked for, USB2 until a cable event says otherwise. */ - set32(atcphy->regs.usb2phy + USB2PHY_SIG, - atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy_usb2_sig_host(atcphy)); if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) return atcphy_configure(atcphy, atcphy->typec_mode); @@ -6246,8 +6256,7 @@ static int atcphy_probe_finalize(struct apple_atcphy *atcphy) * in reset. */ if (atcphy->fixed_usb2) { - set32(atcphy->regs.usb2phy + USB2PHY_SIG, - atcphy->hw->usb2phy_sig_host ?: USB2PHY_SIG_HOST); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, atcphy_usb2_sig_host(atcphy)); ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB2); if (ret) return dev_err_probe(atcphy->dev, ret, "Failed to bring up USB2\n"); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py index c15ae40d30b8d..6c140662b6967 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -123,10 +123,15 @@ def atc_cio_helpers(source): The mux and PIPE functions call these; older sources have neither: ''. """ result = '' - for name in ('atcphy_mode_is_cio', 'atcphy_pipe_state_for'): + for name in ('atcphy_mode_is_cio', 'atcphy_pipe_state_for', + 'atcphy_usb2_sig_host'): marker = re.search(r'^static [^;{}()]+\b' + name + r'\([^;{}]*\)\s*\{', source, re.M) if marker: - result += block(source, marker.start()) + '\n' + helper = block(source, marker.start()) + if name == 'atcphy_usb2_sig_host': + helper = helper.replace('static u32', + 'static __attribute__((unused)) u32', 1) + result += helper + '\n' return result From f6f0bd033746b2ae44dc040956a2bedfff2fa393 Mon Sep 17 00:00:00 2001 From: Sean Dolbec Date: Sat, 7 Nov 2026 15:35:46 -0500 Subject: [PATCH 074/392] arm64: dts: apple: t8132-j773g: describe the front USB hub The M4 Mac mini two front USB-C ports hang off an on-board USB2 hub on ATC2 controller (ADT usb-drd2, hub 05ac:800b on raw root port 1), with no Type-C port controller. The board tree enables the controller as a host but gives it no hub child: dwc3-apple then waits for a role-switch event that nothing on this port ever sends, and the PHY only powers its USB2 path on a cable event, so the controller never starts. Describe the hub as the controller fixed child, as t8140-j700.dts does for its VL122. Link the PHY USB3 port to the controller SuperSpeed port: the PHY driver follows that graph to find the hub child and keeps its USB2 path up from probe on. No SuperSpeed data path is described, so the hub runs at high speed only. Hog the hub reset and SPI-flash output-enable lines high (ADT usb3hub_rst GPIO 198, usb3hub_flash_oe GPIO 101). Both are high under the boot loader and macOS, and a Linux boot that drove both high had a working hub. Whether the flash line must stay high after the hub has loaded its firmware is not known; keeping it as macOS leaves it is the conservative choice. Tested on one J773g (macOS 26.6.2 firmware): with this description and the PHY host-field change, the hub enumerates at 480 Mb/s on the ATC2 controller, a keyboard behind it reports press and release events and survives an unplug/replug. That boot shows the combination works; the hub child, graph link and GPIO hogs have not been removed one at a time, and the tree without the hub child has not been booted on its own. Assisted-by: LLM checkpatch Signed-off-by: Sean Dolbec --- arch/arm64/boot/dts/apple/t8132-j773g.dts | 61 +++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts index 89d2514815914..4be114605cae4 100644 --- a/arch/arm64/boot/dts/apple/t8132-j773g.dts +++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts @@ -166,12 +166,31 @@ }; /* - * The two front USB-C ports: an on-board hub behind ATC2 (ADT usb-drd2, - * hub firmware on spi3), with no port controller. The PHY only enters - * USB3 mode from a Type-C mode switch, so the hub runs at high speed. + * The two front USB-C ports: an on-board USB2 hub (Apple 05ac:800b, firmware + * on spi3) on raw USB2 root port 1 of ATC2's controller (ADT usb-drd2), with + * no port controller. The PHY only enters USB3 mode from a Type-C mode + * switch, so the hub runs at high speed. The hub child tells dwc3-apple the + * controller is a host from probe on; the graph link from the PHY's USB3 port + * to the controller's SuperSpeed port is how the PHY finds that fixed hub + * and keeps its USB2 path up without a cable event (no SuperSpeed data path + * is described). The hub's reset and SPI-flash output-enable lines (ADT + * usb3hub_rst, usb3hub_flash_oe) are AP GPIOs; the boot loader leaves both + * high, as does macOS, and the hub runs with them high. */ &atcphy2 { status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + atcphy2_usb: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; }; &dwc3_2_dart_0 { @@ -184,7 +203,43 @@ &dwc3_2 { dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; status = "okay"; + + hub@1 { + compatible = "usb5ac,800b"; + reg = <1>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* Raw xHCI port 1 is USB2; raw port 2 is SuperSpeed. */ + port@2 { + reg = <2>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb>; + }; + }; + }; +}; + +&pinctrl_ap { + usb3hub-flash-oe-hog { + gpio-hog; + gpios = <101 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb3hub-flash-oe"; + }; + + usb3hub-reset-hog { + gpio-hog; + gpios = <198 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb3hub-reset"; + }; }; &wifi0 { From b695a33695b77c6b3ff743fffd939f3304e9688a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:03:19 +1100 Subject: [PATCH 075/392] nvme: apple: restart pre-M4 ANS left running by a crashed kernel A crashed kernel starts the crash kernel without stopping ANS, so on M1/M2-class machines the crash kernel finds the coprocessor still running the old kernel's RTKit session. The driver then only sends a wake, but a running session does not repeat HELLO and the endpoint map, so the handshake cannot complete and the crash kernel has no disk to save the dump on. In a crash kernel, when the coprocessor is found running on the first reset, take the old session over, disable the controller, shut RTKit down and clear CPU_CONTROL.RUN, so that the normal cold path resets and boots ANS afresh. If the old session does not answer, forget it and let the PMGR reset stop it. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/nvme/host/apple.c | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 150c8079fcfb9..f843f6d4a7939 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1612,6 +1613,38 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv) } } +/* + * Stop the pre-M4 ANS session a crashed kernel left running, so that the + * reset path restarts the coprocessor from scratch. Each step is best effort: + * if the old session does not answer, the PMGR reset still stops it. + */ +static void apple_nvme_stop_crashed_session(struct apple_nvme *anv) +{ + int ret; + + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + anv->ctrl.ctrl_config = readl(anv->mmio_nvme + NVME_REG_CC); + dev_info(anv->dev, + "crash kernel: ANS still running (CC=%#x CSTS=%#x), restarting it\n", + anv->ctrl.ctrl_config, readl(anv->mmio_nvme + NVME_REG_CSTS)); + + ret = apple_rtkit_adopt_running(anv->rtk); + if (!ret && (anv->ctrl.ctrl_config & NVME_CC_ENABLE)) + ret = nvme_disable_ctrl(&anv->ctrl, false); + if (!ret) + ret = apple_rtkit_shutdown(anv->rtk); + if (ret) { + dev_warn(anv->dev, + "crash kernel: old ANS session did not stop (%d), resetting it\n", + ret); + /* Forget the old session; the reset below discards it. */ + apple_rtkit_reinit(anv->rtk); + } + + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + anv->ctrl.ctrl_config = 0; +} + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1678,6 +1711,17 @@ static void apple_nvme_reset_work(struct work_struct *work) goto rtkit_ready; } + /* + * A crashed kernel never stops ANS, so on pre-M4 the crash kernel finds + * the coprocessor still running the old kernel's RTKit session. That + * session does not repeat HELLO and EPMAP, so a wake cannot work. Take + * the session over, stop it and restart ANS through the cold path below. + */ + if (inherited && !anv->hw->needs_ioq_registers && is_kdump_kernel() && + (readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN)) + apple_nvme_stop_crashed_session(anv); + /* * RTKit must be shut down cleanly for the (soft)-reset to work, except * on post-M4: that firmware stays live and only its queue DMA is From c2824e0fd8384e919cf18c25451aa604f9534aff Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:06:14 +1100 Subject: [PATCH 076/392] watchdog: apple: restart a stalled crash kernel When the crashed kernel had the watchdog running, the crash kernel finds it enabled, marks it running and the watchdog core pings it until userspace opens it, which a crash kernel never does. A crash kernel that then stalls hangs the machine for good: on an M2 a stalled crash kernel ran for minutes past a 170 s watchdog that the crashed kernel had armed. When the crashed kernel had no watchdog running, nothing restarts a stalled crash kernel either. In a crash kernel, arm the watchdog at its longest timeout (about 178 s at 24 MHz) and do not mark it running, so the core leaves it unpinged. Saving a dump took 7 to 25 s on a J700 with crashkernel=320M; a crash kernel that has not restarted the machine by the timeout is stuck. Userspace that opens the watchdog takes it over as usual. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/watchdog/apple_wdt.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 46a00a7993a6e..3c867a3ce636e 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -185,12 +186,25 @@ static int apple_wdt_probe(struct platform_device *pdev) wdt->wdd.timeout = APPLE_WDT_TIMEOUT_DEFAULT; wdt_ctrl = readl_relaxed(wdt->regs + APPLE_WDT_WD1_CTRL); - if (wdt_ctrl & APPLE_WDT_CTRL_RESET_EN) + if ((wdt_ctrl & APPLE_WDT_CTRL_RESET_EN) && !is_kdump_kernel()) set_bit(WDOG_HW_RUNNING, &wdt->wdd.status); watchdog_set_nowayout(&wdt->wdd, WATCHDOG_NOWAYOUT); watchdog_init_timeout(&wdt->wdd, 0, dev); apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout); + + /* + * A crash kernel only saves the dump and restarts. If it stalls, the + * machine must still restart rather than hang: arm the watchdog at its + * longest timeout and leave it unpinged, unless userspace opens it. + */ + if (is_kdump_kernel()) { + apple_wdt_set_timeout(&wdt->wdd, + wdt->wdd.max_hw_heartbeat_ms / 1000); + apple_wdt_start(&wdt->wdd); + dev_info(dev, "crash kernel: restarting in %u s unless done\n", + wdt->wdd.timeout); + } watchdog_stop_on_unregister(&wdt->wdd); watchdog_set_restart_priority(&wdt->wdd, 128); From d4b3b4a8bac24539d02e78b69169dc29365a2386 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Sun, 8 Nov 2026 02:10:01 -0300 Subject: [PATCH 077/392] drm/asahi: g17: keep dynamic kernel objects below the MMIO aperture The upper window for dynamic kernel objects ran from KERNEL_OFFSET to the end of the kernel range, past the firmware MMIO aperture at 0xfffffc2180000000. The firmware gives the addresses from the aperture on a fixed meaning: the register windows, and the gart ranges HwData names. An object allocated there would take an address the firmware treats as something else. Put the bounds in one helper, dynamic::upper(), and use it in every place that allocated from that window. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/capacity.rs | 3 +-- drivers/gpu/drm/asahi/g17/initdata.rs | 2 +- drivers/gpu/drm/asahi/g17/kick.rs | 3 +-- drivers/gpu/drm/asahi/g17/mod.rs | 4 +--- drivers/gpu/drm/asahi/g17/object.rs | 3 +-- drivers/gpu/drm/asahi/hw/t8140.rs | 6 ++++++ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 405bb65a8eef6..82a3d1165f7a0 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -71,8 +71,7 @@ impl kernel::debugfs::Writer for View { writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; let (fenced, overruns) = uat.execution_contexts_fenced(super::channel::control_witnessed()); writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?; - let mut upper = uat.geometry().kernel_range(); - upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + let upper = crate::hw::t8140::dynamic::upper(uat.geometry().kernel_range()); for (name, vm, range) in [ ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), ("dynamic_upper_bytes", uat.kernel_vm(), upper), diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 56e7335a23bd2..c0337c510e56b 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -858,7 +858,7 @@ impl InitData { lower: uat.kernel_lower_vm(), render, base: range.start, - dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + dynamic: t8140::dynamic::upper(range.clone()), clusters, }; if variant.graph == super::fw::variant::InitGraph::V25G83 { diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 6499cae106309..69beeac76ed95 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -538,8 +538,7 @@ impl Queue { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; - let mut range = alloc.uat.geometry().kernel_range(); - range.start += t8140::dynamic::KERNEL_OFFSET; + let range = t8140::dynamic::upper(alloc.uat.geometry().kernel_range()); let high = low.alias_in( alloc.uat.kernel_vm(), range, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ffb0c20d0267f..23877de44ac39 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1823,12 +1823,10 @@ impl gpu::Gpu for Gpu { let mapping = { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; bo.map_range_into_range( firmware.uat.kernel_vm(), range, - addresses, + hw::t8140::dynamic::upper(firmware.uat.geometry().kernel_range()), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index aa30da6fcb41e..e60e4f376dd83 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -338,8 +338,7 @@ impl Allocator<'_> { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let mut range = self.uat.geometry().kernel_range(); - range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + let range = crate::hw::t8140::dynamic::upper(self.uat.geometry().kernel_range()); KernelObject::new( self.dev, self.uat.kernel_vm(), diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 02381c2aae344..7f2b81cb6a14b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -192,6 +192,12 @@ pub(crate) mod dynamic { /// Canonical low mappings shared by hardware queues and their clients. /// Client-private allocations start after this arena. pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; + + /// Dynamic kernel objects in the upper window: from `KERNEL_OFFSET` up to the firmware + /// MMIO aperture. The firmware gives the addresses from there on a fixed meaning. + pub(crate) const fn upper(kernel: core::ops::Range) -> core::ops::Range { + kernel.start + KERNEL_OFFSET..super::MMIO_APERTURE + } } /// Fixed addresses in the firmware's context-0 and context-1 address spaces. From b6094a22268844d5aa964dd5d89839a81d825281 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Sun, 8 Nov 2026 02:11:25 -0300 Subject: [PATCH 078/392] drm/asahi: g17: let the firmware write render end timestamps Render end timestamps are written by the host at retirement: the kernel converts the pass's internal stamps to nanoseconds and stores them into the user's timestamp buffer. The firmware can do that itself. macOS aliases timestamp buffers into the gart range HwData.unk_28 names ([0xfffffc2181400000, +64 MiB)) and puts window addresses into each render's second stamp pair, TA +0x90e (vertex start/end) and 3D +0x21a8 (fragment start/end). The firmware then writes those stamps in nanoseconds, equal to the host's conversion of its own internal stamps. Bind timestamp buffers into that window, firmware-only like the other alias, and hand the firmware the window address of each render's fragment end destination. The other three stay host-written: - Start stamps: the firmware latches a start stamp before the command's barriers resolve, so a reset of the same query ordered before the command can still land on top of it. - The vertex end: handed to the firmware, it does not hold the host's value by the end of the wait below. Astro Bot under KytyPS5 with hk's frame profiler, which also stamps every render's vertex end, sent 312,088 stamps through the window; exactly half (156,044) fell back to the host, each after the full wait. Without the profiler none did. At retirement the kernel reads the fragment end destination, without writing, until it holds the expected value (at most 200 us), and does not write it. A host write, or a firmware write landing after the fence, could undo a reset userspace may make once the fence has signalled. A value that does not land in time is written by the host, still before the fence. On an M4 Pro every fragment end stamp had landed by the first read. Compute commands, end-stamp copies and buffers that do not fit in the window keep host-written stamps too. asahi-g17/capacity counts kept and host-written window stamps. No uAPI change. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/capacity.rs | 2 + drivers/gpu/drm/asahi/g17/completion.rs | 140 +++++++++++++++--- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 2 +- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 11 ++ drivers/gpu/drm/asahi/g17/mod.rs | 41 +++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/hw/t8140.rs | 5 + drivers/gpu/drm/asahi/mmu.rs | 11 ++ drivers/gpu/drm/asahi/timestamp.rs | 33 +++++ 10 files changed, 216 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 82a3d1165f7a0..afc43914ff963 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -71,6 +71,8 @@ impl kernel::debugfs::Writer for View { writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; let (fenced, overruns) = uat.execution_contexts_fenced(super::channel::control_witnessed()); writeln!(f, "execution_contexts_fenced free_waiting={} overruns={}", fenced, overruns)?; + let [kept, fallbacks] = super::completion::window_counters(); + writeln!(f, "timestamp_window firmware={} host_fallbacks={}", kept, fallbacks)?; let upper = crate::hw::t8140::dynamic::upper(uat.geometry().kernel_range()); for (name, vm, range) in [ ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 0f027e7e06828..5116e4a3b08d0 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -19,7 +19,7 @@ use crate::{ hw::t8140::{dynamic, resources as cfg}, mmu, }; -use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; +use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU64, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, new_mutex, @@ -35,6 +35,8 @@ struct Destination { // The buffer keeps its GPU alias and its CPU mapping alive. buffer: Arc, offset: usize, + /// The destination's address in the firmware timestamp window, if its buffer is mapped there. + window_va: Option, } impl Destination { @@ -55,25 +57,74 @@ impl Destination { if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } + let window_va = buffer.window_va(offset); let offset = buffer.timestamp_offset(offset)?; - Ok(Some(Self { buffer, offset })) + Ok(Some(Self { buffer, offset, window_va })) } fn write(&self, ticks: u64) { - const NSEC_PER_SEC: u64 = 1_000_000_000; - let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) - .wrapping_mul(NSEC_PER_SEC) - .wrapping_add( - ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, - ); - self.buffer.write_timestamp(self.offset, ns); + self.buffer.write_timestamp(self.offset, ticks_to_ns(ticks)); barrier(Ordering::Release); } + + /// An end stamp the firmware writes itself, through the window, as the same nanoseconds the + /// host would write. Wait for it without writing: a host write, or a firmware write landing + /// after the fence, could undo a reset userspace may make once the fence has signalled. If + /// it does not land in time, the host writes the value, still before the fence. + fn write_window(&self, ticks: u64) { + let ns = ticks_to_ns(ticks); + let mut polls = 0; + while self.buffer.read_timestamp(self.offset) != Some(ns) { + if polls == WINDOW_POLLS { + WINDOW_FALLBACKS.fetch_add(1, Ordering::Relaxed); + self.write(ticks); + return; + } + kernel::time::delay::udelay(kernel::time::Delta::from_micros(WINDOW_POLL_US)); + polls += 1; + } + WINDOW_KEPT.fetch_add(1, Ordering::Relaxed); + } +} + +fn ticks_to_ns(ticks: u64) -> u64 { + const NSEC_PER_SEC: u64 = 1_000_000_000; + (ticks / cfg::COMMAND_TIMESTAMP_HZ) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ) +} + +/// The firmware writes the stamp ~20 us after the pass ends; macOS saw it up to 36 us after the +/// event ring. In practice it has always landed by retirement's first read. +const WINDOW_POLLS: u32 = 40; +const WINDOW_POLL_US: i64 = 5; +static WINDOW_KEPT: AtomicU64 = AtomicU64::new(0); +static WINDOW_FALLBACKS: AtomicU64 = AtomicU64::new(0); + +/// `capacity`: firmware-written end stamps kept, and those the host had to write. +pub(crate) fn window_counters() -> [u64; 2] { + [WINDOW_KEPT.load(Ordering::Relaxed), WINDOW_FALLBACKS.load(Ordering::Relaxed)] +} + +/// Only the fragment end stamp goes through the window (slots: vertex start, vertex end, +/// fragment start, fragment end). The firmware latches a start stamp before the command's +/// barriers resolve, so a reset of the same query ordered before the command can still land +/// on top of it. A vertex end handed to the firmware never held the host's value by the end +/// of the wait, so it would only cost every render that stamps it the full wait. +const fn window_slot(index: usize) -> bool { + index == 3 } /// Timestamp destinations resolved and retained before any command is enqueued. Dropping /// one releases only a buffer reference; the buffer defers its own mapping release. -pub(crate) struct Destinations([Option; 4]); +pub(crate) struct Destinations { + stamps: [Option; 4], + /// A render: the firmware gets its fragment end destination through the window. A compute + /// command's only stamp pair stays kernel-private (recovery trusts it). + render: bool, + /// The firmware was handed window destinations ([`Self::window_vas`]). + handed: AtomicBool, +} impl Destinations { pub(crate) fn resolve( @@ -88,15 +139,48 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - Ok(Self(destinations)) + Ok(Self { + stamps: destinations, + render: timestamps.len() == 2, + handed: AtomicBool::new(false), + }) + } + + /// Window addresses for a render's vertex and fragment start/end destinations: the fragment + /// end's if it is mapped in the window, 0 for the others. + fn window_vas(&self) -> [u64; 4] { + if !self.render { + return [0; 4]; + } + let vas: [u64; 4] = core::array::from_fn(|i| { + let destination = self.stamps[i].as_ref().filter(|_| window_slot(i)); + destination.and_then(|d| d.window_va).unwrap_or(0) + }); + if vas.iter().any(|&va| va != 0) { + self.handed.store(true, Ordering::Release); + } + vas } pub(crate) fn has_outputs(&self) -> bool { - self.0.iter().any(Option::is_some) + self.stamps.iter().any(Option::is_some) } fn iter(&self) -> core::slice::Iter<'_, Option> { - self.0.iter() + self.stamps.iter() + } + + fn write(&self, ticks: [u64; 4]) { + let handed = self.handed.load(Ordering::Acquire); + for (index, (destination, ticks)) in self.stamps.iter().zip(ticks).enumerate() { + match destination { + Some(destination) if handed && window_slot(index) && destination.window_va.is_some() => { + destination.write_window(ticks) + } + Some(destination) => destination.write(ticks), + None => {} + } + } } } @@ -110,13 +194,7 @@ struct Output { impl Output { fn publish(&self, result: Result<[u64; 4]>) { match result { - Ok(ticks) => { - for (destination, ticks) in self.destinations.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks); - } - } - } + Ok(ticks) => self.destinations.write(ticks), Err(error) => self.fence.set_error(error), } self.fence.signal(); @@ -428,6 +506,11 @@ impl Completion { .any(Option::is_some) } + /// The firmware window addresses of a render's vertex and fragment start/end stamps. + pub(crate) fn window_vas(&self) -> [u64; 4] { + self.output.output().destinations.window_vas() + } + pub(crate) fn compute_timestamp_va(&self) -> Result { Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } @@ -770,3 +853,20 @@ impl Deferred { } } } + +#[kernel::macros::kunit_tests(asahi_g17_timestamp_window)] +mod window_tests { + use super::*; + + #[test] + fn only_the_fragment_end_goes_through_the_window() { + assert!(!window_slot(0) && !window_slot(1) && !window_slot(2) && window_slot(3)); + } + + #[test] + fn ticks_convert_to_nanoseconds_at_24_mhz() { + assert_eq!(ticks_to_ns(24_000_000), 1_000_000_000); + assert_eq!(ticks_to_ns(24), 1_000); + assert_eq!(ticks_to_ns(24_000_025), 1_000_001_041); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 387cebb28a368..6cf84c744eebd 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1097,7 +1097,7 @@ impl HwData { self.unk_10 = 0x10_0000_0000; self.unk_18 = 0x10_0000_0000; self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = 0xffff_fc21_8140_0000; + self.unk_28 = crate::hw::t8140::dynamic::TIMESTAMP_WINDOW.start; self.unk_e0 = Self::UNK_E0; self.unk_2d8 = Self::UNK_2D8; self.unk_3e0 = Self::UNK_3E0; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index 863a90fc52a34..af0cf7d55fe72 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -291,7 +291,7 @@ impl HwData25G83 { self.unk_10 = 0x10_0000_0000; self.unk_18 = 0x10_0000_0000; self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = 0xffff_fc21_8140_0000; + self.unk_28 = crate::hw::t8140::dynamic::TIMESTAMP_WINDOW.start; let mut used = [false; REGISTER_MAP_SLOTS]; for window in args.windows { diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 8fcc25500ab2d..d9f81c0dadd7d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -334,6 +334,9 @@ pub(crate) struct RenderResources { pub(crate) aux_fb_va: u64, /// GPU address of the pass's start/end timestamp pair. pub(crate) timestamp_va: u64, + /// Firmware window addresses of the user's vertex and fragment start/end stamps + /// (the second pair at TA +0x90e and 3D +0x21a8), 0 for a stamp the firmware does not write. + pub(crate) timestamp_window: [u64; 4], /// Object IDs of the tiling stage. pub(crate) ta_ids: ObjectIds, /// Object IDs of the fragment stage. @@ -1112,6 +1115,8 @@ static_assert!(core::mem::offset_of!(TaDescriptor, predecessor_id) == 0x86e); static_assert!(core::mem::offset_of!(TaDescriptor, stamp_va) == 0x8a6); static_assert!(core::mem::offset_of!(TaDescriptor, completion) == 0x8c6); static_assert!(core::mem::offset_of!(TaDescriptor, timestamp_start_va) == 0x8fe); +// The firmware-sampled vertex start/end pair (macOS fills it with timestamp-window addresses). +static_assert!(core::mem::offset_of!(TaDescriptor, unk_90e) == 0x90e); static_assert!(core::mem::offset_of!(TaDescriptor, free_list_control_va) == 0x934); static_assert!(core::mem::offset_of!(TaDescriptor, status_va) == 0x945); @@ -1183,6 +1188,8 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; + self.unk_90e[..8].copy_from_slice(&resources.timestamp_window[0].to_le_bytes()); + self.unk_90e[8..16].copy_from_slice(&resources.timestamp_window[1].to_le_bytes()); self.unk_932 = match args.programs == RegisterPrograms::V25G83 { true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), false => Self::UNK_932, @@ -1397,6 +1404,8 @@ static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_2104) == 0x2104); static_assert!(core::mem::offset_of!(FragmentDescriptor, stamp_va) == 0x2140); static_assert!(core::mem::offset_of!(FragmentDescriptor, completion) == 0x2160); static_assert!(core::mem::offset_of!(FragmentDescriptor, timestamp_start_va) == 0x2198); +// The firmware-sampled fragment start/end pair. +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_21a8) == 0x21a8); static_assert!(core::mem::offset_of!(FragmentDescriptor, free_list_control_va) == 0x21ce); static_assert!(core::mem::offset_of!(FragmentDescriptor, status_va) == 0x21d7); static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_222d) == 0x222d); @@ -1545,6 +1554,8 @@ impl FragmentSlot { desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; + desc.unk_21a8[..8].copy_from_slice(&resources.timestamp_window[2].to_le_bytes()); + desc.unk_21a8[8..16].copy_from_slice(&resources.timestamp_window[3].to_le_bytes()); desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), false => Self::UNK_21CC, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 23877de44ac39..24e9e053b2daf 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1820,20 +1820,41 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let mapping = { + let (mapping, window) = { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; - bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - hw::t8140::dynamic::upper(firmware.uat.geometry().kernel_range()), - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )? + // Firmware-only, like the other alias: only the firmware writes these stamps. + let windowed = bo + .map_range_into_range( + firmware.uat.kernel_vm(), + range.clone(), + hw::t8140::dynamic::TIMESTAMP_WINDOW, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + .ok(); + match windowed { + Some(mapping) => (mapping, true), + // A full window: this buffer's stamps are all host-written. + None => ( + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + hw::t8140::dynamic::upper(firmware.uat.geometry().kernel_range()), + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )?, + false, + ), + } }; // The buffer's CPU mapping takes the reservation lock: not under the device mutex. - crate::timestamp::Buffer::firmware(mapping) + match window { + true => crate::timestamp::Buffer::firmware_window(mapping), + false => crate::timestamp::Buffer::firmware(mapping), + } } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 76447a297c5ec..6d533cd1b199f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -440,6 +440,7 @@ impl Pair { fragment_status_fw_va: fragment_fw, aux_fb_va: auxiliary, timestamp_va: timestamp, + timestamp_window: packet.completion.window_vas(), ta_ids: lifecycle[0], fragment_ids: lifecycle[1], unk_2104: self.descriptor_flags[0], diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 7f2b81cb6a14b..7a9591ea65aba 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -193,6 +193,11 @@ pub(crate) mod dynamic { /// Client-private allocations start after this arena. pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; + /// The firmware timestamp window (gart range 11, `HwData.unk_28`): user timestamp buffers + /// are aliased here so that the firmware writes their stamps itself. + pub(crate) const TIMESTAMP_WINDOW: core::ops::Range = + 0xffff_fc21_8140_0000..0xffff_fc21_8540_0000; + /// Dynamic kernel objects in the upper window: from `KERNEL_OFFSET` up to the firmware /// MMIO aperture. The firmware gives the addresses from there on a fixed meaning. pub(crate) const fn upper(kernel: core::ops::Range) -> core::ops::Range { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4cc9bf4223faf..366e34ff8d9b4 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1372,6 +1372,17 @@ impl TimestampVmap { self.0.owner().size() } + /// Reads the naturally aligned timestamp at byte `offset` of the GEM object. + pub(crate) fn read(&self, offset: usize) -> Option { + let end = offset.checked_add(size_of::())?; + if end > self.size() || offset % size_of::() != 0 { + return None; + } + // SAFETY: The complete aligned extent was checked above, inside this pinned vmap. + let source = unsafe { &*self.0.addr.cast::().add(offset).cast::() }; + Some(u64::from_le(source.load(Ordering::Acquire))) + } + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete /// atomic store; byte stores support unaligned destinations without creating mutable /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index fd3de19a96d29..2a18ebbeb5d9c 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -18,6 +18,8 @@ use kernel::{uapi, xarray}; pub(crate) struct FirmwareBuffer { mapping: Arc, cpu: mmu::TimestampVmap, + /// Mapped in the firmware's timestamp window: the firmware may write stamps itself. + window: bool, } /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. @@ -37,13 +39,44 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { + Self::new_firmware(mapping, false) + } + + /// A buffer aliased in the firmware's timestamp window. + pub(crate) fn firmware_window(mapping: mmu::KernelMapping) -> Result { + Self::new_firmware(mapping, true) + } + + fn new_firmware(mapping: mmu::KernelMapping, window: bool) -> Result { let cpu = mapping.timestamp_vmap()?; Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { mapping: Arc::new(mapping, GFP_KERNEL)?, cpu, + window, })?)) } + /// The window address of the timestamp at `offset`, for a window buffer and an aligned + /// destination that lies wholly inside the mapping. + pub(crate) fn window_va(&self, offset: usize) -> Option { + match self { + Self::Firmware(buffer) if buffer.window && offset % size_of::() == 0 => { + let end = offset.checked_add(size_of::())?; + (end <= buffer.mapping.size()).then(|| buffer.mapping.iova() + offset as u64) + } + _ => None, + } + } + + /// Reads the aligned timestamp at an object offset returned by [`Self::timestamp_offset`]. + pub(crate) fn read_timestamp(&self, offset: usize) -> Option { + match self { + Self::Firmware(buffer) => buffer.cpu.read(offset), + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] + Self::Cpu { .. } => None, + } + } + /// Retain a CPU-written byte range without publishing any firmware address. #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { From 29a0eaef4d9775b26ca8e9f368137cc0111c2cf2 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:22:41 +0200 Subject: [PATCH 079/392] dt-bindings: arm: apple: Name the J704 MacBook Pro (14-inch, M5, 2025) The J704 entry names the board only by its code. It is the MacBook Pro (14-inch, M5, 2025), as its device tree's model now says. Signed-off-by: Dmitriy Assisted-by: LLM --- Documentation/devicetree/bindings/arm/apple.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index f5acf01039a14..691716f025522 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -366,7 +366,7 @@ properties: - description: Apple M5 SoC based platforms items: - enum: - - apple,j704 # MacBook Pro (J704) + - apple,j704 # MacBook Pro (14-inch, M5, 2025) - apple,j813 # MacBook Air (13-inch, M5, 2026) - apple,j815 # MacBook Air (J815) - const: apple,t8142 From 9b776acc0d5044e584505ba0ad57d42351343175 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:33:09 +0200 Subject: [PATCH 080/392] dt-bindings: watchdog: apple,wdt: Add the T8142 watchdog The M5 (T8142) watchdog is the T8103 one, as on the other recent SoCs. The J704 and J813 device trees use apple,t8142-wdt. Signed-off-by: Dmitriy Assisted-by: LLM --- Documentation/devicetree/bindings/watchdog/apple,wdt.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml index d09fb81bd506b..e321ee28733d1 100644 --- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml @@ -25,6 +25,7 @@ properties: - apple,t8122-wdt - apple,t8132-wdt - apple,t8140-wdt + - apple,t8142-wdt - const: apple,t8103-wdt - items: - enum: From 2f018d8692d5e30280bc10ad83ccbb7e186676fc Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:33:09 +0200 Subject: [PATCH 081/392] dt-bindings: spi: apple,qspi: Add the T8142 QSPI The M5 (T8142) QSPI controller is the T8140 one, as on T6050 and T8152. The J704 and J813 device trees use apple,t8142-qspi. Signed-off-by: Dmitriy Assisted-by: LLM --- Documentation/devicetree/bindings/spi/apple,qspi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 2cf6f7286efd3..cf140aaab29bd 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -34,6 +34,7 @@ properties: - items: - enum: - apple,t6050-qspi + - apple,t8142-qspi - apple,t8152-qspi - const: apple,t8140-qspi From e5cd689b79d9062bd372f177cc133f1a28f0d6d8 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:33:09 +0200 Subject: [PATCH 082/392] dt-bindings: soc: apple: apple,pmp: Add the T8142 PMP The M5 (T8142) PMP takes the T8140 firmware interface, so it falls back to apple,t8140-pmp-v2. The J704 and J813 device trees use that pair. Signed-off-by: Dmitriy Assisted-by: LLM --- Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index 927dd53f8909e..bcbd0ecf4568a 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -32,6 +32,9 @@ properties: - apple,t8132-pmp-v2 - apple,t8140-pmp-v2 - const: apple,t6000-pmp-v2 + - items: + - const: apple,t8142-pmp-v2 + - const: apple,t8140-pmp-v2 - items: - const: apple,t8103-pmp - const: apple,pmp From d84beb5d225310b050d70aee8782ce9dc882d3a2 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:33:09 +0200 Subject: [PATCH 083/392] dt-bindings: display: apple,display-subsystem: Add T8142 The M5 (T8142) display subsystem takes the same description as the T8140 one. The J704 and J813 device trees use apple,t8142-display-subsystem with apple,display-subsystem. Signed-off-by: Dmitriy Assisted-by: LLM --- .../devicetree/bindings/display/apple,display-subsystem.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml index 0d3e74d6a804c..be4e56cb2a39f 100644 --- a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -23,7 +23,9 @@ properties: - const: apple,t8132-display-diagnostics deprecated: true - items: - - const: apple,t8140-display-subsystem + - enum: + - apple,t8140-display-subsystem + - apple,t8142-display-subsystem - const: apple,display-subsystem - items: - const: apple,t8132-display-subsystem From 8e90594cb08c59d2a0ae9273838536b3cd77f18d Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:33:10 +0200 Subject: [PATCH 084/392] arm64: dts: apple: t8142: Name the MTP DART's address filter region The MTP DART's second register region is its DAPF (ADT dart-mtp instance DAPF). The DART binding requires reg-names once a second region is listed. Signed-off-by: Dmitriy Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8142.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8142.dtsi b/arch/arm64/boot/dts/apple/t8142.dtsi index 9bf8a297f31be..13cae26f21d96 100644 --- a/arch/arm64/boot/dts/apple/t8142.dtsi +++ b/arch/arm64/boot/dts/apple/t8142.dtsi @@ -213,6 +213,7 @@ compatible = "apple,t8142-dart", "apple,t8110-dart"; reg = <0x3 0x94800000 0x0 0xc000>, <0x3 0x94810000 0x0 0x4000>; + reg-names = "dart", "dapf"; interrupts = ; #iommu-cells = <1>; status = "disabled"; From 862f207fa078d99b2d499457a477ff5afe010b44 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:23:01 +0200 Subject: [PATCH 085/392] arm64: dts: apple: t8142-j704: Enable the MacBook Pro (14-inch, M5) The J704 board file includes the disabled T8142 inventory. Enable what runs on this machine and describe the board's own devices from its Apple Device Tree: - The ten CPUs with cpufreq and the PMC voters: E-cluster states up to 3048 MHz and P-cluster states up to 4608 MHz. - AIC, the timer, the Samsung UART as the console (serial0, 115200 baud) and the dockchannel UART (serial1). - PMGR power domains, the PMP and the SMC: battery, AC, fan and temperatures, the power button and lid, reboot and the RTC through the PMU's rtc_scrpad offset. - The keyboard and trackpad on the MTP, with the trackpad firmware name and the keyboard backlight at half brightness. - The three Type-C ports: ATC0 and ATC1 on nub SPMI A0, ATC3 on nub SPMI A1. ATC2 drives the HDMI port and is left out. - The AVD video decoder. - The internal DCP and the 14.2" mini-LED panel, disabled: the loader enables them for a display session it hands over. On macOS 27.0 firmware the panel stays on the boot framebuffer until the DCP driver runs that firmware. The inventory's DARTs are used through their labels. Each USB DART node lists both of its instances in reg; the board keeps instance 0 there and adds instance 1 as its own node, as on the J813. Signed-off-by: Dmitriy Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8142-j704-avd.dtsi | 42 ++ .../boot/dts/apple/t8142-j704-display.dtsi | 102 ++++ .../boot/dts/apple/t8142-j704-hwmon.dtsi | 174 +++++++ .../dts/apple/t8142-j704-peripherals.dtsi | 264 ++++++++++ .../boot/dts/apple/t8142-j704-power.dtsi | 83 +++ arch/arm64/boot/dts/apple/t8142-j704-usb.dtsi | 488 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j704.dts | 312 ++++++++++- 7 files changed, 1464 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-avd.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-display.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-hwmon.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-peripherals.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-power.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-usb.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j704-avd.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-avd.dtsi new file mode 100644 index 0000000000000..0d807f3712162 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-avd.dtsi @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) AVD video decoder: a "Hibiscus" AVD (macOS CAvdMcpuHibiscus, + * avd-version 4, tunables = avd-fw v5 tier 2). The J704 ADT has it where the + * J813 does: avd reg[0] 0x2f0000000 + 0x1404000, here at the arm-io offset + * 0x500000000; 0x14000 each of CM3 code and SRAM, mailbox at +0x10a8000 + * (CAvdApCommHibiscus); IRQs 1077/1078; dart-avd at +0x1020000, IRQ 1084, + * SIDs 0-2; power: PMGR device AVD-SYS-V -> AVD_SYS (+0x310). + */ + +&pmgr { + ps_avd_sys: power-controller@310 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; +}; + +&{/soc} { + avd: avd@501080000 { + compatible = "apple,t8142-avd"; + interrupt-parent = <&aic>; + interrupts = , + ; + power-domains = <&ps_avd_sys>; + resets = <&ps_avd_sys>; + reg = <0x5 0x01080000 0x0 0x14000>, + <0x5 0x01094000 0x0 0x14000>, + <0x5 0x010a8000 0x0 0x4000>, + <0x5 0x01100000 0x0 0x10000>; + reg-names = "code", "sram", "mbox", "ctrl"; + iommus = <&avd_dart 0>, <&avd_dart 1>, <&avd_dart 2>; + }; + +}; + +&avd_dart { + power-domains = <&ps_avd_sys>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-display.dtsi new file mode 100644 index 0000000000000..fa807294542e2 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-display.dtsi @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J704 internal display pipeline: the DCP coprocessor and its DARTs. + * + * The bootloader leaves DCP running with its firmware mapped through + * locked DART tables. The loader reserves those mappings and enables the + * nodes below; Linux attaches to the running coprocessor without resetting + * it. Resources were read from this machine's Apple Device Tree. + */ + +/ { + aliases { + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + }; + + display: display-subsystem { + compatible = "apple,t8142-display-subsystem", "apple,display-subsystem"; + /* disp0_dart must be first since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; +}; + +&{/soc} { + pmgr_dcp: power-management@382720000 { + reg = <3 0x82720000 0 0x8>, <3 0x827a4000 0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp_mbox: mailbox@286e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <2 0x86e08000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dcp: dcp@286e00000 { + compatible = "apple,t8142-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + reg = <2 0x86e00000 0 0x4000>, + <2 0x84000000 0 0xb1c000>, + <2 0x86240000 0 0x10000>, + <2 0x863ac000 0 0x4000>, + <2 0x86800000 0 0x800000>, + <3 0x82700000 0 0x114000>, + <2 0x86850000 0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* Scan out the primary plane on IOMFB surface 2. */ + apple,iomfb-surfaces = <0 0 1 0>; + clocks = <&clkref>; + /* generate phandle explicitly for use in loader */ + phandle = <&dcp>; + status = "disabled"; + /* required bus properties for 'piodma' subdevice */ + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + + /* + * 14.2" mini-LED, 3024x1964 (1890 below the notch, which the + * loader passes as apple,notch-height). Makes the connector eDP + * and gives the DCP a backlight to drive. + */ + panel: panel { + compatible = "apple,panel-j704", "apple,panel-mini-led", + "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; + apple,max-brightness = <600>; + }; + }; +}; + +&disp0_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; +}; + +&dcp_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-hwmon.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-hwmon.dtsi new file mode 100644 index 0000000000000..be16e2982ba45 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-hwmon.dtsi @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * SMC sensors and the fan of the MacBook Pro (14-inch, M5, 2025), J704. + */ + +#include "hwmon-common.dtsi" +#include "hwmon-laptop.dtsi" +#include "hwmon-fan.dtsi" + +&smc_hwmon { + /* + * CPU sensors respond to loads pinned to each cluster. The GPU keys + * follow the M5 macOS sensor group; numbers identify sensors, not cores. + * The hottest of these die readings supplies the driver's SoC zone. + */ + apple,soc-die-keys = "Te04", "Te08", "Te0C", "Te0R", + "Tp00", "Tp04", "Tp0C", "Tp0G", "Tp0O", + "Tp0R", "Tp0X", "Tp0a", "Tp0p", "Tp0u", + "Tp0y", "Tp12", "Tp16", "Tp1E", + "Tg0U", "Tg0X", "Tg0d", "Tg0g", "Tg0j", + "Tg1Y", "Tg1c", "Tg1g"; + apple,thermal-pressure-key = "mTPL"; + + fan-F0Ac { + /* This SMC names the fan mode key in lower case. */ + apple,fan-mode = "F0md"; + apple,fan-manual-control; + }; + + temperature-Te04 { + apple,key-id = "Te04"; + label = "CPU E-cluster Temperature 1"; + }; + + temperature-Te08 { + apple,key-id = "Te08"; + label = "CPU E-cluster Temperature 2"; + }; + + temperature-Te0C { + apple,key-id = "Te0C"; + label = "CPU E-cluster Temperature 3"; + }; + + temperature-Te0R { + apple,key-id = "Te0R"; + label = "CPU E-cluster Temperature 4"; + }; + + temperature-Tp00 { + apple,key-id = "Tp00"; + label = "CPU P-cluster Temperature 1"; + }; + + temperature-Tp04 { + apple,key-id = "Tp04"; + label = "CPU P-cluster Temperature 2"; + }; + + temperature-Tp0C { + apple,key-id = "Tp0C"; + label = "CPU P-cluster Temperature 3"; + }; + + temperature-Tp0G { + apple,key-id = "Tp0G"; + label = "CPU P-cluster Temperature 4"; + }; + + temperature-Tp0O { + apple,key-id = "Tp0O"; + label = "CPU P-cluster Temperature 5"; + }; + + temperature-Tp0R { + apple,key-id = "Tp0R"; + label = "CPU P-cluster Temperature 6"; + }; + + temperature-Tp0X { + apple,key-id = "Tp0X"; + label = "CPU P-cluster Temperature 7"; + }; + + temperature-Tp0a { + apple,key-id = "Tp0a"; + label = "CPU P-cluster Temperature 8"; + }; + + temperature-Tp0p { + apple,key-id = "Tp0p"; + label = "CPU P-cluster Temperature 9"; + }; + + temperature-Tp0u { + apple,key-id = "Tp0u"; + label = "CPU P-cluster Temperature 10"; + }; + + temperature-Tp0y { + apple,key-id = "Tp0y"; + label = "CPU P-cluster Temperature 11"; + }; + + temperature-Tp12 { + apple,key-id = "Tp12"; + label = "CPU P-cluster Temperature 12"; + }; + + temperature-Tp16 { + apple,key-id = "Tp16"; + label = "CPU P-cluster Temperature 13"; + }; + + temperature-Tp1E { + apple,key-id = "Tp1E"; + label = "CPU P-cluster Temperature 14"; + }; + + temperature-Tg0U { + apple,key-id = "Tg0U"; + label = "GPU Temperature 1"; + }; + + temperature-Tg0X { + apple,key-id = "Tg0X"; + label = "GPU Temperature 2"; + }; + + temperature-Tg0d { + apple,key-id = "Tg0d"; + label = "GPU Temperature 3"; + }; + + temperature-Tg0g { + apple,key-id = "Tg0g"; + label = "GPU Temperature 4"; + }; + + temperature-Tg0j { + apple,key-id = "Tg0j"; + label = "GPU Temperature 5"; + }; + + temperature-Tg1Y { + apple,key-id = "Tg1Y"; + label = "GPU Temperature 6"; + }; + + temperature-Tg1c { + apple,key-id = "Tg1c"; + label = "GPU Temperature 7"; + }; + + temperature-Tg1g { + apple,key-id = "Tg1g"; + label = "GPU Temperature 8"; + }; + + temperature-TB1T { + apple,key-id = "TB1T"; + label = "Battery Temperature 1"; + }; + + temperature-TB2T { + apple,key-id = "TB2T"; + label = "Battery Temperature 2"; + }; + + temperature-TAOL { + apple,key-id = "TAOL"; + label = "Ambient (TAOL)"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-peripherals.dtsi new file mode 100644 index 0000000000000..d962ab3291e03 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-peripherals.dtsi @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include +#include + +/ { + aliases { + mtp = &mtp; + keyboard = &keyboard; + nvram = &nvram; + pmp = &pmp; + }; + + /* ADT /arm-io/qspi clock-ids [396]; rate as the T8140 QSPI. */ + clk_qspi: clock-qspi { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_qspi"; + }; + + led-controller { + compatible = "pwm-leds"; + + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-brightness = <127>; + default-state = "on"; + }; + }; +}; + +&pmgr { + ps_fpwm0: power-controller@330 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + + /* ADT pmgr devices: SPI_P id 94 (+0x390), QSPI id 120 (+0x460). */ + ps_spi_p: power-controller@390 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + /* ADT pmgr devices: PMP id 159 (+0x598), PMS_SRAM id 160 (+0x5a0). */ + ps_pmp: power-controller@598 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + /* The PMP's SRAM, which the driver reads first. */ + power-domains = <&ps_pms_sram>; + }; + + ps_pms_sram: power-controller@5a0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + /* T8140 holds it through its always-on PMP report node. */ + apple,always-on; + }; + + ps_qspi: power-controller@460 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; +}; + +&{/soc} { + /* + * PMP, laid out as on T8140 (ADT /arm-io/pmp, dart-pmp): it runs the + * DRAM controller (DCS) and fabric DVFS. Without it the memory stays at + * the bootloader's state (memcpy ~20 GB/s against ~120 GB/s on macOS). + */ + pmp: pmp@380500000 { + compatible = "apple,t8142-pmp-v2", "apple,t8140-pmp-v2"; + reg = <0x3 0x80500000 0x0 0xd0000>, + <0x3 0x80e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + /* + * ADT /arm-io/pmgr pmc, pmc-pmgr (registry-v2 boot arguments). + * The loader adds apple,board-id, apple,dram-vendor-id and the + * apple,tunable-* values from this machine's ADT. + */ + apple,pmc = <1>; + apple,pmc-pmgr = <31>; + }; + + pmp_mbox: mailbox@380e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x80e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + }; + + /* ADT /arm-io/qspi: 0x195118000 and its FIFO window at +0x800, IRQ 1245. */ + qspi: spi@3a5118000 { + compatible = "apple,t8142-qspi", "apple,t8140-qspi"; + reg = <0x3 0xa5118000 0x0 0x4800>; + clocks = <&clk_qspi>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + /* Conservative until the input clock is confirmed. */ + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* ADT spinor/anvram reg: 0x100000 bytes at 0x700000. */ + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; + }; + + fpwm0: pwm@3a5040000 { + compatible = "apple,t8142-fpwm", "apple,s5l-fpwm"; + reg = <3 0xa5040000 0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; + + smc_mbox: mailbox@38c608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x8c608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8142-smc", "apple,t8103-smc"; + reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + }; + + mtp: mtp@394600000 { + compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@394608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x94608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dockchannel: fifo@394b14000 { + compatible = "apple,t8142-dockchannel", "apple,dockchannel"; + reg = <3 0x94b14000 0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 3 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + /* + * ADT dockchannel-mtp/mtp-transport/multi-touch: + * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28, + * as on J813. + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; + + stm { }; + + multi-touch { + firmware-name = "apple/tpmtfw-j704.bin"; + apple,power-method = <2>; + }; + + /* Layout is filled by the loader using this machine's own data. */ + keyboard: keyboard { }; + }; + }; +}; + +&pmp_dart { + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + status = "okay"; +}; + +&mtp_dart { + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-power.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-power.dtsi new file mode 100644 index 0000000000000..6011d77695aa3 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-power.dtsi @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M5) J704: SMC power off / restart, the RTC, the + * PMU on SPMI and the watchdog. + */ + +&{/soc} { + /* ADT /arm-io/wdt (wdt,t8142): restart (and the SMC's fallback). */ + wdt: watchdog@3882b0000 { + compatible = "apple,t8142-wdt", "apple,t8103-wdt"; + reg = <0x3 0x882b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; +}; + +&{/soc} { + /* + * ADT /arm-io/nub-spmi0 (gen 4) and its primary PMU, pmu-main + * ("abbey", slave 0xe). The boot flags live in the legacy scratchpad + * (info-leg_scrpad = 0xf800), laid out as on T8103 and J873. + */ + nub_spmi0: spmi@388714000 { + compatible = "apple,t8142-spmi"; + reg = <0x3 0x88714000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + /* ADT info-rtc_scrpad: 48-bit offset in 32768 Hz ticks. */ + rtc_offset: rtc-offset@2100 { + reg = <0x2100 0x6>; + }; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + /* Set before off1, or the machine powers back on. */ + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; +}; + +&smc { + /* Power off (MBSE = off1) and restart (MBSE = phra). */ + reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-usb.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-usb.dtsi new file mode 100644 index 0000000000000..09c51e0888aa2 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-usb.dtsi @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include +#include + +/* + * Type-C ports: left-front (ATC0) and left-rear (ATC1) on nub SPMI A0, + * right (ATC3) on nub SPMI A1. ATC2 has no connector: it drives the HDMI + * port through a DisplayPort to HDMI converter. + */ + +/ { + aliases { + atcphy0 = &atcphy0; + usb0 = &dwc3_0; + atcphy1 = &atcphy1; + usb1 = &dwc3_1; + atcphy3 = &atcphy3; + usb3 = &dwc3_3; + }; +}; + +&pmgr { + ps_atc0_common: power-controller@2c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-common"; + }; +}; + +&{/soc} { + pmgr_nub: power-management@388280000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x88280000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub-spmi-a0"; + }; + + ps_atc0_usb_aon: power-controller@c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-usb-aon"; + }; + + ps_atc0_usb: power-controller@d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + label = "atc0-usb"; + }; + }; + + nub_spmi_a0: spmi@388908000 { + compatible = "apple,t8142-spmi"; + reg = <3 0x88908000 0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + + port@1 { + reg = <1>; + typec0_ss: endpoint { + remote-endpoint = <&atcphy0_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_0: usb@402280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x02280000 0 0xcd00>, <4 0x0228cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb>; + }; + }; + }; + }; + + dwc3_0_dart_1: iommu@402f80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy0: phy@403000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x03000000 0 0x4c000>, + <4 0x03050000 0 0x8000>, + <4 0x00000000 0 0x8000>, + <4 0x02a90000 0 0x4000>, + <4 0x02a84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy0_typec: endpoint { + remote-endpoint = <&typec0_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy0_usb: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; +}; + +&pmgr { + ps_atc1_common: power-controller@2d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-common"; + }; +}; + +&pmgr_nub { + ps_atc1_usb_aon: power-controller@d8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-usb-aon"; + }; + + ps_atc1_usb: power-controller@e0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + label = "atc1-usb"; + }; +}; + +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-rear"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec1_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + + port@1 { + reg = <1>; + typec1_ss: endpoint { + remote-endpoint = <&atcphy1_typec>; + }; + }; + }; + }; + }; +}; + +&{/soc} { + dwc3_1: usb@40a280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x0a280000 0 0xcd00>, <4 0x0a28cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb>; + }; + }; + }; + }; + + dwc3_1_dart_1: iommu@40af80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy1: phy@40b000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x0b000000 0 0x4c000>, + <4 0x0b050000 0 0x8000>, + <4 0x08000000 0 0x8000>, + <4 0x0aa90000 0 0x4000>, + <4 0x0aa84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy1_typec: endpoint { + remote-endpoint = <&typec1_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy1_usb: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; + }; +}; + +&pmgr { + ps_atc3_common: power-controller@2e0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3-common"; + }; +}; + +&pmgr_nub { + ps_nub_spmi_a1: power-controller@78 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub-spmi-a1"; + }; + + ps_atc3_usb_aon: power-controller@f8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3-usb-aon"; + }; + + ps_atc3_usb: power-controller@100 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>; + label = "atc3-usb"; + }; +}; + +&{/soc} { + nub_spmi_a1: spmi@388a08000 { + compatible = "apple,t8142-spmi"; + reg = <3 0x88a08000 0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a1>; + + hpm3: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec3_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + + port@1 { + reg = <1>; + typec3_ss: endpoint { + remote-endpoint = <&atcphy3_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_3: usb@41a280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x1a280000 0 0xcd00>, <4 0x1a28cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_3_dart 0>, <&dwc3_3_dart_1 1>; + power-domains = <&ps_atc3_usb>; + resets = <&atcphy3>; + phys = <&atcphy3 PHY_TYPE_USB2>, <&atcphy3 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb>; + }; + }; + }; + }; + + dwc3_3_dart_1: iommu@41af80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x1af80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc3_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy3: phy@41b000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x1b000000 0 0x4c000>, + <4 0x1b050000 0 0x8000>, + <4 0x18000000 0 0x8000>, + <4 0x1aa90000 0 0x4000>, + <4 0x1aa84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc3_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy3_typec: endpoint { + remote-endpoint = <&typec3_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy3_usb: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; + }; +}; + +&dwc3_0_dart { + /* Instance 0 only: instance 1 is dwc3_0_dart_1. */ + reg = <4 0x02f00000 0 0xc000>; + power-domains = <&ps_atc0_usb>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + status = "okay"; +}; + +&dwc3_1_dart { + /* Instance 0 only: instance 1 is dwc3_1_dart_1. */ + reg = <4 0x0af00000 0 0xc000>; + power-domains = <&ps_atc1_usb>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + status = "okay"; +}; + +&dwc3_3_dart { + /* Instance 0 only: instance 1 is dwc3_3_dart_1. */ + reg = <4 0x1af00000 0 0xc000>; + power-domains = <&ps_atc3_usb>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704.dts b/arch/arm64/boot/dts/apple/t8142-j704.dts index d3a9329dc59ec..b6ce8a4dc5556 100644 --- a/arch/arm64/boot/dts/apple/t8142-j704.dts +++ b/arch/arm64/boot/dts/apple/t8142-j704.dts @@ -1,10 +1,320 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M5, 2025), J704. + * + * Resources from the J704 Apple Device Tree. The loader supplies usable + * memory, CPU release addresses and framebuffer geometry. + */ /dts-v1/; #include "t8142.dtsi" / { - model = "Apple MacBook Pro (J704)"; compatible = "apple,j704", "apple,t8142", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M5, 2025)"; chassis-type = "laptop"; + + aliases { + serial0 = &serial0; + serial1 = &dockchannel; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:115200n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "disabled"; + }; + }; + + /* + * State 1 is 300 MHz; the ADT voltage-states1-sram (E) and + * voltage-states5-sram (P) tables begin at state 2. + */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + + opp-1584000000 { + opp-hz = /bits/ 64 <1584000000>; + opp-level = <4>; + }; + + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <5>; + }; + + opp-2352000000 { + opp-hz = /bits/ 64 <2352000000>; + opp-level = <6>; + }; + + opp-2700000000 { + opp-hz = /bits/ 64 <2700000000>; + opp-level = <7>; + }; + + opp-2964000000 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <8>; + }; + + opp-3048000000 { + opp-hz = /bits/ 64 <3048000000>; + opp-level = <9>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-1308000000 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <2>; + }; + + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-level = <3>; + }; + + opp-1932000000 { + opp-hz = /bits/ 64 <1932000000>; + opp-level = <4>; + }; + + opp-2220000000 { + opp-hz = /bits/ 64 <2220000000>; + opp-level = <5>; + }; + + opp-2508000000 { + opp-hz = /bits/ 64 <2508000000>; + opp-level = <6>; + }; + + opp-2820000000 { + opp-hz = /bits/ 64 <2820000000>; + opp-level = <7>; + }; + + opp-3072000000 { + opp-hz = /bits/ 64 <3072000000>; + opp-level = <8>; + }; + + opp-3300000000 { + opp-hz = /bits/ 64 <3300000000>; + opp-level = <9>; + }; + + opp-3516000000 { + opp-hz = /bits/ 64 <3516000000>; + opp-level = <10>; + }; + + /* The SoC throttles levels 11 and up when hot. */ + opp-3720000000 { + opp-hz = /bits/ 64 <3720000000>; + opp-level = <11>; + }; + + opp-3876000000 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <12>; + }; + + opp-4020000000 { + opp-hz = /bits/ 64 <4020000000>; + opp-level = <13>; + }; + + opp-4176000000 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <14>; + }; + + opp-4248000000 { + opp-hz = /bits/ 64 <4248000000>; + opp-level = <15>; + }; + + opp-4272000000 { + opp-hz = /bits/ 64 <4272000000>; + opp-level = <16>; + }; + + opp-4332000000 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + + opp-4428000000 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + + opp-4512000000 { + opp-hz = /bits/ 64 <4512000000>; + opp-level = <19>; + }; + + opp-4608000000 { + opp-hz = /bits/ 64 <4608000000>; + opp-level = <20>; + }; + }; }; + +&{/soc} { + cpufreq_e: performance-controller@210e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: performance-controller@211e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + /* + * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): + * floors at 0x382700000 + agent * 0x4000, per-agent states at + * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): + * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so + * no vote is honoured and DRAM and fabric stay at their lowest + * states (CPU DRAM latency 171 ns instead of 70 ns). + */ + pmc: power-management@382700000 { + compatible = "apple,t8142-pmc"; + reg = <0x3 0x82700000 0x0 0x10000>, + <0x3 0x82806000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8142-clpc"; + reg = <1>; + }; + }; +}; + +&cpu0 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu1 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu2 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu3 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu4 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu5 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu6 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu7 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu8 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu9 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&{/timer} { + status = "okay"; +}; + +&aic { + status = "okay"; +}; + +&serial0 { + status = "okay"; +}; + +&dockchannel { + status = "okay"; +}; + +&pmgr { + status = "okay"; +}; + +#include "t8142-j704-display.dtsi" +#include "t8142-j704-peripherals.dtsi" +#include "t8142-j704-usb.dtsi" +#include "t8142-j704-avd.dtsi" +#include "t8142-j704-power.dtsi" +#include "t8142-j704-hwmon.dtsi" From 99e75f5890f09bf03bfb0fee7b2e54ce0b658eca Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 8 Nov 2026 15:23:01 +0200 Subject: [PATCH 086/392] arm64: dts: apple: t8142-j704: Add NVMe, Wi-Fi/Bluetooth and the SD reader Describe the internal ANS2 NVMe SSD as on J813. This machine's ANS firmware ends the nvme register block at 0x10000 (ADT ans reg[9]) and keeps the linear submission queue doorbells in the NVMMU block, which the driver detects from the window size. Describe the internal PCIe as J813 describes its N1: the boot loader trains the links and Linux drives the root ports as a generic ECAM host. Port 0 carries the BCM4388 Wi-Fi/Bluetooth, port 2 the GL9755 SD card reader. Both ports' MSI maps share one AIC range, so one apple,t8142-pcie-msi controller lists both. The ports use the inventory's pcie0 and pcie2 DARTs, stream 0. Signed-off-by: Dmitriy Assisted-by: LLM --- .../arm64/boot/dts/apple/t8142-j704-nvme.dtsi | 103 ++++++++++++++++++ .../arm64/boot/dts/apple/t8142-j704-pcie.dtsi | 91 ++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j704.dts | 4 + 3 files changed, 198 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-nvme.dtsi create mode 100644 arch/arm64/boot/dts/apple/t8142-j704-pcie.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j704-nvme.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-nvme.dtsi new file mode 100644 index 0000000000000..67a08b81d4a29 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-nvme.dtsi @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M5) J704 with the internal ANS2 NVMe SSD. + * + * Laid out as on J813 (ADT /arm-io/ans, /arm-io/sart-ans, arm-io offset + * 0x210000000), but this machine's ANS firmware ends the nvme register + * block at 0x10000 (ADT ans reg[9]) and has the linear submission queue + * doorbells in the NVMMU block. iBoot leaves ANS running with the + * controller disabled, so the driver wakes it with a fresh RTKit handshake. + */ + +&pmgr { + ps_apcie_gp: power-controller@2f0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_ans: power-controller@300 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + apple,always-on; /* handed over running, cannot be reset */ + }; + + ps_apcie_sys_gp: power-controller@408 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_apcie_st: power-controller@410 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_apcie_sys_st: power-controller@520 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@560 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; +}; + +&{/soc} { + ans_mbox: mailbox@481608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x81608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + }; + + sart: iommu@485c50000 { + compatible = "apple,t8142-sart", "apple,t6000-sart"; + reg = <0x4 0x85c50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@4c5cc0000 { + compatible = "apple,t8142-nvme-ans2", "apple,t8140-nvme-ans2"; + reg = <0x4 0xc5cc0000 0x0 0x10000>, + <0x4 0x81600000 0x0 0x4000>, + <0x4 0x85cc0000 0x0 0x60000>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + power-domains = <&ps_ans>, <&ps_apcie_sys_st>, + <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0", "apcie_phy"; + resets = <&ps_ans>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704-pcie.dtsi b/arch/arm64/boot/dts/apple/t8142-j704-pcie.dtsi new file mode 100644 index 0000000000000..a0694d5064555 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j704-pcie.dtsi @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Internal PCIe: BCM4388 Wi-Fi/Bluetooth on apcie port 0 and the GL9755 SD + * card reader on port 2. As on J813, the bootloader trains the links and + * Linux uses the root ports as a generic ECAM host that owns the DARTs and + * the endpoints. Nothing programs the ports' RID-to-SID tables, so every + * function uses stream zero of its port's DART. Ranges, MSI maps and DART + * IRQs are from the J704 ADT; port 0 matches J813. + */ + +&{/} { + pcie0: pcie@1cb0000000 { + compatible = "pci-host-ecam-generic"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0 2>; + reg = <0x1c 0xb0000000 0 0x300000>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0 0x20000000>, + <0x02000000 0 0x80000000 0xb 0x80000000 0 0x40000000>; + dma-coherent; + msi-parent = <&pcie0_msi>; + iommu-map = <0x100 &pcie0_dart 0 1>, + <0x101 &pcie0_dart 0 1>, + <0x200 &pcie2_dart 0 1>; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <1 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + brcm,board-type = "apple,linosa"; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + brcm,board-type = "apple,linosa"; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; + }; + + port02: pci@2,0 { + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + bus-range = <2 2>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + + sdhci0: mmc@0,0 { + compatible = "pci17a0,9755"; + reg = <0x20000 0x0 0x0 0x0 0x0>; + cd-inverted; + wp-inverted; + }; + }; + }; +}; + +&{/soc} { + /* + * The MSI maps of ports 0 and 2. They share the root complex's AIC + * range (ADT msi-vector-offset 1741); 32 vectors are enough for both. + */ + pcie0_msi: msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <0x4 0x88028000 0x0 0x8000>, + <0x4 0x8a028000 0x0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; +}; + +&pcie0_dart { + status = "okay"; +}; + +&pcie2_dart { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j704.dts b/arch/arm64/boot/dts/apple/t8142-j704.dts index b6ce8a4dc5556..aebc8830d136a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j704.dts +++ b/arch/arm64/boot/dts/apple/t8142-j704.dts @@ -17,6 +17,8 @@ aliases { serial0 = &serial0; serial1 = &dockchannel; + bluetooth0 = &bluetooth0; + wifi0 = &wifi0; }; chosen { @@ -316,5 +318,7 @@ #include "t8142-j704-peripherals.dtsi" #include "t8142-j704-usb.dtsi" #include "t8142-j704-avd.dtsi" +#include "t8142-j704-nvme.dtsi" +#include "t8142-j704-pcie.dtsi" #include "t8142-j704-power.dtsi" #include "t8142-j704-hwmon.dtsi" From f6ba35b02115a9a41b0eb647b8c2f10b2632cf5e Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 5 Nov 2026 14:37:46 +0200 Subject: [PATCH 087/392] thunderbolt: apple: Do not start the NHI without its DART The NHI and its DART are both children of the ACIO and are created together when the router starts. The DART normally binds first, and the driver core defers the NHI until it has. Once the core has stopped waiting for missing suppliers, though (for example, the DART driver is a module that is not loaded yet when a cable is plugged in), the NHI probes without an IOMMU and would hand the hardware physical ring addresses. Refuse to probe an NHI that has iommus but is not attached to an IOMMU domain. Report it through the boot status as well, so that the ACIO fails this start right away with -EPROBE_DEFER instead of waiting for the 10 s NHI boot timeout; the next connection retries. Signed-off-by: Dmitriy Assisted-by: LLM --- drivers/thunderbolt/apple.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index ba7ec0bc0296c..6330140715e2b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -83,6 +83,7 @@ #include #include #include +#include #include #include #include @@ -2052,6 +2053,20 @@ static int apple_nhi_probe(struct platform_device *pdev) goto err; } + /* + * The rings must never run untranslated. The DART is populated with + * the NHI, so it is normally bound first; but once the driver core + * has stopped waiting for missing suppliers (DART driver not loaded + * yet), the NHI would probe without its IOMMU. Defer, and fail the + * ACIO start now instead of letting it wait for the NHI to time out. + */ + if (of_property_present(pdev->dev.of_node, "iommus") && + !device_iommu_mapped(&pdev->dev)) { + dev_warn(&pdev->dev, "DART not bound yet, not starting the NHI\n"); + ret = -EPROBE_DEFER; + goto err; + } + anhi = devm_kzalloc(&pdev->dev, sizeof(*anhi), GFP_KERNEL); if (!anhi) { ret = -ENOMEM; From 76b09427c2295ca237e5253c6dcd6f54427bef9f Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sun, 8 Nov 2026 06:15:05 -0500 Subject: [PATCH 088/392] drm/apple: Fix a GPU fence that never signals wedging the display drm_atomic_helper_commit() waits for every plane's in-fence with dma_fence_wait() and no timeout: in the commit worker before the driver's commit tail, and, for blocking commits, before the swap. The in-fence is the scanout buffer's implicit write fence (zink imports its GPU job's fence into each exported dma-buf) or the compositor's IN_FENCE_FD. When a GPU fence never signals, after a GPU recovery for example, the worker sleeps for good: commit_planes() never runs, no swap reaches DCP, the commit's flip event never comes, and every later nonblocking commit fails with -EBUSY (stall_checks(), only a debug message). The 10 s flip_done timeout is never reached, so nothing is logged. The panel stays black across compositor restarts, which see "Cannot commit when a page-flip is awaiting", until a reboot. Commit through a copy of drm_atomic_helper_commit() whose fence wait is bounded: all of a commit's planes share one second, and a fence that does not signal in time is dropped with a rate-limited warning naming its driver and timeline. The buffer is scanned out as it is, so a GPU bug costs a stale or torn frame instead of the panel. The copy keeps the helper's sequence (setup, prepare, the interruptible pre-swap wait for blocking commits, swap, the worker's wait for dependencies, the tail, cleanup) and its fence deadline hint, and leaves out only the self-refresh accounting, which this driver does not use. Once the commit goes on, its flip completes as usual. If DCP then does not complete the swap within drm_atomic_helper_wait_for_flip_done()'s 10 s, the commit's event stayed pending and a compositor would still wait for it before committing again. Send that event as failed (-ETIMEDOUT on its out-fence) after the wait; only the event that belongs to this commit is sent. The commit then leaves its CRTC's list at cleanup_done as before, so later nonblocking commits are not held by it either. Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- drivers/gpu/drm/apple/apple_drv.c | 201 +++++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.c | 18 +++ drivers/gpu/drm/apple/dcp.h | 2 + 3 files changed, 220 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 763ba82ad29ac..fbd0325b963d2 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -9,7 +9,9 @@ #include #include +#include #include +#include #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -562,12 +565,206 @@ static void apple_atomic_state_free(struct drm_atomic_commit *state) kfree(to_apple_atomic_state(state)); } +/* + * A plane's in-fence (its buffer's implicit write fence, or the compositor's + * IN_FENCE_FD) is waited for at most this long per commit. The helpers wait + * for it without a timeout: a GPU fence that never signals, after a GPU + * recovery for example, leaves the commit worker asleep for good. No later + * frame reaches the display, the commit's flip event never comes, and every + * later nonblocking commit fails with -EBUSY. One second is many frames even + * at the panel's slowest refresh; past it the buffer is scanned out as it is, + * so a GPU bug costs a stale or torn frame instead of the panel. + */ +#define APPLE_PLANE_FENCE_TIMEOUT_MS 1000 + +static void apple_warn_fence_timeout(struct drm_device *dev, + struct drm_plane *plane, + struct dma_fence *fence) +{ + static DEFINE_RATELIMIT_STATE(rs, 60 * HZ, 1); + const char __rcu *driver; + const char __rcu *timeline; + + if (!__ratelimit(&rs)) + return; + rcu_read_lock(); + driver = dma_fence_driver_name(fence); + timeline = dma_fence_timeline_name(fence); + drm_warn(dev, + "[PLANE:%d:%s] in-fence %s:%s seqno %llu not signalled after %u ms, scanning out without it\n", + plane->base.id, plane->name, rcu_dereference(driver), + rcu_dereference(timeline), fence->seqno, + APPLE_PLANE_FENCE_TIMEOUT_MS); + rcu_read_unlock(); +} + +/* As the helpers do: the GPU may finish the frame sooner knowing the vblank. */ +static void apple_set_fence_deadline(struct drm_atomic_commit *state) +{ + struct drm_crtc_state *new_crtc_state; + struct drm_plane_state *new_plane_state; + struct drm_plane *plane; + struct drm_crtc *crtc; + ktime_t vbltime = 0; + int i; + + for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { + ktime_t v; + + if (drm_atomic_crtc_needs_modeset(new_crtc_state) || + !new_crtc_state->active || + drm_crtc_next_vblank_start(crtc, &v)) + continue; + if (!vbltime || ktime_before(v, vbltime)) + vbltime = v; + } + if (!vbltime) + return; + for_each_new_plane_in_state(state, plane, new_plane_state, i) + if (new_plane_state->fence) + dma_fence_set_deadline(new_plane_state->fence, vbltime); +} + +/* + * drm_atomic_helper_wait_for_fences() with one bound for all of the commit's + * planes (APPLE_PLANE_FENCE_TIMEOUT_MS). A fence that does not signal in time + * is dropped. Before the swap (blocking commits) the wait is interruptible, + * as the helper's is. + */ +static int apple_wait_for_fences(struct drm_device *dev, + struct drm_atomic_commit *state, + bool pre_swap) +{ + unsigned long deadline = + jiffies + msecs_to_jiffies(APPLE_PLANE_FENCE_TIMEOUT_MS); + struct drm_plane_state *new_plane_state; + struct drm_plane *plane; + int i; + + apple_set_fence_deadline(state); + + for_each_new_plane_in_state(state, plane, new_plane_state, i) { + struct dma_fence *fence = new_plane_state->fence; + long left, ret; + + if (!fence) + continue; + + WARN_ON(!new_plane_state->fb); + + left = time_before(jiffies, deadline) ? deadline - jiffies : 0; + ret = dma_fence_wait_timeout(fence, pre_swap, left); + if (ret < 0) + return ret; + if (!ret) + apple_warn_fence_timeout(dev, plane, fence); + + dma_fence_put(fence); + new_plane_state->fence = NULL; + } + + return 0; +} + +/* + * A flip DCP did not complete within the helper's 10 s + * (drm_atomic_helper_wait_for_flip_done() gave up) leaves its event pending. + * A compositor waits for that event before it commits again, so send it now, + * with -ETIMEDOUT on its out-fence, and the next commit can go ahead. A swap + * completion that still comes later finds no event. + */ +static void apple_fail_lost_flips(struct drm_atomic_commit *state) +{ + for (int i = 0; i < state->dev->mode_config.num_crtc; i++) { + struct drm_crtc_commit *commit = state->crtcs[i].commit; + struct drm_crtc *crtc = state->crtcs[i].ptr; + + if (!crtc || !commit || t8132_ext_crtc(crtc) || + completion_done(&commit->flip_done)) + continue; + dcp_drm_crtc_flip_lost(to_apple_crtc(crtc), &commit->flip_done, + -ETIMEDOUT); + } +} + +static void apple_atomic_commit_tail(struct drm_atomic_commit *state); + +/* The helper's commit_tail() with the bounded fence wait. */ +static void apple_commit_tail(struct drm_atomic_commit *state) +{ + apple_wait_for_fences(state->dev, state, false); + drm_atomic_helper_wait_for_dependencies(state); + apple_atomic_commit_tail(state); + drm_atomic_helper_commit_cleanup_done(state); + drm_atomic_commit_put(state); +} + +static void apple_commit_work(struct work_struct *work) +{ + apple_commit_tail(container_of(work, struct drm_atomic_commit, + commit_work)); +} + +/* + * drm_atomic_helper_commit() with apple_wait_for_fences() in place of the + * helper's unbounded wait. The driver keeps no self-refresh state, so the + * helper's self-refresh accounting is left out. + */ +static int apple_atomic_commit(struct drm_device *dev, + struct drm_atomic_commit *state, bool nonblock) +{ + int ret; + + if (state->async_update) { + ret = drm_atomic_helper_prepare_planes(dev, state); + if (ret) + return ret; + + drm_atomic_helper_async_commit(dev, state); + drm_atomic_helper_unprepare_planes(dev, state); + + return 0; + } + + ret = drm_atomic_helper_setup_commit(state, nonblock); + if (ret) + return ret; + + INIT_WORK(&state->commit_work, apple_commit_work); + + ret = drm_atomic_helper_prepare_planes(dev, state); + if (ret) + return ret; + + if (!nonblock) { + ret = apple_wait_for_fences(dev, state, true); + if (ret) + goto err; + } + + ret = drm_atomic_helper_swap_state(state, true); + if (ret) + goto err; + + drm_atomic_commit_get(state); + if (nonblock) + queue_work(system_dfl_wq, &state->commit_work); + else + apple_commit_tail(state); + + return 0; + +err: + drm_atomic_helper_unprepare_planes(dev, state); + return ret; +} + static const struct drm_mode_config_funcs apple_mode_config_funcs = { .atomic_state_alloc = apple_atomic_state_alloc, .atomic_state_clear = apple_atomic_state_clear, .atomic_state_free = apple_atomic_state_free, .atomic_check = drm_atomic_helper_check, - .atomic_commit = drm_atomic_helper_commit, + .atomic_commit = apple_atomic_commit, .fb_create = drm_gem_fb_create, }; @@ -600,9 +797,11 @@ static void apple_atomic_commit_tail(struct drm_atomic_commit *state) drm_atomic_helper_fake_vblank(state); drm_atomic_helper_commit_hw_done(state); drm_atomic_helper_wait_for_flip_done(state->dev, state); + apple_fail_lost_flips(state); drm_atomic_helper_cleanup_planes(state->dev, state); } +/* apple_atomic_commit() calls the tail itself; this serves other helper users. */ static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { .atomic_commit_tail = apple_atomic_commit_tail, }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f56c4423ea247..4669e4fb983c1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -531,6 +531,24 @@ void dcp_drm_crtc_error(struct apple_crtc *crtc, spin_unlock_irqrestore(&drm->event_lock, flags); } +/* + * The flip of the commit whose flip_done is @flip_done never completed: send + * its still pending event as failed. Only that commit's event is sent; a newer + * commit's stays for its own swap. + */ +void dcp_drm_crtc_flip_lost(struct apple_crtc *crtc, + struct completion *flip_done, int error) +{ + unsigned long flags; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event && crtc->event->base.completion == flip_done) { + dcp_failed_page_flip(crtc, crtc->event, error); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index d65367c8af13f..aaaf2918c4b75 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -88,6 +88,8 @@ void apple_crtc_vblank(struct apple_crtc *apple); void dcp_drm_crtc_vblank(struct apple_crtc *crtc); void dcp_drm_crtc_error(struct apple_crtc *crtc, struct drm_crtc_state *state, int error); +void dcp_drm_crtc_flip_lost(struct apple_crtc *crtc, + struct completion *flip_done, int error); int dcp_get_modes(struct drm_connector *connector); enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode); From 0ace6788393262036f3b8ee068cf8b78b074fa35 Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sun, 8 Nov 2026 06:09:55 -0500 Subject: [PATCH 089/392] drm/asahi: g17: Fix a GPU recovery failing the compositor for compute work it dropped When a GPU recovery cannot name the culprit (on M5 every firmware restart is attributed to the firmware, and classify() gives up when a guilty queue has no in-flight owner), it classifies nobody. Compute commands it dropped before they ran then come back from their queue's records as unexecuted (ENODATA), unclassified, and poisons() is true for unclassified ENODATA: the queue's terminal records the error and fails every queue of the VM, and the client's driver reads the VM status as a lost device. A Chromium GPU fault took Hyprland down this way: right after the faulting client's EIO, the log reported "GPU work of a client failed (ENODATA, ...)" for Hyprland too, and zink lost its device. The render side already handles this: groups a restart dropped are classified Spared before they fail with ENODATA. Do the same for compute: - When a compute queue's front comes back unexecuted after crossing a recovery, and the firmware reported no fault on the queue, classify every unclassified command of the queue Spared before quarantining it. All of them are ordered behind the front, so none of them ran. The queue then takes the existing spared path: its commands replay when their records prove they never ran, and otherwise fail with ENODATA without touching the VM status. - Without a recovery in between, a zero-duration record is the firmware's report of the queue's own failed command (see note_fault()). Those commands are classified Guilty and fail the VM as before. A fault report that arrives after the commands were spared as dropped reclassifies them Guilty, stops the queue from being replayed or returned to service, and queues the VM failure the report would have caused. - A compute replay that cannot be published was spared work: classify it Spared (unless guilty) before it fails with ENODATA, as dropped render groups are. - poisons(): ENODATA no longer poisons unless the command is guilty. A command that never ran cannot have faulted the GPU; a blamed render still fails with EIO, and a faulting compute command is classified Guilty first. The faulting client still fails: a blamed render fails with EIO, a watchdog timeout with ETIMEDOUT, and a reported compute fault now classifies its queue's work Guilty. What changes is only work that never ran on a queue nobody blamed. When the recovery attributes nothing, the command that was running when the firmware stopped and comes back with no record is spared too: its fence still fails with ENODATA, but its VM is not poisoned. A spared command is replayed only when its record shows it never ran, and a second fault on a replayed command is blamed on its VM (oldest_spared() counts replays), so this costs at most one more recovery. Completion::signalled() is added for note_fault(): a spared command whose fence already signalled is not reclassified. Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- drivers/gpu/drm/asahi/g17/completion.rs | 16 +++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 50 +++++++++++++++++++ .../drm/asahi/g17/runtime/compute_replay.rs | 5 ++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 0f027e7e06828..1fbe95a53c90f 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -564,9 +564,21 @@ impl Completion { self.interrupted.load(Ordering::Acquire) } - /// Whether this failure makes the VM unusable. Spared work, and interrupted - /// work no recovery found guilty, fail without poisoning it. + /// Whether the fence has been signalled (completed or failed). + pub(crate) fn signalled(&self) -> bool { + self.signalled.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, interrupted + /// work no recovery found guilty, and work that never ran (ENODATA) fail + /// without poisoning it. pub(crate) fn poisons(&self, error: Error) -> bool { + // A command that never ran cannot have faulted the GPU. The queue's own failed + // command also comes back as ENODATA (a zero-duration record); it is classified + // guilty before it fails, by the firmware's fault report or by a recovery. + if error == ENODATA && !self.guilty() { + return false; + } let interrupted = self.interrupted(); (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4dab923c9e1bb..294ad42d9dc2a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -467,6 +467,9 @@ struct Active { observed_at: Option, failure_deferred: bool, qos_pending: bool, + /// Spared because it came back unexecuted after a recovery ([`Queue::classify_unexecuted`]), + /// not by the recovery's own attribution. A later fault report on the queue makes it guilty. + spared_as_dropped: bool, } struct Previous> { binding: Option, @@ -698,6 +701,23 @@ impl Queue { { self.fault_stamp = Some(stamp); } + // The report can arrive after the queue's unexecuted commands were spared as dropped by + // a recovery: the failed command is among them. They are guilty, as without the + // recovery: the VM fails, and the queue is neither replayed nor returned to service. + let mut guilty = None; + for active in self.active.iter_mut().filter(|active| active.spared_as_dropped) { + active.spared_as_dropped = false; + if !active.packet.completion.signalled() { + active.packet.completion.classify(recovery::Class::Guilty); + guilty.get_or_insert_with(|| active.packet.completion.status().clone()); + } + } + if let Some(status) = guilty { + self.spared_quarantine = false; + if self.terminal.is_none() { + self.terminal = Some((status, ENODATA)); + } + } } /// The firmware failed a command of this quarantined queue, and the queue still waits for @@ -999,6 +1019,7 @@ impl Queue { let mut oldest = true; while let Some(front) = self.active.front() { let mut restart = None; + let mut crossed = false; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -1006,6 +1027,7 @@ impl Queue { return Err(EAGAIN); } let recordless = if epoch != ticket.epoch { + crossed = true; Recordless::CrossedRecovery } else if masks.is_none() || (finished_before && oldest) { Recordless::Allowed @@ -1097,6 +1119,9 @@ impl Queue { break; } Err(error) => { + if error == ENODATA { + self.classify_unexecuted(crossed); + } self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); return Ok(()); } @@ -1105,6 +1130,30 @@ impl Queue { Ok(()) } + /// The front command came back unexecuted (ENODATA from [`retirement::poll`]), and every + /// later command of the queue is ordered behind it, so none of them ran either. After a + /// recovery (`crossed`) the recovery dropped them: they cannot have faulted the GPU and are + /// spared, as the render groups a restart drops are, so they fail or replay without + /// poisoning their VM. Without a recovery in between, or once the firmware reported a fault + /// on this queue ([`Self::note_fault`]), the zero-duration record is the firmware's report + /// of the queue's own failed command: the work is guilty and fails its VM as before. Work a + /// recovery already classified keeps its class. + fn classify_unexecuted(&mut self, crossed: bool) { + let dropped = crossed && self.fault_stamp.is_none(); + for active in self.active.iter_mut() { + let completion = &active.packet.completion; + if completion.spared() || completion.guilty() { + continue; + } + if dropped { + completion.classify(recovery::Class::Spared); + active.spared_as_dropped = true; + } else { + completion.classify(recovery::Class::Guilty); + } + } + } + /// Exact terminal witness releases command pins, not installed graph ownership. Spared /// zero-duration work can return its queue to service after settling its accounting. pub(crate) fn release_witnessed( @@ -2328,6 +2377,7 @@ impl Queue { observed_at: None, failure_deferred: false, qos_pending: false, + spared_as_dropped: false, }) .map_err(|_| EBUSY)?; self.retirement_ready = false; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 32b9acef94c0f..4f6b11eac8d4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -96,6 +96,11 @@ impl Firmware { } queue.fail_replays(error); if !queue.owns_packet(&packet) { + // Replays are spared work a recovery dropped before it ran: it fails + // without poisoning its VM, as a dropped render group does. + if !packet.completion.guilty() { + packet.completion.classify(recovery::Class::Spared); + } deferred .push(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; } From 9c1cc06ccc38a07bab38905e22e726a3f4db9651 Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sun, 8 Nov 2026 06:11:35 -0500 Subject: [PATCH 090/392] drm/asahi: g17: Fix fences of work a GPU recovery dropped failing as a lost device or never signalling A fence on a scanout buffer that never signals wedges KMS: the atomic commit worker waits for it with no timeout, so no frame reaches the display again, even across compositor restarts. Work a recovery dropped must signal within a bound, and since it never ran, it must not cost its client the device either. Three paths did not do that: - A compute queue whose commands a recovery dropped waits, spared, for its retirement witness before it replays or fails them. When the queue's job watchdog expires first, timeout() failed them with ETIMEDOUT, and fail_while_owned() reports every error but ENODATA and ECANCELED to the client's VM status: the client lost its device for work it never got to run. They now fail with ENODATA. - That witness needs every entry to hold a ticket. A spared queue with an entry without one would never get it, and the entry's own watchdog does nothing for an unpublished packet. quarantine() now fails such spared work at once with ENODATA, and timeout_head() reports a queue whose spared work still waits, ticketed or not, so that other VMs' watchdogs see it as a failed blocker and defer to it within their bounded renewals rather than fail their own work immediately. - Render groups parked for replay after a recovery that do not publish within 10 s made replay_render_front() return ETIMEDOUT. That failed the GPU event service, which loses every client's device and stops the runtime. The parked groups are innocent work that never ran (replay::decide() parks only those), so they now fail with ENODATA like fail_dropped()'s, spared, and the replay pass goes on. Bounds: no new timeout. A spared compute fence is settled by its job's scheduler watchdog, TIMEOUT_MS (2 s) per expiry; on M5 a job is renewed only while it waits behind an older node of its own context and the firmware's progress word moves (at most 14 renewals), plus 4 for ray traversal and AS builds, so 2 s typically and 38 s at most after the quarantine. A parked render replay keeps its 10 s bound (background_stalled(), the same as the recovery handshake's wait). Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 28 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 22 +++++++++++++++ .../drm/asahi/g17/runtime/render/replay.rs | 7 ++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 294ad42d9dc2a..69d6bacd42986 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -785,9 +785,13 @@ impl Queue { && active.packet.command.long_job() }) } - /// Quarantines without a pending witness no longer describe live pipe order. + /// Quarantines without a pending witness no longer describe live pipe order. Spared work + /// whose fences still wait (`spared_deferred`) does, whether or not every entry holds a + /// ticket: it holds the GPU until the firmware drains it or this queue's watchdog settles it + /// ([`Self::timeout`]), so other VMs' watchdogs see it as a failed blocker and defer to it + /// within their bounded renewals instead of failing their own work at once. pub(crate) fn timeout_head(&self) -> Option { - if self.quarantined && !self.awaiting_witness { + if self.quarantined && !self.awaiting_witness && !self.spared_deferred { return None; } self.active.front().map(|active| crate::g17::timeout::Head { @@ -998,6 +1002,19 @@ impl Queue { self.failure_status_pending = false; self.awaiting_witness = self.active.len != 0 && self.active.iter().all(|a| a.ticket.is_some()); + // Spared work waits for its retirement witness to replay or fail. An entry without a + // ticket never gives one (release_witnessed() validates every ticket), and an + // unpublished entry's own watchdog does nothing, so nothing would bound that wait. Fail + // it now instead: ENODATA, which stays out of the VM status. + if self.spared_deferred && !self.awaiting_witness { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned(active.packet.completion.clone(), ENODATA))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } Ok(terminal) } @@ -2001,6 +2018,11 @@ impl Queue { /// A watchdog settles spared, deferred waiters without changing their VM status. /// Fully ticketed spared work can still return this queue after its drain witness. + /// + /// Spared, deferred work is work a recovery dropped before it ran (the quarantine's + /// ENODATA) whose witness did not come before this queue's job watchdog expired: it fails + /// with ENODATA too. ETIMEDOUT would reach the client as a lost device (`fail_while_owned` + /// reports every error but ENODATA and ECANCELED), for work it never got to run. pub(crate) fn timeout( &mut self, defer: &mut impl FnMut(Deferred) -> Result, @@ -2016,7 +2038,7 @@ impl Queue { if !active.failure_deferred { defer(Deferred::FailedOwned( active.packet.completion.clone(), - ETIMEDOUT, + ENODATA, ))?; active.failure_deferred = true; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 76447a297c5ec..a41e9b48b112a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1784,6 +1784,28 @@ impl Pair { Ok(()) } + /// The parked groups did not publish within the replay bound + /// ([`crate::g17::timeout::background_stalled`], 10 s after they were parked). They are + /// groups a restart dropped before they ran, which `replay::decide` parks only for innocent + /// VMs: like `fail_dropped`'s, they fail with ENODATA without poisoning their VM, instead of + /// the replay pass failing the whole device. + pub(crate) fn fail_stalled_replay( + &mut self, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: replay of {} parked groups stalled; failing them\n", + self.slot, + self.replay.len() + ); + for packet in self.replay.iter() { + if !packet.completion.guilty() { + packet.completion.classify(recovery::Class::Spared); + } + } + self.fail_replay(ENODATA, defer) + } + /// A closed preparation epoch invalidates only this unpublished descriptor. /// Its completion remains available for a retry under the next epoch. pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 499e3858fcca1..7dda54f710eff 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -55,7 +55,12 @@ impl Firmware { return Ok(false); }; if stalled { - return Err(ETIMEDOUT); + // The parked groups' fences must signal; a stalled replay is no reason to lose + // every client's device. + self.with_render(slot, |pair, _| { + pair.fail_stalled_replay(&mut |event| deferred.push(event)) + })?; + return Ok(false); } if busy { return Ok(false); From 9e8fa1e28ac5df677694ceb187b85873eb6cce34 Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sun, 8 Nov 2026 11:19:20 -0500 Subject: [PATCH 091/392] drm/asahi: g17: address the review of the recovery-dropped work fixes Follow-up to the automated review of this pull request, kept as a separate commit for review and squashed on request. drm/asahi: g17: Fix fences of work a GPU recovery dropped failing as a lost device or never signalling A fence on a scanout buffer that never signals wedges KMS: the atomic commit worker waits for it with no timeout, so no frame reaches the display again, even across compositor restarts. Work a recovery dropped must signal within a bound, and since it never ran, it must not cost its client the device either. Three paths did not do that: - A compute queue whose commands a recovery dropped waits, spared, for its retirement witness before it replays or fails them. When the queue's job watchdog expires first, timeout() failed them with ETIMEDOUT, and fail_while_owned() reports every error but ENODATA and ECANCELED to the client's VM status: the client lost its device for work it never got to run. They now fail with ENODATA. - That witness needs every entry to hold a ticket. A spared queue with an entry without one would never get it, and the entry's own watchdog does nothing for an unpublished packet. quarantine() now fails such spared work at once with ENODATA, and timeout_head() reports a queue whose spared work still waits, ticketed or not, so that other VMs' watchdogs see it as a failed blocker and defer to it within their bounded renewals rather than fail their own work immediately. - Render groups parked for replay after a recovery that do not publish within 10 s made replay_render_front() return ETIMEDOUT. That failed the GPU event service, which loses every client's device and stops the runtime. The parked groups are innocent work that never ran (replay::decide() parks only those), so they now fail with ENODATA like fail_dropped()'s, spared, and the replay pass goes on. A replay that publish_render() refused before publishing it (outer work ring full, an owner switch still releasing, a recovery pending) kept its prepared descriptor, so the next pass could not prepare it again and the replay always ran into that bound; failing it then left the descriptor installed, and the pair took no work again. Such a descriptor is now discarded when the publication is refused, so the next pass prepares the group again, and fail_replay() discards one it finds for the front. Bounds: no new timeout. A spared compute fence is settled by its job's scheduler watchdog, TIMEOUT_MS (2 s) per expiry; on M5 a job is renewed only while it waits behind an older node of its own context and the firmware's progress word moves (at most 14 renewals), plus 4 for ray traversal and AS builds, so 2 s typically and 38 s at most after the quarantine. A parked render replay keeps its 10 s bound (background_stalled(), the same as the recovery handshake's wait). drm/asahi: g17: Fix a GPU recovery failing the compositor for compute work it dropped When a GPU recovery cannot name the culprit (on M5 every firmware restart is attributed to the firmware, and classify() gives up when a guilty queue has no in-flight owner), it classifies nobody. Compute commands it dropped before they ran then come back from their queue's records as unexecuted (ENODATA), unclassified, and poisons() is true for unclassified ENODATA: the queue's terminal records the error and fails every queue of the VM, and the client's driver reads the VM status as a lost device. A Chromium GPU fault took Hyprland down this way: right after the faulting client's EIO, the log reported "GPU work of a client failed (ENODATA, ...)" for Hyprland too, and zink lost its device. The render side already handles this: groups a restart dropped are classified Spared before they fail with ENODATA. Do the same for compute: - When a compute queue's front comes back unexecuted after crossing a recovery, and the firmware reported no fault on the queue, classify every unclassified command of the queue Spared before quarantining it. All of them are ordered behind the front, so none of them ran. The queue then takes the existing spared path: its commands replay when their records prove they never ran, and otherwise fail with ENODATA Compile-checked with W=1 CLIPPY=1 over drivers/gpu/drm/{asahi,apple}: no new warnings against the base; checkpatch --strict clean; rustfmt unchanged. Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 11 +++++++---- drivers/gpu/drm/asahi/g17/queue/render.rs | 11 +++++++++++ drivers/gpu/drm/asahi/g17/runtime/render/replay.rs | 9 ++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 69d6bacd42986..9f847eb383931 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -468,7 +468,8 @@ struct Active { failure_deferred: bool, qos_pending: bool, /// Spared because it came back unexecuted after a recovery ([`Queue::classify_unexecuted`]), - /// not by the recovery's own attribution. A later fault report on the queue makes it guilty. + /// not by the recovery's own attribution, or a replay of spared work, which runs again from + /// the start. A later fault report on the queue makes it guilty. spared_as_dropped: bool, } struct Previous> { @@ -702,8 +703,9 @@ impl Queue { self.fault_stamp = Some(stamp); } // The report can arrive after the queue's unexecuted commands were spared as dropped by - // a recovery: the failed command is among them. They are guilty, as without the - // recovery: the VM fails, and the queue is neither replayed nor returned to service. + // a recovery, or while replays of spared work run: the failed command is among them. + // They are guilty, as without the recovery: the VM fails, and the queue is neither + // replayed nor returned to service. let mut guilty = None; for active in self.active.iter_mut().filter(|active| active.spared_as_dropped) { active.spared_as_dropped = false; @@ -2399,7 +2401,8 @@ impl Queue { observed_at: None, failure_deferred: false, qos_pending: false, - spared_as_dropped: false, + // A replay keeps its spared class: its own fault must still make it guilty. + spared_as_dropped: packet.completion.spared(), }) .map_err(|_| EBUSY)?; self.retirement_ready = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index a41e9b48b112a..81948e738769c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1778,6 +1778,17 @@ impl Pair { defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { self.replay_since = None; + // A descriptor prepared for the front and never published goes with it, or the pair + // would take no work again. + if let Some(front) = self.replay.front().cloned() { + if self + .prepared + .as_ref() + .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, &front)) + { + self.discard_prepared(&front)?; + } + } while let Some(packet) = self.replay.pop() { defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 7dda54f710eff..c4fb20fa0952a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -110,9 +110,16 @@ impl Firmware { self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; Ok(true) } - Err(error) if error == EBUSY || error == EAGAIN => Ok(false), Err(error) => { self.with_render(slot, |pair, _| { + // A refusal before publication leaves the descriptor prepared; the next + // pass prepares the group again, which it cannot while that one is held. + if !packet.is_published() { + pair.discard_prepared(&packet)?; + } + if error == EBUSY || error == EAGAIN { + return Ok(()); + } pair.fail_replay(error, &mut |event| deferred.push(event)) })?; Ok(false) From f2f745f21544211e1598618c6aa2f9599a4f0878 Mon Sep 17 00:00:00 2001 From: Rolando Trevino Date: Sun, 8 Nov 2026 11:20:19 -0500 Subject: [PATCH 092/392] drm/apple: address the review of the bounded fence wait Follow-up to the automated review of this pull request, kept as a separate commit for review and squashed on request. drm/apple: Fix a GPU fence that never signals wedging the display drm_atomic_helper_commit() waits for every plane's in-fence with dma_fence_wait() and no timeout: in the commit worker before the driver's commit tail, and, for blocking commits, before the swap. The in-fence is the scanout buffer's implicit write fence (zink imports its GPU job's fence into each exported dma-buf) or the compositor's IN_FENCE_FD. When a GPU fence never signals, after a GPU recovery for example, the worker sleeps for good: commit_planes() never runs, no swap reaches DCP, the commit's flip event never comes, and every later nonblocking commit fails with -EBUSY (stall_checks(), only a debug message). The 10 s flip_done timeout is never reached, so nothing is logged. The panel stays black across compositor restarts, which see "Cannot commit when a page-flip is awaiting", until a reboot. Commit through a copy of drm_atomic_helper_commit() whose fence wait is bounded: all of a commit's planes share one second, and a fence that does not signal in time is dropped with a rate-limited warning naming its driver and timeline. The buffer is scanned out as it is, so a GPU bug costs a stale or torn frame instead of the panel. The copy keeps the helper's sequence (setup, prepare, the interruptible pre-swap wait for blocking commits, swap, the worker's wait for dependencies, the tail, cleanup) and its fence deadline hint, and leaves out only the self-refresh accounting, which this driver does not use. Once the commit goes on, its flip completes as usual. If DCP then does not complete the swap within drm_atomic_helper_wait_for_flip_done()'s 10 s, the commit's event stayed pending and a compositor would still wait for it before committing again. Send that event as failed (-ETIMEDOUT on its out-fence) after the wait; only the event that belongs to this commit is sent. The commit then leaves its CRTC's list at cleanup_done as before, so later nonblocking commits are not held by it either. The swap of that flip may still complete after the next commit installed its event. The H17P and desktop 26.x paths match a completion to its present; on the older firmware (12.3, 13.5, the 14.7 template, H17G) a completion retires whatever flip is pending, so there the lost swap's completion is now ignored. The framebuffers the lost swap displaced are unarmed, as the swap watchdog does for a dropped swap, so that the next swap re-arms them and its completion releases them; they were otherwise kept until the display powered off. Compile-checked with W=1 CLIPPY=1 over drivers/gpu/drm/{asahi,apple}: no new warnings against the base; checkpatch --strict clean; rustfmt unchanged. Signed-off-by: Rolando Trevino Assisted-by: LLM Claude --- drivers/gpu/drm/apple/apple_drv.c | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 10 +++++++ drivers/gpu/drm/apple/dcp.c | 39 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 8 ++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index fbd0325b963d2..1981a33aff91f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -652,7 +652,7 @@ static int apple_wait_for_fences(struct drm_device *dev, WARN_ON(!new_plane_state->fb); - left = time_before(jiffies, deadline) ? deadline - jiffies : 0; + left = max_t(long, (long)(deadline - jiffies), 0); ret = dma_fence_wait_timeout(fence, pre_swap, left); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5b9f50b60d300..26ff331cc27d0 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -73,6 +73,7 @@ struct apple_dcp_typec_port; void dcp_swap_watchdog_arm(struct apple_dcp *dcp); void dcp_swap_watchdog_complete(struct apple_dcp *dcp); +bool dcp_swap_lost(struct apple_dcp *dcp, u32 swap_id); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); bool dcp_typec_relink(struct apple_dcp *dcp); @@ -573,6 +574,15 @@ struct apple_dcp { u32 last_swap_id; /* last_swap_id is only meaningful after the first swap completes */ bool have_swap_complete; + /* swap id of the last started swap */ + u32 started_swap_id; + /* + * A started swap whose flip a commit gave up on + * (dcp_drm_crtc_flip_lost()); event_lock. Firmware that does not track + * its presents ignores its completion, see dcp_swap_lost(). + */ + u32 lost_swap_id; + bool swap_lost; ktime_t swap_start; u64 swap_submit_timestamp; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4669e4fb983c1..efbe331d2467a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -535,18 +535,57 @@ void dcp_drm_crtc_error(struct apple_crtc *crtc, * The flip of the commit whose flip_done is @flip_done never completed: send * its still pending event as failed. Only that commit's event is sent; a newer * commit's stays for its own swap. + * + * The swap started last is that flip's. Its displaced framebuffers are + * unarmed, as the swap watchdog does, so that the next swap arms them and its + * completion releases them. Its completion may still come, after the next + * commit installed its event: firmware that does not track its presents + * ignores it (dcp_swap_lost()), so that it does not retire the newer flip. */ void dcp_drm_crtc_flip_lost(struct apple_crtc *crtc, struct completion *flip_done, int error) { + struct apple_dcp *dcp = platform_get_drvdata(crtc->dcp); unsigned long flags; + bool lost = false; + u32 swap_id = 0; spin_lock_irqsave(&crtc->base.dev->event_lock, flags); if (crtc->event && crtc->event->base.completion == flip_done) { dcp_failed_page_flip(crtc, crtc->event, error); crtc->event = NULL; + swap_id = READ_ONCE(dcp->started_swap_id); + dcp->lost_swap_id = swap_id; + dcp->swap_lost = true; + lost = true; } spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); + + if (lost) + dcp_unarm_retained_framebuffers(dcp, swap_id); +} + +/* + * Is @swap_id the swap whose flip dcp_drm_crtc_flip_lost() gave up on? On + * firmware that does not track its presents, its completion would retire + * whichever flip is pending; it is ignored instead. Swaps complete in order, + * so once another swap completed, the lost one will not. + */ +bool dcp_swap_lost(struct apple_dcp *dcp, u32 swap_id) +{ + unsigned long flags; + bool lost; + + spin_lock_irqsave(&dcp->crtc->base.dev->event_lock, flags); + lost = dcp->swap_lost && dcp->lost_swap_id == swap_id; + dcp->swap_lost = false; + spin_unlock_irqrestore(&dcp->crtc->base.dev->event_lock, flags); + + if (lost) + dev_warn_ratelimited(dcp->dev, + "completion for lost present %u ignored\n", + swap_id); + return lost; } void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 82e5310f3fd6e..19615309aa2b9 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -649,6 +649,12 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, if (!dcp_desktop_present_complete(dcp, resp->swap_id)) return; dcp_backlight_desktop_finish(dcp, resp->swap_id, false, true); +#else + /* Nor can a lost flip's on firmware that does not track its presents. */ + if ((!DCP_FW_H17(DCP_FW_VER) || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) && + dcp_swap_lost(dcp, resp->swap_id)) + return; #endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; @@ -2088,6 +2094,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, dcp_prepare_clear_swap(dcp, &DCP_FW_SWAP(dcp)); #endif DCP_FW_SWAP(dcp).swap.swap_id = resp->swap_id; + WRITE_ONCE(dcp->started_swap_id, resp->swap_id); if (dcp_present_retires(dcp)) dcp_arm_retained_framebuffers(dcp, resp->swap_id); @@ -2943,6 +2950,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) return; } DCP_FW_SWAP(dcp).swap.swap_id = resp->swap_id; + WRITE_ONCE(dcp->started_swap_id, resp->swap_id); if (dcp_present_retires(dcp)) dcp_arm_retained_framebuffers(dcp, resp->swap_id); From ae1b7ec4049cfbc4dca8155c2c8e11b16c606c3a Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:00:35 -0600 Subject: [PATCH 093/392] perf: avoid charging mmap aliases twice Charge normal-ring and AUX pages only on first allocation. Repeated aliases share the existing charge; the final close releases it once. A failed alias mapping previously left an extra per-user charge. Add a source-level host regression for duplicate and failed aliases, metadata-only rings, limit splitting and alternate close orders. Against the exact base the harness reports 16 failing alias cases; on this tree it passes 26/26. A focused x86-64 core.o build passed. This is single- user, single-mm, single-threaded host testing with stubbed kernel plumbing; it is not runtime or hardware validation. Fixes: 0c8a4e4139ad ("perf/core: Further simplify perf_mmap()") Assisted-by: LLM Signed-off-by: Joshua Warren --- kernel/events/core.c | 15 +- .../asahi/perf-mmap-alias-lifecycle-tests.py | 1504 +++++++++++++++++ 2 files changed, 1516 insertions(+), 3 deletions(-) create mode 100755 tools/asahi/perf-mmap-alias-lifecycle-tests.py diff --git a/kernel/events/core.c b/kernel/events/core.c index db8783602d08e..93574bee7b4b2 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7349,9 +7349,11 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (refcount_inc_not_zero(&event->rb->mmap_count)) { /* * Success -- managed to mmap() the same buffer - * multiple times. + * multiple times. The alias shares the ring's + * already-locked pages; the existing charge stays + * as-is and is removed once, when the last mapping + * closes. */ - perf_mmap_account(vma, user_extra, extra); refcount_inc(&event->mmap_count); return 0; } @@ -7463,9 +7465,16 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) refcount_set(&rb->aux_mmap_count, 1); rb->aux_mmap_locked = extra; + + /* + * Charge the AUX area once, at first allocation. The alias + * branch above shares the already-locked pages and adds no + * charge; the single charge is removed when the last AUX + * mapping closes. + */ + perf_mmap_account(vma, user_extra, extra); } - perf_mmap_account(vma, user_extra, extra); refcount_inc(&event->mmap_count); return 0; diff --git a/tools/asahi/perf-mmap-alias-lifecycle-tests.py b/tools/asahi/perf-mmap-alias-lifecycle-tests.py new file mode 100755 index 0000000000000..55f385052ea29 --- /dev/null +++ b/tools/asahi/perf-mmap-alias-lifecycle-tests.py @@ -0,0 +1,1504 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Duplicate-alias and failed-alias lifecycle regression tests for the perf +mmap locked-memory accounting (normal ring and AUX area). + +The tool compiles the accounting code straight out of each tree under test +(kernel/events/core.c + internal.h: perf_mmap, perf_mmap_rb, perf_mmap_aux, +perf_mmap_close, perf_mmap_account, perf_mmap_unaccount, perf_mmap_calc_limits +and the internal.h inline helpers) into a host harness and drives full +lifecycles through it: + + - duplicate aliases of the normal ring: charge per alias, uncharge once at + final close (user-reported leak), in both close orders -- aliases first + and the original mapping first while every alias is still live, + - duplicate aliases of the AUX area: charge per alias, uncharge once at the + last AUX close (user-reported leak), in both close orders -- AUX first + and the normal ring VMA first while the AUX area stays live, + - failed aliases: AUX first-map failures (-EPERM limit check, rb_alloc_aux + failure, early -EINVAL), and map_range() failures through the real + perf_mmap() dispatcher error path, both for a first mapping (minimal + perf_mmap_close()) and for an alias (full perf_mmap_close()), + - real limit splitting through the tree's own perf_mmap_calc_limits() + (partial spill and full spill into pinned_vm), + - ring geometries: full data ring, metadata-only ring, and (user4k trees) + a sub-native-page data ring on a 4K-page mm. + +This is a source-level test: the extracted C is the tree's real code, but +VMA/file/user/mutex plumbing, map_range(), rb_alloc()/rb_alloc_aux() page +backing and ring_buffer_attach() are signature-faithful host stubs, it is +single-threaded, and nothing runs in a kernel. It is not evidence about +hardware or a booted kernel; it is evidence about the source's charge/ +uncharge/refcount behaviour. + +Assertions encode the desired invariant (every counter and refcount returns +to its initial value when a lifecycle completes; perf_mlock and locked_vm +are checked individually, so a transfer between them cannot mask a leak). +A scenario that ends with +a nonzero counter therefore FAILS and prints the exact leaked page count -- +leaks are reported, not hidden. The tool exits nonzero if any invariant +broke: a tree that charges every alias fails the duplicate-alias scenarios; +a tree whose aliases share the first map's single charge balances every +lifecycle and passes. +""" + +from pathlib import Path +import argparse +import re +import subprocess +import sys +import tempfile +from string import Template + +HERE = Path(__file__).resolve().parent + +DEFAULT_TREES = [base for base in HERE.parents + if (base / 'kernel/events/core.c').is_file()] + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--tree', type=Path, action='append', dest='trees', + default=None, + help='kernel tree holding kernel/events ' + '(repeatable; default: the kernel tree this test ' + 'lives in)') +parser.add_argument('--cc', default='cc', help='C compiler (default: cc)') +args = parser.parse_args() +if args.trees is None: + if not DEFAULT_TREES: + parser.error('--tree is required: no kernel/events/core.c found ' + 'above %s' % HERE) + args.trees = DEFAULT_TREES + + +def definition(text, name): + """Return the C text of a function definition, not of its prototype.""" + for match in re.finditer(r'\b' + re.escape(name) + r'\s*\(', text): + index = match.end() - 1 + depth = 0 + while index < len(text): + if text[index] == '(': + depth += 1 + elif text[index] == ')': + depth -= 1 + if depth == 0: + break + index += 1 + brace = text.find('{', index) + semi = text.find(';', index) + if brace == -1 or (semi != -1 and semi < brace): + continue + if '*/' in text[match.start():brace]: + continue # match sat inside a comment; find the real definition + depth = 0 + for position in range(brace, len(text)): + if text[position] == '{': + depth += 1 + elif text[position] == '}': + depth -= 1 + if depth == 0: + start = text.rfind('\n', 0, match.start()) + 1 + return text[start:position + 1] + return None + + +def rb_alloc_profile(ring): + """Detect the rb_alloc() shape: user4k data_size form or plain nr_pages.""" + func = definition(ring, 'rb_alloc') + if func is None: + raise SystemExit('no rb_alloc() definition in ring_buffer.c') + head = func.split(')')[0] + user4k = 'data_size' in head + match = re.search(r'int nr_pages\s*=\s*([^;]+);', func) + nr_pages_expr = match.group(1).strip() if match else 'nr_pages' + match = re.search(r'rb->data_page_shift\s*=\s*([^;]+);', func) + return user4k, nr_pages_expr, match.group(1).strip() if match else None + + +def rb_init_lines(ring): + """The mmap_count/mmap_user initialisation the tree does at rb_alloc.""" + lines = [] + if re.search(r'rb->mmap_user\s*=\s*get_current_user\(\);', ring): + lines.append('\trb->mmap_user = current_user();') + if re.search(r'refcount_set\(&rb->mmap_count,\s*1\);', ring): + lines.append('\trefcount_set(&rb->mmap_count, 1);') + return '\n'.join(lines) + + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include + +#define RB_USER4K $RB_USER4K + +typedef long atomic_long_t; +typedef long long atomic64_t; +typedef unsigned long long u64; +typedef unsigned long pgoff_t; + +#define PAGE_SHIFT 14 +#define PAGE_SIZE (1UL << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define ilog2(n) (31 - __builtin_clz((unsigned int)(n))) +#define READ_ONCE(x) (x) +#define IS_ALIGNED(x, a) (((x) & ((a) - 1)) == 0) +#define VM_SHARED 0x1 +#define VM_WRITE 0x2 +#define VM_DONTCOPY 0x0 +#define VM_DONTEXPAND 0x0 +#define VM_DONTDUMP 0x0 +#define CAP_IPC_LOCK 14 +#define RLIMIT_MEMLOCK 0 +#define PERF_EVENT_STATE_REVOKED (-1) + +static int warn_count, warn_printed; +static void warn_hit(int condition, const char *tag, int line) +{ + if (!condition) + return; + warn_count++; + if (warn_printed++ < 20) + printf(" WARN %s line %d fired\n", tag, line); +} +#define WARN_ON(c) warn_hit(!!(c), "WARN_ON", __LINE__) +#define WARN_ON_ONCE(c) warn_hit(!!(c), "WARN_ON_ONCE", __LINE__) + +/* --- environment knobs the real code reads ---------------------------- */ +static long sysctl_perf_event_mlock = 256UL << 20; /* KiB, kernel units */ +static unsigned long h_rlimit_memlock = 1UL << 40; /* bytes */ +static int h_paranoid = 1; +static int h_capable = 0; +static int h_cpus = 1; +static int rb_alloc_aux_fail = 0; +static int map_range_fail = 0; + +static int num_online_cpus(void) { return h_cpus; } +static unsigned long rlimit(int which) { (void)which; return h_rlimit_memlock; } +static int perf_is_paranoid(void) { return h_paranoid; } +static int capable(int cap) { (void)cap; return h_capable; } + +/* --- shims ------------------------------------------------------------ */ +struct user_struct { + atomic_long_t perf_mlock; + atomic_long_t locked_vm; +}; +static struct user_struct test_user; +static struct user_struct *current_user(void) { return &test_user; } +#define get_current_user() current_user() + +struct mm_struct { + atomic64_t pinned_vm; + unsigned int page_shift; +}; +static unsigned int mm_page_shift(struct mm_struct *mm) { return mm->page_shift; } +static unsigned long mm_page_size(struct mm_struct *mm) { return 1UL << mm->page_shift; } + +struct file; +struct perf_event; +struct perf_buffer; +struct vm_operations_struct; + +struct vm_area_struct { + unsigned long vm_start, vm_end; + unsigned long vm_pgoff; + unsigned long vm_flags; + unsigned int vm_subpage_offset; + struct mm_struct *vm_mm; + struct file *vm_file; + const struct vm_operations_struct *vm_ops; +}; +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ + return vma->vm_subpage_offset; +} +#define vm_flags_set(v, f) ((void)(f)) + +struct file { void *private_data; }; + +struct perf_event_mmap_page { + u64 aux_offset; + u64 aux_size; +}; + +struct list_head { struct list_head *next, *prev; }; +#define INIT_LIST_HEAD(l) do { (l)->next = (l)->prev = (l); } while (0) +static void list_add_rcu(struct list_head *new_, struct list_head *head) +{ + new_->next = head->next; + new_->prev = head; + head->next->prev = new_; + head->next = new_; +} +static void list_del_rcu(struct list_head *entry) +{ + entry->prev->next = entry->next; + entry->next->prev = entry->prev; + entry->prev = entry->next = entry; +} +#define container_of_(p, t, m) ((t *)((char *)(p) - __builtin_offsetof(t, m))) +#define list_for_each_entry_rcu(pos, head, m) \ + for (pos = container_of_((head)->next, typeof(*pos), m); \ + &pos->m != (head); \ + pos = container_of_(pos->m.next, typeof(*pos), m)) +static void rcu_read_lock(void) { } +static void rcu_read_unlock(void) { } + +typedef struct { int refs; } refcount_t; +static void refcount_set(refcount_t *r, int v) { r->refs = v; } +static int refcount_read(refcount_t *r) { return r->refs; } +static bool refcount_inc_not_zero(refcount_t *r) +{ + if (r->refs <= 0) + return false; + r->refs++; + return true; +} +static void refcount_inc(refcount_t *r) { assert(r->refs > 0); r->refs++; } +static void refcount_dec(refcount_t *r) { assert(r->refs > 0); r->refs--; } +static bool refcount_dec_and_test(refcount_t *r) +{ + assert(r->refs > 0); + return --r->refs == 0; +} + +struct mutex { int locked; }; +static void mutex_lock(struct mutex *m) { assert(!m->locked); m->locked = 1; } +static void mutex_unlock(struct mutex *m) { assert(m->locked); m->locked = 0; } +static void mutex_init(struct mutex *m) { m->locked = 0; } +static void h_mutex_unlock_cb(struct mutex **m) +{ + if (*m) + mutex_unlock(*m); /* scoped_guard step already nulled it */ +} +static bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *m) +{ + assert(r->refs > 0); + if (--r->refs != 0) + return false; + mutex_lock(m); + return true; +} +#define H_CAT_(a, b) a##b +#define H_CAT(a, b) H_CAT_(a, b) +#define scoped_guard(scope, lock) \ + for (struct mutex *H_CAT(h_sgl_, __LINE__) \ + __attribute__((cleanup(h_mutex_unlock_cb))) = \ + (mutex_lock(lock), (struct mutex *)(lock)), \ + *H_CAT(h_sgq_, __LINE__) = H_CAT(h_sgl_, __LINE__); \ + H_CAT(h_sgq_, __LINE__) != NULL; \ + H_CAT(h_sgq_, __LINE__) = NULL) +/* The tree writes guard(mutex)(&lock): the expansion must end with the + * guard function so the trailing parenthesised group becomes its call. */ +static struct mutex *h_mutex_guard_fn(struct mutex *m) +{ + mutex_lock(m); + return m; +} +#define guard(scope) \ + struct mutex *H_CAT(h_g_, __LINE__) \ + __attribute__((cleanup(h_mutex_unlock_cb))) = h_mutex_guard_fn + +#define RING_BUFFER_WRITABLE 0x01 +#define atomic_long_sub(v, p) (*(p) -= (v)) +#define atomic_long_add(v, p) (*(p) += (v)) +#define atomic_long_set(p, v) (*(p) = (v)) +#define atomic_long_read(p) (*(p)) +#define atomic64_sub(v, p) (*(p) -= (v)) +#define atomic64_add(v, p) (*(p) += (v)) +#define atomic64_read(p) (*(p)) +#define atomic_long_inc_not_zero(p) \ + ({ long *h_p_ = (long *)(p); bool h_r_ = false; \ + if (*h_p_ > 0) { (*h_p_)++; h_r_ = true; } \ + h_r_; }) +#define likely(x) (x) +#define unlikely(x) (x) + +static bool is_power_of_2(unsigned long n) +{ + return n != 0 && (n & (n - 1)) == 0; +} + +typedef void (*mapped_f)(struct perf_event *, struct mm_struct *); +enum perf_event_mapped_state { event_unmapped, event_mapped }; +static inline mapped_f get_mapped(struct perf_event *event, int which) +{ + (void)event; (void)which; + return NULL; +} + +struct vm_operations_struct { int unused_; }; +static const struct vm_operations_struct perf_mmap_vmops = { 0 }; + +/* --- tree shapes ------------------------------------------------------- */ +struct perf_buffer { + refcount_t refcount; + refcount_t mmap_count; + refcount_t aux_mmap_count; + refcount_t aux_refcount; + unsigned long mmap_locked; + unsigned long aux_mmap_locked; + struct user_struct *mmap_user; + struct list_head event_list; + struct mutex aux_mutex; + int nr_pages; + int aux_nr_pages; + long aux_watermark; + struct perf_event_mmap_page *user_page; +#if RB_USER4K + unsigned int data_page_shift; + unsigned int mmap_page_shift; + u64 aux_offset; +#else + unsigned long aux_pgoff; + int page_order; +#endif +}; + +struct perf_event_context { void *parent_ctx; }; +struct perf_event_attr { + long watermark; + long wakeup_watermark; + long aux_watermark; + int inherit; +}; +struct perf_event { + int cpu; + int state; + struct perf_event_attr attr; + struct perf_event_context *ctx; + /* atomic_long_t as in the tree: the rb->event_list walk in + * perf_mmap_close() takes a ref on every event, so the stub must + * hold list-sized counts */ + atomic_long_t refcount; + refcount_t mmap_count; + struct mutex mmap_mutex; + struct perf_buffer *rb; + struct list_head rb_entry; +}; + +/* --- stub declarations (definitions below the extracted helpers) ------- */ +$rb_alloc_decl +$rb_alloc_aux_decl +static void rb_free_aux(struct perf_buffer *rb); +static void rb_free(struct perf_buffer *rb); +static struct perf_buffer *ring_buffer_get(struct perf_event *event); +static void ring_buffer_put(struct perf_buffer *rb); +static void ring_buffer_attach(struct perf_event *event, struct perf_buffer *rb); +static void perf_event_update_time(struct perf_event *event); +static void perf_event_init_userpage(struct perf_event *event); +static void perf_event_update_userpage(struct perf_event *event); +static void perf_pmu_output_stop(struct perf_event *event); +static int security_perf_event_read(struct perf_event *event); +static void put_event(struct perf_event *event); +static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma); + +/* --- accounting code taken from the tree under test -------------------- */ +$helpers + +/* --- harness stub definitions ------------------------------------------ */ +static int rb_allocs, rb_frees, aux_allocs, aux_frees, map_range_calls; + +#if RB_USER4K +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) +{ + struct perf_buffer *rb; + int nr_pages = $rb_nr_pages; + + (void)watermark; (void)cpu; (void)flags; + rb = calloc(1, sizeof(*rb) + nr_pages * sizeof(void *)); + if (!rb) + return NULL; + rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; +$rb_data_shift + rb->user_page = calloc(1, sizeof(*rb->user_page)); + if (!rb->user_page) { + free(rb); + return NULL; + } +$rb_init + INIT_LIST_HEAD(&rb->event_list); + mutex_init(&rb->aux_mutex); + refcount_set(&rb->refcount, 1); + rb_allocs++; + return rb; +} +#else +struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +{ + struct perf_buffer *rb; + + (void)watermark; (void)cpu; (void)flags; + rb = calloc(1, sizeof(*rb) + nr_pages * sizeof(void *)); + if (!rb) + return NULL; + rb->nr_pages = nr_pages; + rb->page_order = 0; + rb->user_page = calloc(1, sizeof(*rb->user_page)); + if (!rb->user_page) { + free(rb); + return NULL; + } +$rb_init + INIT_LIST_HEAD(&rb->event_list); + mutex_init(&rb->aux_mutex); + refcount_set(&rb->refcount, 1); + rb_allocs++; + return rb; +} +#endif + +$rb_alloc_aux_def + +static void rb_free_aux(struct perf_buffer *rb) +{ + assert(rb->aux_nr_pages); + rb->aux_nr_pages = 0; + refcount_set(&rb->aux_refcount, 0); + aux_frees++; +} + +static void rb_free(struct perf_buffer *rb) +{ + rb_frees++; + free(rb->user_page); + free(rb); +} + +/* Mirrors the tree's ring_buffer_attach(): the rb_alloc() initial refcount + * IS the first event's attach reference, attach() adds no reference, and + * detach puts the old rb (which frees it when it hits zero). */ +static void ring_buffer_attach(struct perf_event *event, struct perf_buffer *rb) +{ + struct perf_buffer *old_rb = NULL; + + if (rb && rb == event->rb) + return; + if (event->rb) { + old_rb = event->rb; + list_del_rcu(&event->rb_entry); + } + event->rb = rb; + if (rb) + list_add_rcu(&event->rb_entry, &rb->event_list); + if (old_rb) + ring_buffer_put(old_rb); +} + +static struct perf_buffer *ring_buffer_get(struct perf_event *event) +{ + struct perf_buffer *rb = event->rb; + + if (rb) + refcount_inc(&rb->refcount); + return rb; +} + +static void ring_buffer_put(struct perf_buffer *rb) +{ + if (refcount_dec_and_test(&rb->refcount)) + rb_free(rb); +} + +static void perf_event_update_time(struct perf_event *event) { (void)event; } +static void perf_event_init_userpage(struct perf_event *event) { (void)event; } +static void perf_event_update_userpage(struct perf_event *event) { (void)event; } +static void perf_pmu_output_stop(struct perf_event *event) { (void)event; } +static int security_perf_event_read(struct perf_event *event) { (void)event; return 0; } +static void put_event(struct perf_event *event) { (void)event; } + +static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) +{ + (void)rb; (void)vma; + map_range_calls++; + if (map_range_fail) + return -ENOMEM; + return 0; +} + +/* --- lifecycle driver --------------------------------------------------- */ +static struct perf_event ev; +static struct perf_event_context ctx; +static struct mm_struct mm; +static struct file file; + +static void reset_all(unsigned int mm_shift) +{ + memset(&ev, 0, sizeof(ev)); + memset(&ctx, 0, sizeof(ctx)); + memset(&mm, 0, sizeof(mm)); + memset(&file, 0, sizeof(file)); + ev.cpu = 0; + ev.ctx = &ctx; + ev.state = 0; + mutex_init(&ev.mmap_mutex); + atomic_long_set(&ev.refcount, 2); + mm.page_shift = mm_shift; + file.private_data = &ev; + test_user.perf_mlock = 0; + test_user.locked_vm = 0; + mm.pinned_vm = 0; + rb_allocs = rb_frees = aux_allocs = aux_frees = 0; + map_range_calls = 0; + warn_count = 0; + warn_printed = 0; + rb_alloc_aux_fail = 0; + map_range_fail = 0; + sysctl_perf_event_mlock = 256UL << 20; + h_rlimit_memlock = 1UL << 40; + h_paranoid = 1; + h_capable = 0; + h_cpus = 1; +} + +/* Point user_lock_limit at `pages` pages for one CPU (kernel units: KiB). */ +static void set_user_lock_limit(unsigned long pages) +{ + sysctl_perf_event_mlock = (long)pages << (PAGE_SHIFT - 10); +} + +static struct vm_area_struct make_vma_bytes(unsigned long bytes, unsigned long pgoff) +{ + struct vm_area_struct v; + + memset(&v, 0, sizeof(v)); + v.vm_start = 0x100000000UL; + v.vm_end = v.vm_start + bytes; + v.vm_pgoff = pgoff; + v.vm_flags = VM_SHARED | VM_WRITE; + v.vm_mm = &mm; + v.vm_file = &file; + return v; +} + +static void dump(const char *tag) +{ + struct perf_buffer *rb = ev.rb; + + printf(" %-26s perf_mlock=%ld locked_vm=%ld pinned=%ld " + "ev.mmap_count=%d rb->mmap_count=%d rb=%d " + "alloc=%d/%d aux=%d/%d warns=%d\n", tag, + test_user.perf_mlock, test_user.locked_vm, (long)mm.pinned_vm, + ev.mmap_count.refs, rb ? rb->mmap_count.refs : -1, rb != NULL, + rb_allocs, rb_frees, aux_allocs, aux_frees, warn_count); +} + +static long user_perf_mlock(void) { return test_user.perf_mlock; } +static long user_locked_vm(void) { return test_user.locked_vm; } + +/* The tree charges exactly one user counter ($charge_counter) and must + * leave the other ($spare_counter) untouched: assert both individually, + * so a transfer between them cannot pose as unchanged or balanced. */ +static int user_charge_broken(const char *what, long want) +{ + int bad = 0; + + if (user_$charge_counter() != want) { + printf(" OBSERVED %s: $charge_counter %ld, expected %ld\n", + what, user_$charge_counter(), want); + bad = 1; + } + if (user_$spare_counter() != 0) { + printf(" OBSERVED %s: $spare_counter nonzero (%ld), expected 0\n", + what, user_$spare_counter()); + bad = 1; + } + return bad; +} + +static int invariants_broken(const char *what) +{ + int bad = 0; + struct perf_buffer *rb = ev.rb; + + /* each user counter individually: a +leak on one and a -leak on + * the other would hide behind a summed total */ + if (user_perf_mlock()) { + printf(" OBSERVED %s: perf_mlock leaked %ld pages\n", + what, user_perf_mlock()); + bad = 1; + } + if (user_locked_vm()) { + printf(" OBSERVED %s: locked_vm leaked %ld pages\n", + what, user_locked_vm()); + bad = 1; + } + if (mm.pinned_vm) { + printf(" OBSERVED %s: pinned_vm leaked %ld pages\n", + what, (long)mm.pinned_vm); + bad = 1; + } + if (rb_allocs != rb_frees || aux_allocs != aux_frees) { + printf(" OBSERVED %s: allocation imbalance rb %d/%d aux %d/%d\n", + what, rb_allocs, rb_frees, aux_allocs, aux_frees); + bad = 1; + } + if (ev.mmap_count.refs != 0 || rb != NULL || ev.rb != NULL) { + printf(" OBSERVED %s: stale refs ev.mmap_count=%d rb=%d\n", + what, ev.mmap_count.refs, ev.rb != NULL); + bad = 1; + } + if (warn_count) { + printf(" OBSERVED %s: %d WARN_ON fires\n", what, warn_count); + bad = 1; + } + return bad; +} + +/* Assert the user counters and pinned_vm individually at an intermediate + * step, so a charge/uncharge mismatch on either counter is reported on + * its own instead of only via the final zero check. */ +static int check_split(const char *what, long want_user, long want_pinned) +{ + int bad = user_charge_broken(what, want_user); + + if (mm.pinned_vm != want_pinned) { + printf(" OBSERVED %s: pinned_vm %ld, expected %ld\n", + what, (long)mm.pinned_vm, want_pinned); + bad |= 1; + } + return bad; +} + +/* --- scenarios ---------------------------------------------------------- + * A ring geometry is (vma_pages, mm_shift): vma_pages = data pages + 1 + * metadata page; vma_pages == 1 is a metadata-only ring; the user4k + * sub-native geometry uses a 4K-page mm so the data chunk (4K) is smaller + * than the native (16K) page. charge = pages the tree charges for the + * first ring mmap. split = pages of the first charge spilled into + * pinned_vm via the tree's own perf_mmap_calc_limits(): 0 none, >0 partial, + * -1 full spill. + */ +static long ring_charge(unsigned long vma_pages, unsigned long mm_shift) +{ +#if RB_USER4K + return DIV_ROUND_UP((long)(vma_pages - 1) << mm_shift, PAGE_SIZE) + 1; +#else + return vma_pages; +#endif +} + +static void setup_limits(long split) +{ + if (split == 0) + return; /* default generous limit, no split */ + if (split < 0) { + set_user_lock_limit(0); /* full spill */ + return; + } + set_user_lock_limit(charge_expect - split); +} + +static int sc_rb_baseline(unsigned long vma_pages, unsigned long mm_shift, long split) +{ + struct vm_area_struct v; + + reset_all(mm_shift); + charge_expect = ring_charge(vma_pages, mm_shift); + setup_limits(split); + + v = make_vma_bytes(vma_pages << mm_shift, 0); + if (perf_mmap(&file, &v)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + dump("after first mmap"); + perf_mmap_close(&v); + dump("after close"); + return invariants_broken("baseline ring lifecycle"); +} + +/* One duplicate-ring-alias lifecycle. Close order: aliases first when + * orig_first is 0; the original mapping first, while every alias is + * still live, when orig_first is 1. */ +static int rb_dup_cycle(unsigned long vma_pages, unsigned long mm_shift, + long split, int orig_first) +{ + struct vm_area_struct v[4]; /* 0 = ring, 1..3 = aliases */ + int i, bad = 0; + long user_first, pinned_first; + + reset_all(mm_shift); + charge_expect = ring_charge(vma_pages, mm_shift); + setup_limits(split); + user_first = split < 0 ? 0 : charge_expect - (split > 0 ? split : 0); + pinned_first = split < 0 ? charge_expect : (split > 0 ? split : 0); + + v[0] = make_vma_bytes(vma_pages << mm_shift, 0); + if (perf_mmap(&file, &v[0])) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + dump("after first mmap"); + bad |= check_split("after first mmap", user_first, pinned_first); + for (i = 1; i < 4; i++) { + v[i] = make_vma_bytes(vma_pages << mm_shift, 0); + if (perf_mmap(&file, &v[i])) { + printf(" UNEXPECTED alias mmap %d failed\n", i); + return 1; + } + dump("after alias mmap"); + /* an alias shares the already-locked pages: no new charge */ + bad |= check_split("after alias mmap", user_first, pinned_first); + } + if (orig_first) { + /* alternate order: the original mapping closes while all + * three aliases are still live; the charge must stay put + * until the last alias closes */ + perf_mmap_close(&v[0]); + dump("after original close"); + bad |= check_split("after original close", user_first, + pinned_first); + for (i = 1; i < 4; i++) { + perf_mmap_close(&v[i]); + dump(i == 3 ? "after final close" : "after alias close"); + /* the last close completes the lifecycle: its zero + * check is invariants_broken() below */ + if (i < 3) + bad |= check_split("after alias close", + user_first, pinned_first); + } + } else { + for (i = 1; i < 4; i++) { + perf_mmap_close(&v[i]); + dump("after alias close"); + bad |= check_split("after alias close", user_first, + pinned_first); + } + perf_mmap_close(&v[0]); + dump("after final close"); + } + bad |= invariants_broken(orig_first + ? "duplicate ring aliases (3), original closed first" + : "duplicate ring aliases (3)"); + printf(" expected first charge: %ld user + %ld pinned; aliases add no charge\n", + user_first, pinned_first); + return bad; +} + +static int sc_rb_duplicate(unsigned long vma_pages, unsigned long mm_shift, long split) +{ + /* both close orders must balance */ + return rb_dup_cycle(vma_pages, mm_shift, split, 0) | + rb_dup_cycle(vma_pages, mm_shift, split, 1); +} + +static int sc_rb_alias_map_range_fail(long split) +{ + struct vm_area_struct v[2]; + int bad = 0; + long user_first, pinned_first; + + reset_all(PAGE_SHIFT); + charge_expect = ring_charge(17, PAGE_SHIFT); + setup_limits(split); + user_first = split < 0 ? 0 : charge_expect - (split > 0 ? split : 0); + pinned_first = split < 0 ? charge_expect : (split > 0 ? split : 0); + + v[0] = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &v[0])) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + dump("after first mmap"); + + map_range_fail = 1; + v[1] = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (!perf_mmap(&file, &v[1])) { + printf(" UNEXPECTED failing alias mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump("after FAILED alias mmap (map_range)"); + /* the failed alias charged nothing: counters still hold only the + * first map's split */ + bad |= check_split("after FAILED alias mmap (map_range)", + user_first, pinned_first); + + if (ev.mmap_count.refs != 1 || ev.rb == NULL || + ev.rb->mmap_count.refs != 1) { + printf(" UNEXPECTED refcount unwind after failed alias\n"); + bad = 1; + } + perf_mmap_close(&v[0]); + dump("after final close"); + return invariants_broken("failed ring alias via map_range") || bad; +} + +static int sc_rb_first_map_range_fail(long split) +{ + struct vm_area_struct v; + int bad; + + reset_all(PAGE_SHIFT); + charge_expect = ring_charge(17, PAGE_SHIFT); + setup_limits(split); + + map_range_fail = 1; + v = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (!perf_mmap(&file, &v)) { + printf(" UNEXPECTED failing first mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump("after FAILED first mmap (map_range)"); + bad = invariants_broken("failed first ring mmap via map_range"); + return bad; +} + +static unsigned long aux_request(struct perf_buffer *rb, unsigned long aux_pages) +{ + unsigned long base; + + base = (perf_data_size(rb) +#if RB_USER4K + + perf_metadata_size(rb) +#else + + PAGE_SIZE +#endif + ) >> PAGE_SHIFT; + /* Userspace announces the AUX layout through the shared user page + * before the first AUX mmap (same values the tree checks for). */ + rb->user_page->aux_offset = (u64)base << PAGE_SHIFT; + rb->user_page->aux_size = (u64)aux_pages << PAGE_SHIFT; + return base; +} + +static int sc_aux_baseline(void) +{ + struct vm_area_struct vring, vaux; + int bad; + + reset_all(PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + aux_pgoff_target = aux_request(ev.rb, 8); + vaux = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux)) { + printf(" UNEXPECTED aux mmap failed\n"); + return 1; + } + dump("after aux mmap"); + perf_mmap_close(&vaux); + dump("after aux close"); + perf_mmap_close(&vring); + dump("after ring close"); + bad = invariants_broken("baseline aux lifecycle"); + return bad; +} + +/* One duplicate-AUX-alias lifecycle. Close order: AUX aliases then the + * normal ring VMA when ring_first is 0; the normal ring VMA first, + * while the AUX area stays live, when ring_first is 1. */ +static int aux_dup_cycle(long split, int ring_first) +{ + struct vm_area_struct vring, vaux[4]; + int i, bad = 0; + long ring_chg = ring_charge(17, PAGE_SHIFT); + long aux_user, aux_pinned; /* the AUX charge's own split */ + + reset_all(PAGE_SHIFT); + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + if (split < 0) + set_user_lock_limit(0); + else if (split > 0) + /* the ring already sits in the user counter: the AUX charge + * of 8 pages must split 5 user / `split` pinned against a + * limit of current-user + (8 - split) */ + set_user_lock_limit(ring_chg + 8 - split); + aux_user = split < 0 ? 0 : 8 - split; + aux_pinned = split < 0 ? 8 : split; + + aux_pgoff_target = aux_request(ev.rb, 8); + vaux[0] = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux[0])) { + printf(" UNEXPECTED aux mmap failed\n"); + return 1; + } + dump("after first aux mmap"); + bad |= check_split("after first aux mmap", ring_chg + aux_user, + aux_pinned); + for (i = 1; i < 4; i++) { + vaux[i] = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux[i])) { + printf(" UNEXPECTED aux alias mmap %d failed\n", i); + return 1; + } + dump("after aux alias mmap"); + /* an alias shares the already-locked pages: no new charge */ + bad |= check_split("after aux alias mmap", ring_chg + aux_user, + aux_pinned); + } + if (ring_first) { + /* alternate order: the normal ring VMA closes while the + * AUX area stays live; the AUX mappings still hold + * rb->mmap_count, so neither charge may uncharge yet */ + perf_mmap_close(&vring); + dump("after ring close (aux live)"); + bad |= check_split("after ring close (aux live)", + ring_chg + aux_user, aux_pinned); + } + for (i = 1; i < 4; i++) { + perf_mmap_close(&vaux[i]); + dump("after aux alias close"); + bad |= check_split("after aux alias close", ring_chg + aux_user, + aux_pinned); + } + perf_mmap_close(&vaux[0]); + if (ring_first) { + /* the last AUX close empties rb->mmap_count: the ring and + * AUX charges uncharge together here */ + dump("after last aux close (all uncharge)"); + } else { + dump("after last aux close (aux uncharge)"); + perf_mmap_close(&vring); + dump("after ring close"); + } + bad |= invariants_broken(ring_first + ? "duplicate aux aliases (3), ring closed first" + : "duplicate aux aliases (3)"); + return bad; +} + +static int sc_aux_duplicate(long split) +{ + /* both close orders must balance */ + return aux_dup_cycle(split, 0) | aux_dup_cycle(split, 1); +} + +static int sc_aux_limit_fail(void) +{ + struct vm_area_struct vring, vaux; + int bad = 0; + + reset_all(PAGE_SHIFT); + charge_expect = ring_charge(17, PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + aux_pgoff_target = aux_request(ev.rb, 8); + + /* Force the tree's own calc_limits to reject: pinned_vm pressure. */ + h_rlimit_memlock = 4UL << 20; /* 256 native pages */ + mm.pinned_vm = 300; + vaux = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (!perf_mmap(&file, &vaux)) { + printf(" UNEXPECTED over-limit aux mmap succeeded\n"); + return 1; + } + dump("after REJECTED aux mmap (-EPERM)"); + bad |= user_charge_broken("after REJECTED aux mmap (-EPERM)", + charge_expect); + if (ev.rb->mmap_count.refs != 1 || ev.rb->aux_mmap_count.refs != 0 || + ev.mmap_count.refs != 1 || aux_allocs != 0) { + printf(" UNEXPECTED unwound state after -EPERM aux " + "(rbmc=%d auxmc=%d evmc=%d auxa=%d)\n", + ev.rb->mmap_count.refs, ev.rb->aux_mmap_count.refs, + ev.mmap_count.refs, aux_allocs); + bad = 1; + } + mm.pinned_vm = 0; /* the -EPERM trigger preload, not a leak */ + perf_mmap_close(&vring); + dump("after ring close"); + return invariants_broken("aux limit-rejected first map") || bad; +} + +static int sc_aux_alloc_fail(void) +{ + struct vm_area_struct vring, vaux; + int bad = 0; + + reset_all(PAGE_SHIFT); + charge_expect = ring_charge(17, PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + aux_pgoff_target = aux_request(ev.rb, 8); + + rb_alloc_aux_fail = 1; + vaux = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (!perf_mmap(&file, &vaux)) { + printf(" UNEXPECTED failing aux mmap succeeded\n"); + rb_alloc_aux_fail = 0; + return 1; + } + rb_alloc_aux_fail = 0; + dump("after FAILED aux mmap (rb_alloc_aux)"); + bad |= user_charge_broken("after FAILED aux mmap (rb_alloc_aux)", + charge_expect); + if (ev.rb->mmap_count.refs != 1 || ev.rb->aux_mmap_count.refs != 0 || + ev.mmap_count.refs != 1 || aux_allocs != 0) { + printf(" UNEXPECTED unwound state after rb_alloc_aux failure " + "(rbmc=%d auxmc=%d evmc=%d auxa=%d)\n", + ev.rb->mmap_count.refs, ev.rb->aux_mmap_count.refs, + ev.mmap_count.refs, aux_allocs); + bad = 1; + } + perf_mmap_close(&vring); + dump("after ring close"); + return invariants_broken("aux rb_alloc_aux failure") || bad; +} + +static int sc_aux_alias_size_reject(void) +{ + struct vm_area_struct vring, vaux, vbad; + struct perf_buffer *rb; + int bad = 0; + long before_mlock, before_locked; + + reset_all(PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + vaux = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux)) { + printf(" UNEXPECTED aux mmap failed\n"); + return 1; + } + before_mlock = user_perf_mlock(); + before_locked = user_locked_vm(); + + /* A wrong-size alias must be rejected before any refcount or charge. */ + vbad = make_vma_bytes(4 << PAGE_SHIFT, aux_pgoff_target); + if (!perf_mmap(&file, &vbad)) { + printf(" UNEXPECTED wrong-size aux alias succeeded\n"); + return 1; + } + dump("after REJECTED wrong-size aux alias"); + if (user_perf_mlock() != before_mlock || + user_locked_vm() != before_locked || + rb->mmap_count.refs != 2 || + rb->aux_mmap_count.refs != 1 || ev.mmap_count.refs != 2) { + printf(" UNEXPECTED state after rejected alias\n"); + bad = 1; + } + perf_mmap_close(&vaux); + perf_mmap_close(&vring); + dump("after closes"); + return invariants_broken("wrong-size aux alias") || bad; +} + +static int sc_aux_alias_map_range_fail(long split) +{ + struct vm_area_struct vring, vaux[2]; + struct perf_buffer *rb; + int bad = 0; + + reset_all(PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + rb = ev.rb; + if (split < 0) + set_user_lock_limit(0); + else if (split > 0) + /* the ring already sits in the user counter (see + * sc_aux_duplicate): limit = current-user + (8 - split) */ + set_user_lock_limit(ring_charge(17, PAGE_SHIFT) + 8 - split); + aux_pgoff_target = aux_request(rb, 8); + vaux[0] = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux[0])) { + printf(" UNEXPECTED aux mmap failed\n"); + return 1; + } + dump("after first aux mmap"); + + map_range_fail = 1; + vaux[1] = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (!perf_mmap(&file, &vaux[1])) { + printf(" UNEXPECTED failing aux alias mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump("after FAILED aux alias mmap (map_range)"); + /* the failed alias charged nothing: counters still hold only the + * ring's charge plus the first AUX map's split */ + bad |= check_split("after FAILED aux alias mmap (map_range)", + ring_charge(17, PAGE_SHIFT) + + (split < 0 ? 0 : 8 - split), + split < 0 ? 8 : split); + if (ev.mmap_count.refs != 2 || rb->mmap_count.refs != 2 || + rb->aux_mmap_count.refs != 1) { + printf(" UNEXPECTED refcount unwind after failed aux alias\n"); + bad = 1; + } + perf_mmap_close(&vaux[0]); + dump("after last aux close"); + perf_mmap_close(&vring); + dump("after ring close"); + return invariants_broken("failed aux alias via map_range") || bad; +} + +static int sc_aux_first_map_range_fail(void) +{ + struct vm_area_struct vring, vaux; + int bad = 0; + + reset_all(PAGE_SHIFT); + charge_expect = ring_charge(17, PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + aux_pgoff_target = aux_request(ev.rb, 8); + + map_range_fail = 1; + vaux = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (!perf_mmap(&file, &vaux)) { + printf(" UNEXPECTED failing first aux mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump("after FAILED first aux mmap (map_range)"); + /* perf_mmap_close() must have unwound the whole AUX allocation: + * the ring keeps its own charge, the AUX charge and allocation + * are gone again, each counter checked individually */ + bad |= user_charge_broken("after FAILED first aux mmap (map_range)", + charge_expect); + if (mm.pinned_vm != 0 || + ev.rb->mmap_count.refs != 1 || ev.rb->aux_mmap_count.refs != 0 || + ev.mmap_count.refs != 1 || aux_allocs != aux_frees) { + printf(" UNEXPECTED unwound state after failed first aux " + "(pinned=%ld rbmc=%d auxmc=%d evmc=%d " + "auxa=%d auxf=%d)\n", + (long)mm.pinned_vm, + ev.rb->mmap_count.refs, ev.rb->aux_mmap_count.refs, + ev.mmap_count.refs, aux_allocs, aux_frees); + bad = 1; + } + perf_mmap_close(&vring); + dump("after ring close"); + return invariants_broken("failed first aux mmap via map_range") || bad; +} + +int main(void) +{ + int failures = 0; + unsigned long geos[][2] = { /* vma_pages, mm_shift */ + { 17, PAGE_SHIFT }, /* 16 data pages + metadata */ + { 1, PAGE_SHIFT }, /* metadata-only */ + { 2, 12 }, /* user4k: 1 data chunk of 4K on a 4K mm */ + }; + long splits[] = { 0, 3, -1 }; /* none, partial, full spill */ + const char *geo_names[] = { "ring16+meta", "metadata-only", "sub4k-1chunk+meta" }; + const char *lim_names[] = { "nosplit", "partialsplit3", "fullspill" }; + int g, s; + + for (g = 0; g < 3; g++) { + if (g == 2 && !RB_USER4K) + continue; + printf("-- scenario rb-baseline geometry=%s\n", geo_names[g]); + if (sc_rb_baseline(geos[g][0], geos[g][1], 0)) { + printf("RESULT scenario=rb-baseline geometry=%s status=FAIL\n", + geo_names[g]); + failures++; + } else { + printf("RESULT scenario=rb-baseline geometry=%s status=PASS\n", + geo_names[g]); + } + } + for (s = 0; s < 3; s++) { + for (g = 0; g < 3; g++) { + long split = splits[s]; + const char *lim = lim_names[s]; + char limname[32]; + long ringchg = ring_charge(geos[g][0], geos[g][1]); + + if (g == 2 && !RB_USER4K) + continue; + if (split > 0 && split >= ringchg) { + /* this charge cannot give up `split` pages + * to pinned_vm; use the largest valid + * partial spill instead of letting the + * limit underflow into "no limit" */ + split = ringchg - 1; + if (split <= 0) { + split = -1; /* cannot split at all */ + lim = lim_names[2]; + } else { + snprintf(limname, sizeof(limname), + "partialsplit%ld", split); + lim = limname; + } + } + printf("-- scenario rb-duplicate geometry=%s limits=%s\n", + geo_names[g], lim); + if (sc_rb_duplicate(geos[g][0], geos[g][1], split)) { + printf("RESULT scenario=rb-duplicate geometry=%s limits=%s status=FAIL\n", + geo_names[g], lim); + failures++; + } else { + printf("RESULT scenario=rb-duplicate geometry=%s limits=%s status=PASS\n", + geo_names[g], lim); + } + } + } + for (s = 0; s < 3; s += 2) { /* nosplit and fullspill */ + printf("-- scenario rb-alias-map-range-fail limits=%s\n", lim_names[s]); + if (sc_rb_alias_map_range_fail(splits[s])) { + printf("RESULT scenario=rb-alias-map-range-fail limits=%s status=FAIL\n", + lim_names[s]); + failures++; + } else { + printf("RESULT scenario=rb-alias-map-range-fail limits=%s status=PASS\n", + lim_names[s]); + } + printf("-- scenario rb-first-map-range-fail limits=%s\n", lim_names[s]); + if (sc_rb_first_map_range_fail(splits[s])) { + printf("RESULT scenario=rb-first-map-range-fail limits=%s status=FAIL\n", + lim_names[s]); + failures++; + } else { + printf("RESULT scenario=rb-first-map-range-fail limits=%s status=PASS\n", + lim_names[s]); + } + } + printf("-- scenario aux-baseline\n"); + if (sc_aux_baseline()) { + printf("RESULT scenario=aux-baseline status=FAIL\n"); + failures++; + } else { + printf("RESULT scenario=aux-baseline status=PASS\n"); + } + for (s = 0; s < 3; s++) { + printf("-- scenario aux-duplicate limits=%s\n", lim_names[s]); + if (sc_aux_duplicate(splits[s])) { + printf("RESULT scenario=aux-duplicate limits=%s status=FAIL\n", + lim_names[s]); + failures++; + } else { + printf("RESULT scenario=aux-duplicate limits=%s status=PASS\n", + lim_names[s]); + } + } + printf("-- scenario aux-limit-reject\n"); + if (sc_aux_limit_fail()) { + printf("RESULT scenario=aux-limit-reject status=FAIL\n"); + failures++; + } else { + printf("RESULT scenario=aux-limit-reject status=PASS\n"); + } + printf("-- scenario aux-alloc-fail\n"); + if (sc_aux_alloc_fail()) { + printf("RESULT scenario=aux-alloc-fail status=FAIL\n"); + failures++; + } else { + printf("RESULT scenario=aux-alloc-fail status=PASS\n"); + } + printf("-- scenario aux-alias-size-reject\n"); + if (sc_aux_alias_size_reject()) { + printf("RESULT scenario=aux-alias-size-reject status=FAIL\n"); + failures++; + } else { + printf("RESULT scenario=aux-alias-size-reject status=PASS\n"); + } + for (s = 0; s < 3; s += 2) { + printf("-- scenario aux-alias-map-range-fail limits=%s\n", lim_names[s]); + if (sc_aux_alias_map_range_fail(splits[s])) { + printf("RESULT scenario=aux-alias-map-range-fail limits=%s status=FAIL\n", + lim_names[s]); + failures++; + } else { + printf("RESULT scenario=aux-alias-map-range-fail limits=%s status=PASS\n", + lim_names[s]); + } + } + printf("-- scenario aux-first-map-range-fail\n"); + if (sc_aux_first_map_range_fail()) { + printf("RESULT scenario=aux-first-map-range-fail status=FAIL\n"); + failures++; + } else { + printf("RESULT scenario=aux-first-map-range-fail status=PASS\n"); + } + + printf("SUMMARY failures=%d\n", failures); + return failures ? 1 : 0; +} +''' + +HARNESS_GLOBALS = ''' +static unsigned long aux_pgoff_target; +static long charge_expect; +''' + + +def source_notes(core): + """Probe the source shape so the tree under test is reported honestly.""" + notes = [] + rb = definition(core, 'perf_mmap_rb') + close = definition(core, 'perf_mmap_close') + if rb: + marker = rb.find('refcount_inc_not_zero(&event->rb->mmap_count)') + tail = rb[marker:marker + 400] if marker != -1 else '' + if 'perf_mmap_account' in tail: + notes.append('ring alias path charges perf_mmap_account() per alias') + else: + notes.append('ring alias path adds no charge ' + '(aliases share the first map\'s charge)') + if close: + count = close.count('perf_mmap_unaccount(') + notes.append('close has %d perf_mmap_unaccount call site(s) ' + '(ring uncharge happens only when rb->mmap_count ' + 'reaches 0)' % count) + return notes + + +def build(tree, out): + core_path = tree / 'kernel/events/core.c' + internal_path = tree / 'kernel/events/internal.h' + ring_path = tree / 'kernel/events/ring_buffer.c' + core = core_path.read_text() + internal = internal_path.read_text() + ring = ring_path.read_text() + + user4k, nr_pages_expr, data_shift_expr = rb_alloc_profile(ring) + init = rb_init_lines(ring) + + helpers = [] + if not user4k: + body = definition(internal, 'page_order') + if body is None: + raise SystemExit('missing page_order() in internal.h') + helpers.append(body) + for name in ('rb_has_aux', 'perf_data_size', 'data_page_nr', 'perf_aux_size'): + body = definition(internal, name) + if body is None: + raise SystemExit('missing inline %s in internal.h' % name) + helpers.append(body) + if user4k: + body = definition(internal, 'perf_metadata_size') + if body: + helpers.append(body) + body = definition(core, 'perf_mmap_offset') + if body is None: + raise SystemExit('user4k tree lacks perf_mmap_offset()') + helpers.append(body) + for name in ('perf_mmap_calc_limits', 'perf_mmap_account', + 'perf_mmap_unaccount', 'perf_mmap_rb', 'perf_mmap_aux', + 'perf_mmap_close', 'perf_mmap'): + body = definition(core, name) + if body is None: + raise SystemExit('missing %s in core.c' % name) + helpers.append(body) + + aux_decl = re.search(r'extern int rb_alloc_aux\(([^;]+)\);', internal) + if aux_decl is None: + raise SystemExit('missing rb_alloc_aux prototype in internal.h') + params = aux_decl.group(1).strip().rstrip(')') + names = [p.strip().split()[-1].lstrip('*') for p in params.split(',')] + aux_def = ('static int rb_alloc_aux(%s)\n' + '{\n' + '\tif (rb_alloc_aux_fail)\n' + '\t\treturn -ENOMEM;\t/* the real allocator frees its own partial work */\n' + '\trb->aux_nr_pages = nr_pages;\n' + '\trb->aux_watermark = watermark;\n' + '\trefcount_set(&rb->aux_refcount, 1);\n' + '#if RB_USER4K\n' + '\trb->aux_offset = %s;\n' + '#else\n' + '\trb->aux_pgoff = %s;\n' + '#endif\n' + '\taux_allocs++;\n' + '\treturn 0;\n' + '}' % (params, names[2], names[2])) + + unaccount = definition(core, 'perf_mmap_unaccount') + if unaccount is None: + raise SystemExit('missing perf_mmap_unaccount in core.c') + counter = 'perf_mlock' if 'user->perf_mlock' in unaccount else 'locked_vm' + + if user4k: + rb_alloc_decl = ('struct perf_buffer *rb_alloc(unsigned long data_size, ' + 'unsigned int mmap_page_shift, long watermark, int cpu, ' + 'int flags);') + else: + rb_alloc_decl = ('struct perf_buffer *rb_alloc(int nr_pages, ' + 'long watermark, int cpu, int flags);') + + source = HARNESS_GLOBALS + Template(HARNESS).substitute({ + 'RB_USER4K': '1' if user4k else '0', + 'rb_alloc_decl': rb_alloc_decl, + 'rb_alloc_aux_decl': 'static int rb_alloc_aux(%s);' % params, + 'rb_alloc_aux_def': aux_def, + 'helpers': '\n\n'.join(helpers), + 'rb_nr_pages': nr_pages_expr, + 'rb_data_shift': ('\trb->data_page_shift = %s;' % data_shift_expr) + if data_shift_expr else '\t/* none */', + 'rb_init': init or '\t/* tree leaves mmap_count at 0 */', + 'charge_counter': counter, + 'spare_counter': 'locked_vm' if counter == 'perf_mlock' + else 'perf_mlock', + }) + path = out / (tree.name + '.c') + path.write_text(source) + binary = out / (tree.name + '.bin') + subprocess.run([args.cc, '-std=gnu11', '-Wall', '-Werror', + '-Wno-unused-function', str(path), '-o', str(binary)], + check=True) + return binary, user4k, counter, source_notes(core) + + +overall = 0 +with tempfile.TemporaryDirectory(prefix='perf-alias-lifecycle-') as directory: + for tree in args.trees: + print('=' * 72) + print('TREE %s' % tree) + try: + binary, user4k, counter, notes = build(tree, Path(directory)) + except SystemExit as error: + print('FAIL: %s' % error) + overall = 2 + continue + except subprocess.CalledProcessError: + print('FAIL: harness build failed for %s' % tree) + src = Path(directory) / (tree.name + '.c') + if src.exists(): + print('harness source kept at %s' % src) + overall = 2 + continue + print('profile: %s, user counter: %s' % + ('user4k (data_size rb_alloc, per-mm page shift)' if user4k + else 'plain (nr_pages rb_alloc, PAGE_SHIFT)', counter)) + for note in notes: + print('source: %s' % note) + result = subprocess.run([str(binary)], capture_output=True, text=True) + sys.stdout.write(result.stdout) + if result.stderr.strip(): + sys.stdout.write(result.stderr) + if result.returncode != 0 and overall == 0: + overall = 1 +print('=' * 72) +print('OVERALL: %s' % ('tested invariants failed' if overall == 1 + else 'build/source error' if overall == 2 + else 'tested single-user, single-mm lifecycles balanced')) +raise SystemExit(overall) From 37c5cb7deebfd46106950536d70482352135843b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:18:37 -0600 Subject: [PATCH 094/392] test: register perf mmap accounting host check Register the alias lifecycle harness so the host-test runner executes it instead of rejecting the unregistered file. Verified with the runner before and after registration. Assisted-by: LLM Signed-off-by: Joshua Warren --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 84ced6b0f27d5..77acf649e7d39 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -153,6 +153,7 @@ def mutations(flag, names, base, expect=KILL): + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', 'offset-sign', 'event-bit', 'wrong-control', 'maverick-event', 'stowe-offset'), out()), + 'perf-mmap-alias-lifecycle-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-oslog-tests.py': [(out(), PASS)] + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), From 0c521040cf6c8c766b8a5e335b5f7fa9806e169b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:12:19 +1100 Subject: [PATCH 095/392] media: apple: prores: budget the T6000 v0 descriptor per profile The v0 descriptor left its two rate-control limits at 0xffff, so the APR a0 engine never limited a frame: on detailed or noisy input every profile came out far above its data rate (1080p 422 HQ on noise at 1028 Mbit/s instead of about 220) and the engine slowed down accordingly. The v0 rate-control block is the v1 block 0x20 bytes earlier (the thresholds, weights, betas, quantiser words and matrix selector already line up), so 0xf0/0xf2 are the target and maximum Q3 bytes per macroblock. Write the same per-profile budgets as v1. Frame sizes on the M1 Pro now equal the Neo's and macOS VideoToolbox's on the same input in all six profiles, on three clip types, at 1080p and 2160p. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../platform/apple/prores/apple-prores-desc.c | 14 ++++++++++---- .../platform/apple/prores/apple-prores-test.c | 8 +++++++- .../platform/apple/prores/t6000-qualification.rst | 15 +++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/apple/prores/apple-prores-desc.c b/drivers/media/platform/apple/prores/apple-prores-desc.c index 17a87ea2e6353..82abe6acce3f9 100644 --- a/drivers/media/platform/apple/prores/apple-prores-desc.c +++ b/drivers/media/platform/apple/prores/apple-prores-desc.c @@ -425,19 +425,25 @@ EXPORT_SYMBOL_IF_KUNIT(apr_build_desc); * Descriptor v0 (APR a0). Source: MIT m1n1 f3ac615eb137, * proxyclient/m1n1/hw/prores.py:7-111 and * proxyclient/experiments/prores.py:592-644. Only the input/output prefix - * is shared with v1. The public experiment leaves both legacy limits at - * 0xffff; their meaning is unqualified, so do not apply v1 budget formulas. + * is shared with v1. The rate-control block is v1's, 0x20 bytes earlier: + * the target and maximum Q3 bytes per macroblock at 0xf0/0xf2 take the same + * per-profile budgets, so every profile lands on its data rate. (The public + * experiment's 0xffff limits leave the engine unbudgeted.) */ int apr_build_desc_v0(const struct apr_frame *f, u8 *desc) { bool is444 = apr_input_is_444(f->input); enum apr_qtable table = apr_profile_qtable(f->profile); + u32 slices, mbs, target, max; unsigned int i; int ret; ret = apr_build_desc_common(f, desc); if (ret) return ret; + slices = apr_slice_count(f->width, f->height, APR_SLICE_LOG2); + mbs = DIV_ROUND_UP(f->width, 16) * DIV_ROUND_UP(f->height, 16); + apr_frame_budget(f->profile, f->width, f->height, &target, &max); /* Frame header, including the legacy aspect/rate byte at 0x065. */ put_unaligned_be16(D_FH_BYTES, desc + 0x058); @@ -454,8 +460,8 @@ int apr_build_desc_v0(const struct apr_frame *f, u8 *desc) /* Deprecated slice count at 0x0ec stays zero, as in m1n1. */ desc[0x0ee] = APR_SLICE_LOG2 << 4; desc[0x0ef] = apr_profile_policy[f->profile].quant_index; - put_unaligned_le16(0xffff, desc + 0x0f0); - put_unaligned_le16(0xffff, desc + 0x0f2); + put_unaligned_le16(apr_mb_budget(target, slices, mbs), desc + 0x0f0); + put_unaligned_le16(apr_mb_budget(max, slices, mbs), desc + 0x0f2); put_unaligned_le64(APR_RC_THRESHOLDS, desc + 0x0f4); put_unaligned_le32(0x2c8080, desc + 0x0fc); for (i = 0; i < ARRAY_SIZE(apr_rc_beta); i++) diff --git a/drivers/media/platform/apple/prores/apple-prores-test.c b/drivers/media/platform/apple/prores/apple-prores-test.c index 3e727ff0ae3dc..9ac7080e072a2 100644 --- a/drivers/media/platform/apple/prores/apple-prores-test.c +++ b/drivers/media/platform/apple/prores/apple-prores-test.c @@ -189,6 +189,7 @@ static void apr_test_desc_v0(struct kunit *test) struct apr_frame f = apr_test_frame(); u8 *d = kunit_kmalloc(test, APR_DESC_BYTES + 1, GFP_KERNEL); unsigned int i; + u32 budget; KUNIT_ASSERT_NOT_NULL(test, d); memset(d, 0xa5, APR_DESC_BYTES + 1); @@ -221,9 +222,11 @@ static void apr_test_desc_v0(struct kunit *test) KUNIT_EXPECT_EQ_MSG(test, d[i], 0, "padding/slices %#x", i); KUNIT_EXPECT_EQ(test, d[0xee], 0x30); KUNIT_EXPECT_EQ(test, d[0xef], 2); - for (i = 0; i < ARRAY_SIZE(legacy); i++) + /* Word 0 is the rate budget, checked against v1 below. */ + for (i = 1; i < ARRAY_SIZE(legacy); i++) KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le32(d + 0xf0 + 4 * i), legacy[i], "legacy word %u", i); + budget = get_unaligned_le32(d + 0xf0); for (i = 0x154; i < APR_DESC_BYTES; i++) KUNIT_EXPECT_EQ_MSG(test, d[i], 0, "v0 trailing padding %#x", i); @@ -232,6 +235,9 @@ static void apr_test_desc_v0(struct kunit *test) KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x58), 0); KUNIT_EXPECT_EQ(test, get_unaligned_be16(d + 0x7c), 148); KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 0x170), 0x23); + /* v0 carries v1's target and maximum budget words. */ + KUNIT_EXPECT_EQ(test, budget, get_unaligned_le32(d + 0x110)); + KUNIT_EXPECT_NE(test, budget, 0xffffffff); } static void apr_test_desc_v0_profiles(struct kunit *test) diff --git a/drivers/media/platform/apple/prores/t6000-qualification.rst b/drivers/media/platform/apple/prores/t6000-qualification.rst index 17a44eb892328..c8f021988687e 100644 --- a/drivers/media/platform/apple/prores/t6000-qualification.rst +++ b/drivers/media/platform/apple/prores/t6000-qualification.rst @@ -16,9 +16,12 @@ Standard, HQ and 4444 at 3840x2160. ffmpeg decoded every frame at the expected size and chroma format. There were no DART faults, and the engine runtime-suspended with its power domain off afterwards. +Rate control matches macOS: with the per-profile budgets, frame sizes +equal VideoToolbox's on the same input (measured against macOS on the +T8140 engine), on easy, detailed and noisy clips at 1080p and 2160p. + Not yet covered: cancellation, failed-reset retention (#925), system -suspend, rate-control parity with macOS, and the SCODEC performance -request below. +suspend and the SCODEC performance request below. Implementation sources ---------------------- @@ -30,10 +33,10 @@ f3ac615eb1376d3bbba24cd84c414b9e41892adb: * ``proxyclient/experiments/prores.py:42-89``: all 48 core RMWs, in order. * ``proxyclient/m1n1/hw/prores.py:7-111``: the v0 descriptor layout. * ``proxyclient/experiments/prores.py:592-644``: legacy header, slice and - rate-control words. The v0 builder retains the two public 0xffff limits; - it does not assign them v1 coefficient-budget semantics. Profile matrices - and initial quantisers follow the existing driver's policy, but profile - bitrate/quality equivalence remains to be measured on v0. + rate-control words. The v0 rate-control block is the v1 one, 0x20 bytes + earlier; the builder writes the per-profile target and maximum budgets + at 0xf0/0xf2 instead of the experiment's unbudgeted 0xffff. Profile + matrices and initial quantisers follow the existing driver's policy. The MIT notice for the adapted constants is retained in ``m1n1-LICENSE``. T6000 selects neither wrapper nor SCODEC-system writes. It has no BWR entry; From 2f86a048520d3c5dd59cb4aaa2b2b3dcbee6f220 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:37:37 +1100 Subject: [PATCH 096/392] drm/apple: answer the DCP PMU temperature query with the battery temperature DCP asks the AP for "PMUS"/"Temperature" once while it starts and, when no 2D panel temperature has been set, takes the answer as the initial panel temperature for its compensation. Linux answered with a constant 3029 on every machine. Our own macOS observation on the J700 (macOS 26.6.2) shows what macOS sends: the system battery's gas gauge temperature in 0.1 K, the same value the SMC reports in B0AT. At one boot the battery read 2997, the DCP logged "using PMU temperature as initial guess: 29.969985961" and published InitialPanelTemperature 26.5 C in Q16.16, which is (2997 - 2732) / 10, the battery temperature. An earlier boot matched in the same way (29.78 logged, 24.59 C published). Read the temperature from the system battery power supply at the time of the query and convert it back to 0.1 K. Without a battery (desktops, or no battery driver bound) keep the old reply. Load macsmc-power before the DCP so the battery exists when DCP starts. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/apple_drv.c | 4 +- drivers/gpu/drm/apple/iomfb.c | 58 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.h | 5 +++ drivers/gpu/drm/apple/iomfb_h17p.c | 4 +- drivers/gpu/drm/apple/iomfb_template.c | 6 +-- drivers/gpu/drm/apple/iomfb_v14_7.c | 7 +--- 6 files changed, 71 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 763ba82ad29ac..41b9c672ae398 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1806,5 +1806,7 @@ MODULE_LICENSE("Dual MIT/GPL"); * crossbar. Without this, initramfs generators leave it out, it loads only * from the root filesystem, and a DCP probe that waits past the deferred * probe timeout (an encrypted root's passphrase prompt) gives up its routes. + * DCP also asks for the battery temperature once while it starts (see + * dcp_pmu_temperature()), so have the SMC battery registered by then. */ -MODULE_SOFTDEP("pre: mux-apple-display-crossbar"); +MODULE_SOFTDEP("pre: mux-apple-display-crossbar macsmc-power"); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a504dca5a6bea..8bca8d2cebeca 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,63 @@ struct dcp_identity_trace { atomic_t *count; }; +/* + * DCP asks the AP for "PMUS"/"Temperature" once at start-up and uses it as + * the initial panel temperature when no 2D panel temperature has been set. + * macOS answers with the system battery's gas gauge temperature in units of + * 0.1 K (the AppleSmartBattery "Temperature" property, the SMC B0AT key); + * the firmware takes off 273.2 K itself. Observed on the J700 under macOS + * 26.6: a reply of 2997 gave an InitialPanelTemperature of 26.5 C, equal to + * the battery temperature at that boot. + * + * Linux power supplies report temperature in 0.1 C, so add the 273.2 K back. + */ +struct dcp_battery_temp { + u64 value; + bool found; +}; + +static int dcp_battery_temp_cb(struct power_supply *psy, void *data) +{ + struct dcp_battery_temp *bt = data; + union power_supply_propval val; + + if (psy->desc->type != POWER_SUPPLY_TYPE_BATTERY) + return 0; + if (!power_supply_get_property(psy, POWER_SUPPLY_PROP_SCOPE, &val) && + val.intval == POWER_SUPPLY_SCOPE_DEVICE) + return 0; + if (power_supply_get_property(psy, POWER_SUPPLY_PROP_TEMP, &val)) + return 0; + if (val.intval <= -2732) + return 0; + + bt->value = val.intval + 2732; + bt->found = true; + return 1; +} + +u64 dcp_pmu_temperature(struct device *dev) +{ + struct dcp_battery_temp bt = { 0 }; + + if (IS_REACHABLE(CONFIG_POWER_SUPPLY)) + power_supply_for_each_psy(&bt, dcp_battery_temp_cb); + + if (bt.found) { + dev_dbg(dev, "PMU temperature from the battery: %llu (0.1 K)\n", + bt.value); + return bt.value; + } + + /* + * No system battery (desktops, or the battery driver is not bound): + * keep the reply Linux has always sent, 302.9 K. + */ + dev_dbg(dev, "no system battery, PMU temperature fallback\n"); + return DCP_PMU_TEMPERATURE_FALLBACK; +} + /* * set_matrix coefficients for @ctm, the identity when @ctm is NULL. DRM * carries S31.32 sign-magnitude; the firmware takes signed (two's diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 5e5795ba3c379..2ae15b13b8d43 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -391,6 +391,11 @@ struct iomfb_set_matrix_req { u8 padding[3]; } __packed; +/* "PMUS"/"Temperature" reply in 0.1 K: the battery temperature, as macOS. */ +#define DCP_PMU_TEMPERATURE_FALLBACK 3029 +struct device; +u64 dcp_pmu_temperature(struct device *dev); + struct drm_color_ctm; void dcp_ctm_to_fw(u64 fw[9], const struct drm_color_ctm *ctm); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ad8307f602ea7..1e24d0e617a79 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -516,7 +516,7 @@ static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, return trampoline_get_uint_prop(dcp, tag, out, in); trace_iomfb_callback(dcp, tag, __func__); - /* The observed startup query uses Asahi's existing Temperature reply. */ + /* The observed startup query: the battery temperature in 0.1 K. */ if (memcmp(request->obj, "SUMP", sizeof(request->obj)) || strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || strcmp(request->key, "Temperature") || request->value_null || @@ -526,7 +526,7 @@ static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, return false; } - put_unaligned_le64(3029, out); + put_unaligned_le64(dcp_pmu_temperature(dcp->dev), out); *((u8 *)out + offsetof(struct dcp_get_uint_prop_resp, ret)) = true; return true; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 82e5310f3fd6e..ad87814e9bb9b 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -877,11 +877,7 @@ dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) if (dcp->panel.has_mini_led && memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { - /* - * TODO: value from j314c, find out if it is temperature in - * centigrade C and which temperature sensor reports it - */ - resp.value = 3029; + resp.value = dcp_pmu_temperature(dcp->dev); resp.ret = true; } } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 5c1b20c730c4c..1f98ace1d9a89 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -1342,12 +1342,9 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz } /* Unsigned property read. */ if (tag == D(401) && SHAPE(80, 12)) { - /* - * The PMU temperature the M1/M2 mini-LED hosts report: a fixed - * placeholder in centidegrees, not a measurement. - */ + /* The PMU temperature: the battery temperature in 0.1 K. */ if (!memcmp(in, "SUMP", 4) && !strncmp(in + 4, "Temperature", 64)) { - put_unaligned_le64(3029, out); + put_unaligned_le64(dcp_pmu_temperature(v14->dev), out); out[8] = 1; return 0; } From 349bc3bc104186d36bca018a1eb8f9e655e0c138 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:37:56 +1100 Subject: [PATCH 097/392] PM: hibernate: allow pm_test dry runs on Apple silicon Hibernation is disabled on Apple silicon Macs with a blanket check in hibernation_available(). That is right for real images: restoring one means a cold boot through the platform boot chain, which restarts the coprocessor firmware, and most of the Apple drivers have no freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state from a cold start. The check is silent, though, and it also blocks the pm_test dry runs that are needed to bring those callbacks up. Every pm_test level stops before create_image() takes a snapshot, so a dry run never writes or restores an image. Keep hibernation_available() false on these machines, so /sys/power/disk stays "[disabled]", "disk" is not listed in /sys/power/state, resume from an image is skipped and /dev/snapshot stays closed. Then: - print the reason once at boot, - let hibernate() run when pm_test is set to a level other than "none", checked under system_transition_mutex so the level cannot change during the run, - refuse the real transition with a clear message and -EPERM, as before, - drop the snapshot and bail out, with a warning, if one is ever taken during a dry run. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- kernel/power/hibernate.c | 66 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index af4a1943b1eee..890bf7b447f89 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -107,13 +107,45 @@ bool hibernation_in_progress(void) return !atomic_read(&hibernate_atomic); } -bool hibernation_available(void) +/* + * Apple silicon Macs run firmware on their coprocessors (storage, secure + * enclave, display, always-on processor, system controller, GPU and more). + * Restoring an image means a cold boot through the platform boot chain, which + * restarts that firmware, but most of the Linux drivers have no + * freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state + * from a cold start. The restored kernel would keep using stale state, so + * writing and restoring images stays disabled on these machines. + * + * pm_test dry runs are allowed (see hibernate()): every pm_test level stops + * before create_image() takes a snapshot, so no image is written or restored. + * They exercise the freezer and the device freeze/thaw callbacks that a real + * implementation needs. + */ +static bool hibernation_platform_blocked(void) +{ + return of_machine_is_compatible("apple,arm-platform"); +} + +static bool hibernation_generic_available(void) { return nohibernate == 0 && !security_locked_down(LOCKDOWN_HIBERNATION) && - !secretmem_active() && !cxl_mem_active() && - !of_machine_is_compatible("apple,arm-platform"); + !secretmem_active() && !cxl_mem_active(); +} + +bool hibernation_available(void) +{ + return hibernation_generic_available() && + !hibernation_platform_blocked(); +} + +static int __init hibernation_platform_report(void) +{ + if (hibernation_platform_blocked()) + pr_info("disabled on this platform: coprocessor firmware state is not restored from an image\n"); + return 0; } +late_initcall(hibernation_platform_report); /** * hibernation_set_ops - Set the global hibernate operations. @@ -764,12 +796,17 @@ static int load_image_and_restore(void) int hibernate(void) { bool snapshot_test = false; + bool dry_run_only = false; unsigned int sleep_flags; int error; if (!hibernation_available()) { - pm_pr_dbg("Hibernation not available.\n"); - return -EPERM; + if (!hibernation_generic_available() || + !hibernation_platform_blocked()) { + pm_pr_dbg("Hibernation not available.\n"); + return -EPERM; + } + dry_run_only = true; } /* @@ -784,6 +821,16 @@ int hibernate(void) } sleep_flags = lock_system_sleep(); + /* + * pm_test_level cannot change while system_transition_mutex is held, + * so a dry run started here stays a dry run. + */ + if (dry_run_only && pm_test_level == TEST_NONE) { + pr_info("not supported on this platform yet; only pm_test dry runs are allowed\n"); + error = -EPERM; + goto Unlock; + } + /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { error = -EBUSY; @@ -816,6 +863,15 @@ int hibernate(void) if (error || freezer_test_done) goto Free_bitmaps; + if (in_suspend && WARN_ON_ONCE(dry_run_only)) { + /* A snapshot exists although only a dry run was allowed: drop it. */ + swsusp_free(); + in_suspend = 0; + pm_restore_gfp_mask(); + error = -EPERM; + goto Free_bitmaps; + } + if (in_suspend) { unsigned int flags = 0; From e242c9bfc8b14b09999155827ccffcf935f60ed5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:06:42 +1100 Subject: [PATCH 098/392] thunderbolt: apple: order PCIe-C after the NHI with a device link Every boot with a USB4 router on the M2 and the M4 Pro logs thunderbolt-apple-nhi ...: device links to tunneled native ports are missing! tb_probe() looks for links only in ACPI tables and on Intel Macs. The Apple Silicon hosts have no such tables: their one tunneled native port is the PCIe-C controller, which the driver populates itself once a PCIe tunnel is up. Add an add_links hook to tb_nhi_ops that tb_probe() asks first. The Apple NHI answers true and links PCIe-C to the NHI when it populates it. That gives the order the ACPI links give on PCs: PCIe-C suspends before the NHI and resumes after it rebuilds the tunnel. Until now it held only because PCIe-C is registered later. A port without PCIe-C tunnels no native port, so it needs no link. The link is stateless. It orders system sleep and does not tie probe or unbind together, because apple_cio_remove_children() has to stop the NHI while PCIe-C is still bound. Both devices are removed together, and that drops the link. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/thunderbolt/apple.c | 39 +++++++++++++++++++++++++++++++++++++ drivers/thunderbolt/nhi.h | 6 ++++++ drivers/thunderbolt/tb.c | 9 ++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index ba7ec0bc0296c..3374e0d3e794f 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -1354,6 +1354,33 @@ static int apple_cio_pcie_tunnel_ready_locked(struct apple_cio *acio) return ret; } +/* + * PCIe-C is the only native port this host tunnels. Order it after the NHI, + * which rebuilds the tunnel underneath it at resume (the job of the ACPI + * device links on PC hosts). The link is stateless: it orders system sleep + * only and leaves probe and unbind alone, because apple_cio_remove_children() + * must stop the NHI while PCIe-C is still bound. Removing either device drops + * the link; both are removed together and PCIe-C is populated only once per + * NHI, so this runs once per pair. + */ +static void apple_cio_link_pcie_tunnel(struct apple_cio *acio) +{ + struct platform_device *nhi_pdev = READ_ONCE(acio->nhi_pdev); + struct platform_device *pcie_pdev; + + lockdep_assert_held(&acio->pcie_tunnel_lock); + pcie_pdev = apple_cio_find_pcie_tunnel(acio); + if (!pcie_pdev) + return; + if (!nhi_pdev || + !device_link_add(&pcie_pdev->dev, &nhi_pdev->dev, DL_FLAG_STATELESS)) + dev_warn(acio->dev, "PCIe-C: device link to the NHI failed\n"); + else + dev_dbg(acio->dev, "PCIe-C: linked %s to %s\n", + dev_name(&pcie_pdev->dev), dev_name(&nhi_pdev->dev)); + put_device(&pcie_pdev->dev); +} + static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) { int ret = 0; @@ -1383,6 +1410,7 @@ static int apple_cio_activate_pcie_tunnel_locked(struct apple_cio *acio) "failed to populate tunneled PCIe controller\n"); acio->pcie_tunnel_populated = true; + apple_cio_link_pcie_tunnel(acio); dev_info(acio->dev, "PCIe-C populated after USB4 PCIe tunnel adapter enable\n"); return apple_cio_pcie_tunnel_ready_locked(acio); @@ -2015,7 +2043,18 @@ static bool apple_nhi_dp_tunnel_awaits_display(struct tb_nhi *nhi, return apple_dpin_awaits_display(&c->state, c->acio->dp); } +/* + * There are no firmware tables naming the tunneled ports: PCIe-C, when the + * port has one, is linked to the NHI as it is populated (see + * apple_cio_link_pcie_tunnel()). A port without PCIe-C tunnels no native port. + */ +static bool apple_nhi_add_links(struct tb_nhi *nhi) +{ + return true; +} + static const struct tb_nhi_ops apple_nhi_ops = { + .add_links = apple_nhi_add_links, .request_ring_irq = apple_nhi_request_irq, .release_ring_irq = apple_nhi_release_irq, .ring_interrupt_active = apple_nhi_ring_interrupt_active, diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 7201e0c279cc3..75795fbf865a1 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -100,6 +100,11 @@ struct tb_nhi_ring_layout { * meanwhile, so the tunnel is kept instead of being * torn down at the DPRX timeout. Called with the * domain lock held. + * @add_links: NHI specific hook that makes the native ports tunneled + * through this host (PCIe) resume after the NHI, like the device + * links tb_acpi_add_links() creates from firmware tables. Returns + * %true if the host orders all of its tunneled native ports, so + * none is left without a link. * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook * @reset_interface: Resets the host interface @@ -127,6 +132,7 @@ struct tb_nhi_ops { struct tb_port *out, u64 generation); void (*dp_tunnel_deactivate)(struct tb_nhi *nhi, struct tb_port *in, struct tb_port *out, u64 generation); + bool (*add_links)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); void (*dp_tunnel_changed)(struct tb_nhi *nhi, u8 in_port, u64 generation, bool active); bool (*dp_tunnel_awaits_display)(struct tb_nhi *nhi, struct tb_port *in); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 94eb59113dd66..a4d439f2394df 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -4388,6 +4388,12 @@ static bool tb_apple_add_links(struct tb_nhi *nhi) return ret; } +/* Hosts without ACPI tables may order their tunneled native ports themselves. */ +static bool tb_nhi_add_links(struct tb_nhi *nhi) +{ + return nhi->ops && nhi->ops->add_links && nhi->ops->add_links(nhi); +} + static void tb_cm_init(struct tb_cm *tcm) { INIT_LIST_HEAD(&tcm->tunnel_list); @@ -4425,7 +4431,8 @@ struct tb *tb_probe(struct tb_nhi *nhi) * before the PCIe/USB stack is resumed so complain here if we * found them missing. */ - if (!tb_apple_add_links(nhi) && !tb_acpi_add_links(nhi)) + if (!tb_nhi_add_links(nhi) && !tb_apple_add_links(nhi) && + !tb_acpi_add_links(nhi)) tb_warn(tb, "device links to tunneled native ports are missing!\n"); return tb; From a10defefab5d9585fa961f857e03e0873e522fc3 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:20:28 +1100 Subject: [PATCH 099/392] pmdomain: apple: clear a latched bit 11 with a PS reset on power-on On the Neo (T8140) ANE_SYS sometimes fails to come back on when genpd powers it on in the noirq resume of a system sleep: PS ane_sys: Failed to reach power state 0xf (requested 0x200f, now: 0x2b4f) Target 0xf, actual 0x4 (clock-gated), bit 11 set. genpd ignores the error there, the ANE finds ANE_SYS off at PM_POST_SUSPEND and quarantines itself until reboot, and every later power-on of the domain stops the same way. It happens in about one boot in four, within the first ten sleeps. Bit 11 is not a parent that comes up late here. A test kernel that kept watching the word saw no change for 20 ms; the fabric domains above ANE_SYS (0x1a8, 0x1e8, 0x228) and its siblings were all on. Gated again, the domain reports actual 0 at once, but bit 11 stays set (0x2b00), and the next power-on stops at 0x2b4f again. Writing bit 11 back does nothing. A PS reset pulse while the domain is off clears it (0x3300, then 0x2300), and the power-on then reaches 0x23ff in 8 us. So when a power-on times out with bit 11 set and the actual state short of the target, gate the domain, pulse the PS reset, release it and request the on state again, each step with the usual 100 us budget. The devices of the domain lose their state while it is off anyway. Domains whose bit 11 means a parent that is not up yet (wait_parent, T6000 GFX), strict-state and externally clocked domains are left as they were. A latch the reset does not clear still fails and logs. KUnit: four new cases (latch cleared, cleared in the noirq phase, a latch the reset does not clear, no reset for wait_parent domains or a plain stall). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/pmdomain/apple/pmgr-pwrstate-test.c | 87 +++++++++++++++++++++ drivers/pmdomain/apple/pmgr-pwrstate.c | 53 +++++++++++++ 2 files changed, 140 insertions(+) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate-test.c b/drivers/pmdomain/apple/pmgr-pwrstate-test.c index b73e2973cb6f1..b3a8cd1f5715c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate-test.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate-test.c @@ -15,6 +15,9 @@ struct pmgr_ps_test_reg { bool parent_read_error; unsigned int busy_writes; unsigned int auto_writes; /* writes that enabled automatic PM */ + bool latched; /* bit 11 latched: on stops at clock-gated, off still reaches 0 */ + bool latch_sticky; /* not even a PS reset clears it */ + unsigned int reset_writes; /* writes with the PS reset bit */ }; static int pmgr_ps_test_read(void *ctx, unsigned int reg, unsigned int *val) @@ -43,6 +46,19 @@ static int pmgr_ps_test_write(void *ctx, unsigned int reg, unsigned int val) u32 parent = r->val & APPLE_PMGR_BUSY; u32 actual; + if (val & APPLE_PMGR_PS_RESET) { + r->reset_writes++; + if (r->latched && !r->latch_sticky) { + r->latched = false; + parent = 0; + } + } + if (r->latched) { + actual = FIELD_GET(APPLE_PMGR_PS_TARGET, val) ? APPLE_PMGR_PS_CLKGATE : 0; + r->val = (val & ~(APPLE_PMGR_PS_ACTUAL | APPLE_PMGR_BUSY)) | APPLE_PMGR_BUSY | + FIELD_PREP(APPLE_PMGR_PS_ACTUAL, actual); + return 0; + } if (FIELD_GET(APPLE_PMGR_PS_TARGET, val) == APPLE_PMGR_PS_ACTIVE && (r->parent_stuck || r->parent_reads)) parent = APPLE_PMGR_BUSY; @@ -261,6 +277,73 @@ static void pmgr_ps_power_on_sleep_stuck_test(struct kunit *test) KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), APPLE_PMGR_PS_ACTIVE); } +/* + * A latched bit 11 (T8140 ANE_SYS after sleep: 0x2b4f) is cleared by a PS reset + * while the domain is off, and the power-on then completes. + */ +static void pmgr_ps_power_on_latch_cleared_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, false, &r); + r->latched = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), 0); + KUNIT_EXPECT_EQ(test, r->reset_writes, 1U); + KUNIT_EXPECT_FALSE(test, r->val & (APPLE_PMGR_BUSY | APPLE_PMGR_PS_RESET)); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val), APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ(test, r->auto_writes, 1U); +} + +/* The same in the noirq phase of a system sleep, where genpd ignores errors. */ +static void pmgr_ps_power_on_latch_sleep_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, true, &r); + ps->system_sleep = true; + r->latched = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), 0); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val), APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ(test, r->auto_writes, 0U); +} + +/* A latch the reset does not clear still fails, and the domain is requested off. */ +static void pmgr_ps_power_on_latch_sticky_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, true, &r); + r->latched = true; + r->latch_sticky = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, r->reset_writes, 1U); + KUNIT_EXPECT_FALSE(test, r->val & APPLE_PMGR_PS_RESET); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), APPLE_PMGR_PS_PWRGATE); + KUNIT_EXPECT_EQ(test, r->auto_writes, 0U); +} + +/* No PS reset where bit 11 means a parent not up yet, nor on a plain stall. */ +static void pmgr_ps_power_on_no_reset_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, false, &r); + ps->wait_parent = true; + r->latched = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, r->reset_writes, 0U); + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, true, &r); + r->stuck = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, r->reset_writes, 0U); +} + static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_test), KUNIT_CASE(pmgr_ps_power_off_undo_test), @@ -272,6 +355,10 @@ static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_parent_only_test), KUNIT_CASE(pmgr_ps_power_on_parent_read_error_test), KUNIT_CASE(pmgr_ps_power_on_sleep_stuck_test), + KUNIT_CASE(pmgr_ps_power_on_latch_cleared_test), + KUNIT_CASE(pmgr_ps_power_on_latch_sleep_test), + KUNIT_CASE(pmgr_ps_power_on_latch_sticky_test), + KUNIT_CASE(pmgr_ps_power_on_no_reset_test), {} }; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index b84bfc4432111..0c825a64b42bd 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -84,6 +84,48 @@ static int apple_pmgr_ps_wait_parent(struct apple_pmgr_ps *ps, u32 *cur) return err ?: ret; } +/* + * A power-on can stop with the actual state short of the target and bit 11 + * latched: on the T8140 ANE_SYS after system sleep the word reads 0x2b4f + * (target 0xf, actual 0x4) and stays so. That is not a parent that comes up + * late: gated again, the domain reaches actual 0 at once, but bit 11 stays + * set and every later power-on stops the same way. Only a PS reset clears it. + * Power the domain off, pulse the PS reset while it is off (its devices lose + * their state there anyway), then request @reg, the on state, again. + * Domains whose bit 11 means a parent that is not up yet (wait_parent) and + * those with their own state contract are left alone. + */ +static int apple_pmgr_ps_clear_latch(struct apple_pmgr_ps *ps, u32 reg, u32 *cur) +{ + u32 off = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_PS_PWRGATE; + int ret; + + if (ps->wait_parent || ps->strict_state || ps->externally_clocked || + !(*cur & APPLE_PMGR_BUSY) || + FIELD_GET(APPLE_PMGR_PS_ACTUAL, *cur) == APPLE_PMGR_PS_ACTIVE) + return -ETIMEDOUT; + + regmap_write(ps->regmap, ps->offset, off); + ret = regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + !FIELD_GET(APPLE_PMGR_PS_ACTUAL, *cur), 1, + APPLE_PMGR_PS_SET_TIMEOUT); + if (ret) + return ret; + regmap_write(ps->regmap, ps->offset, off | APPLE_PMGR_PS_RESET); + ret = regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + !(*cur & APPLE_PMGR_BUSY), 1, + APPLE_PMGR_PS_SET_TIMEOUT); + regmap_write(ps->regmap, ps->offset, off); + if (ret) + return ret; + regmap_write(ps->regmap, ps->offset, reg); + return regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, *cur) == + APPLE_PMGR_PS_ACTIVE && + !(*cur & APPLE_PMGR_BUSY), 1, + APPLE_PMGR_PS_SET_TIMEOUT); +} + static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool auto_enable) { int ret; @@ -174,6 +216,17 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a return ret; } + if (ret < 0 && pstate == APPLE_PMGR_PS_ACTIVE) { + u32 stalled = cur; + + if (!apple_pmgr_ps_clear_latch(ps, reg, &cur)) { + dev_info_ratelimited(ps->dev, + "PS %s: power-on stopped at 0x%x with bit 11 latched; cleared by a PS reset\n", + genpd->name, stalled); + ret = 0; + } + } + report: if (ret < 0) dev_err_ratelimited(ps->dev, From d1ad27fe1fbc6ffe98c492550a1e8440e7165f77 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 14:49:08 -0500 Subject: [PATCH 100/392] dt-bindings: media: apple: Describe the T6050 capture interface Describe named register resources, host stream pairs, camera power gates and loader-provided memory for the T6050 ISP. Retain the disabled hardware inventory format used by the other T6050 boards. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../bindings/media/apple,t6050-isp.yaml | 228 ++++++++++++++++-- 1 file changed, 206 insertions(+), 22 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,t6050-isp.yaml b/Documentation/devicetree/bindings/media/apple,t6050-isp.yaml index ea791487d642e..61892d3c8c17c 100644 --- a/Documentation/devicetree/bindings/media/apple,t6050-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t6050-isp.yaml @@ -4,19 +4,23 @@ $id: http://devicetree.org/schemas/media/apple,t6050-isp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T6050 image signal processor +title: Apple T6050 Image Signal Processor maintainers: - Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> + - Erik Bourget description: - The camera image signal processor of the T6050 MacBook Pros (ADT isp0). - Unlike earlier generations its ADT node lists the coprocessor window and - two PMGR windows rather than mailbox and handshake windows inside the ISP - block. Its buffers go through the three instances of the ISP DART. It is - described from the ADT so the device tree lists all of the SoC's - hardware; no driver supports this generation yet and the node stays - disabled. + The T6050 ISP controls the attached camera sensor and processes its frames + through firmware the bootloader loads. The firmware segments are described + as reserved memory mapped at the addresses the firmware runs them at. The + host exchanges commands and image buffers through streams 0 and 12 of three + DART instances. A capture watchdog must be serviced while streaming. + + Disabled hardware inventory nodes use three register windows, five interrupts, + three stream references and two power domains. Driver-ready nodes name their + five register windows and describe the host streams and individual power gates. + The bootloader supplies the reserved memory and platform ID before enabling them. select: properties: @@ -33,31 +37,124 @@ properties: - const: apple,isp reg: + minItems: 3 + maxItems: 5 + description: + Inventory nodes list the coprocessor and two PMGR windows. Named driver + resources list the coprocessor, mailbox, handshake, doorbell and watchdog. + + reg-names: items: - - description: Coprocessor registers - - description: The ISP's PMGR control window - - description: The ISP's PMGR power-state window + - const: coproc + - const: mbox + - const: gpio + - const: mbox2 + - const: wdt interrupts: - minItems: 5 + minItems: 1 maxItems: 5 iommus: - items: - - description: Stream 0 of the first ISP DART - - description: Stream 0 of the second ISP DART - - description: Stream 0 of the third ISP DART + minItems: 3 + maxItems: 6 + description: + Inventory nodes list stream 0 of each DART. Driver-ready nodes list + streams 0 and 12 of each of the three DARTs, in controller order. power-domains: - items: - - description: ISP system domain - - description: Camera sensor domain + minItems: 2 + maxItems: 8 + description: + Inventory nodes list system and sensor domains. Driver-ready nodes list + system, CPU, FE, sensor, DPRX, secure, BE and CLR domains, in that order. clocks: maxItems: 2 - status: - const: disabled + memory-region: + items: + - description: Firmware text loaded by the bootloader + - description: Firmware data loaded by the bootloader + - description: Firmware main memory window; host allocations follow it + + memory-region-names: + items: + - const: text + - const: data + - const: heap + + apple,dart-vm-size: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Size in bytes of the host allocation aperture used by the ISP driver. + + apple,platform-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Platform identifier passed in the firmware boot descriptor. + + apple,cam-connections-scheme: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Camera connection layout of the board passed in the firmware boot + descriptor. + + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Version of the firmware loaded by the bootloader. + + apple,firmware-compat: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Compatibility version supplied by the bootloader. + + apple,temporal-filter: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Firmware temporal-filter strength selected for the attached sensor. + + sensor-presets: + type: object + description: + Sensor firmware presets and the host output geometry associated with them. + patternProperties: + "^preset[0-9]+$": + type: object + properties: + apple,config-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Sensor configuration index understood by the firmware + apple,input-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Input width in pixels + - description: Input height in pixels + description: Sensor input dimensions + apple,output-size: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Output width in pixels + - description: Output height in pixels + description: Delivered frame dimensions + apple,crop: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: Horizontal crop origin in pixels + - description: Vertical crop origin in pixels + - description: Crop width in pixels + - description: Crop height in pixels + description: Cropped sensor rectangle + required: + - apple,config-index + - apple,input-size + - apple,output-size + - apple,crop + additionalProperties: false + additionalProperties: false required: - compatible @@ -65,6 +162,93 @@ required: - interrupts - iommus - power-domains - - status + +allOf: + - if: + required: + - reg-names + then: + properties: + reg: + minItems: 5 + interrupts: + maxItems: 1 + iommus: + minItems: 6 + power-domains: + minItems: 8 + clocks: false + required: + - apple,dart-vm-size + - sensor-presets + allOf: + - if: + properties: + status: + const: disabled + required: + - status + then: {} + else: + required: + - memory-region + - memory-region-names + - apple,platform-id + else: + properties: + reg: + maxItems: 3 + interrupts: + minItems: 5 + iommus: + maxItems: 3 + power-domains: + maxItems: 2 + status: + const: disabled + required: + - status additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@580000000 { + compatible = "apple,t6050-isp", "apple,isp"; + reg = <0x5 0x80000000 0x0 0x1604000>, + <0x5 0x82924000 0x0 0x40>, + <0x5 0x829241d0 0x0 0x20>, + <0x5 0x82924464 0x0 0x10>, + <0x5 0xc3b50000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "mbox2", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart0 12>, + <&isp_dart1 0>, <&isp_dart1 12>, + <&isp_dart2 0>, <&isp_dart2 12>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_ispsens0>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_be>, <&ps_isp_clr>; + memory-region = <&isp_fw_text>, <&isp_fw_data>, <&isp_heap>; + memory-region-names = "text", "data", "heap"; + apple,dart-vm-size = /bits/ 64 <0xde000000>; + apple,platform-id = <0x75>; + apple,cam-connections-scheme = <0x10>; + + sensor-presets { + preset0 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; + }; + }; From 562e67859dc0f356548bff552e0ad97115984cc5 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 14:49:09 -0500 Subject: [PATCH 101/392] media: apple: isp: Add T6050 camera capture support The T6050 ISP uses the H17 capture path with its own MMIO and DSID clear windows, host stream filter mask and mailbox interrupt enables. Describe these resources and keep the existing per-SoC behavior as the default. Exercise both the default and T6050 address-filter masks in KUnit. The source implementation captured 1080p NV12 frames at 30 fps on J716c. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../media/platform/apple/isp/isp-drv-test.c | 18 ++++- drivers/media/platform/apple/isp/isp-drv.c | 80 ++++++++++++++++++- drivers/media/platform/apple/isp/isp-drv.h | 3 + drivers/media/platform/apple/isp/isp-fw.c | 14 ++-- 4 files changed, 104 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c index dbddb079a0f57..53f731234e64c 100644 --- a/drivers/media/platform/apple/isp/isp-drv-test.c +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -43,15 +43,16 @@ static void isp_test_dapf_in_fw_mmio(struct kunit *test) * leaves every other byte alone. The bank is plain memory here: the driver * is arm64 only, where the MMIO accessors work on it unchanged. */ -static void isp_test_dapf_program(struct kunit *test) +static void isp_test_dapf_program_mask(struct kunit *test, u32 sidmask) { static const struct isp_dapf_window windows[] = { { 0x380704000, 0x380704063 }, { 0x382824000, 0x382824004 }, }; - static const struct apple_isp_hw hw = { + const struct apple_isp_hw hw = { .dapf = windows, .num_dapf = ARRAY_SIZE(windows), + .dapf_sidmask = sidmask, }; size_t size = (ARRAY_SIZE(windows) + 1) * DAPF_WINDOW_STRIDE; u8 expect[DAPF_WINDOW_STRIDE]; @@ -76,7 +77,7 @@ static void isp_test_dapf_program(struct kunit *test) put_unaligned_le32(0, &expect[0x04]); put_unaligned_le64(windows[i].start, &expect[0x08]); put_unaligned_le64(windows[i].end, &expect[0x10]); - put_unaligned_le32(1, &expect[0x20]); + put_unaligned_le32(sidmask ?: 1, &expect[0x20]); KUNIT_EXPECT_MEMEQ_MSG(test, bank + i * DAPF_WINDOW_STRIDE, expect, sizeof(expect), "window %u", i); } @@ -86,9 +87,20 @@ static void isp_test_dapf_program(struct kunit *test) ISP_TEST_DAPF_FILL, DAPF_WINDOW_STRIDE)); } +static void isp_test_dapf_program(struct kunit *test) +{ + isp_test_dapf_program_mask(test, 0); +} + +static void isp_test_dapf_program_t6050(struct kunit *test) +{ + isp_test_dapf_program_mask(test, 0xcc3); +} + static struct kunit_case isp_dapf_test_cases[] = { KUNIT_CASE(isp_test_dapf_in_fw_mmio), KUNIT_CASE(isp_test_dapf_program), + KUNIT_CASE(isp_test_dapf_program_t6050), {} }; diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 627a9bdb9e94d..21dd9cdc1be2a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -255,6 +255,7 @@ static int apple_isp_map_dapf(struct apple_isp *isp) int apple_isp_program_dapf(struct apple_isp *isp) { const struct apple_isp_hw *hw = isp->hw; + u32 sidmask = hw->dapf_sidmask ?: DAPF_CTRL2_ISP; void __iomem *w; unsigned int i; @@ -264,7 +265,7 @@ int apple_isp_program_dapf(struct apple_isp *isp) writeq(hw->dapf[i].start, w + DAPF_START); writeq(hw->dapf[i].end, w + DAPF_END); writel(DAPF_CTRL0_ISP, w + DAPF_CTRL0); - writel(DAPF_CTRL2_ISP, w + DAPF_CTRL2); + writel(sidmask, w + DAPF_CTRL2); } /* Only bits 7:0 of CTRL0 and bits 63:2 of END read back. */ @@ -274,7 +275,7 @@ int apple_isp_program_dapf(struct apple_isp *isp) readq(w + DAPF_START) != hw->dapf[i].start || (readq(w + DAPF_END) | 3) != (hw->dapf[i].end | 3) || (readl(w + DAPF_CTRL0) & 0xff) != DAPF_CTRL0_ISP || - readl(w + DAPF_CTRL2) != DAPF_CTRL2_ISP) { + readl(w + DAPF_CTRL2) != sidmask) { dev_err(isp->dev, "DAPF window %u (0x%llx-0x%llx) failed readback\n", i, hw->dapf[i].start, hw->dapf[i].end); return -EIO; @@ -1141,7 +1142,82 @@ static const struct apple_isp_hw apple_isp_hw_t8142 = { .setfile_stride_fixup = true, }; +/* T6050 ISP17c: CPU-visible firmware MMIO windows and DART filters. */ +static const struct isp_mmio_window apple_isp_fw_mmio_t6050[] = { + { 0x280680000, 0x8000 }, + { 0x280900000, 0x4000 }, + { 0x284240000, 0x28000 }, + { 0x284e3c000, 0x4000 }, + { 0x286724000, 0x4000 }, + { 0x29062c000, 0x20000 }, + { 0x509660000, 0x4000 }, + { 0x2220004000, 0x14000 }, + { 0x2222004000, 0x14000 }, + { 0x2224004000, 0x14000 }, + { 0x2226004000, 0x14000 }, + { 0x2228004000, 0x14000 }, + { 0x222a004000, 0x14000 }, + { 0x222c004000, 0x14000 }, + { 0x222e004000, 0x14000 }, +}; + +static const struct isp_dapf_window apple_isp_dapf_t6050[] = { + { 0x2809003b0, 0x2809003f3 }, + { 0x280680000, 0x280687fff }, + { 0x284e3c000, 0x284e3c003 }, + { 0x29062c000, 0x290648003 }, + { 0x509660000, 0x509660003 }, + { 0x286724000, 0x286724004 }, + { 0x284240000, 0x284264000 }, + { 0x2220004000, 0x22200162fc }, + { 0x2222004000, 0x22220162fc }, + { 0x2224004000, 0x22240162fc }, + { 0x2226004000, 0x22260162fc }, + { 0x2228004000, 0x22280162fc }, + { 0x222a004000, 0x222a0162fc }, + { 0x222c004000, 0x222c0162fc }, + { 0x222e004000, 0x222e0162fc }, +}; + +static const u64 apple_isp_dsid_t6050[] = { + 0x2220016000, + 0x2222016000, + 0x2224016000, + 0x2226016000, + 0x2228016000, + 0x222a016000, + 0x222c016000, + 0x222e016000, +}; + +static const struct apple_isp_hw apple_isp_hw_t6050 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, + .dsid_count = ARRAY_SIZE(apple_isp_dsid_t6050), + .dsid_clr_bases = apple_isp_dsid_t6050, + .dsid_clr_range0 = 0x2fc, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + .mbox_extra_enables = true, + .scl1 = true, + .meta_size = ISP_META_SIZE_T8140, + .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, + .fw_mmio = apple_isp_fw_mmio_t6050, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t6050), + .dapf = apple_isp_dapf_t6050, + .num_dapf = ARRAY_SIZE(apple_isp_dapf_t6050), + .dapf_sidmask = 0xcc3, + .coproc_control = ISP_COPROC_CONTROL_T8140, + .coproc_status = ISP_COPROC_STATUS_T8140, + .coproc_quiesce = ISP_COPROC_QUIESCE_T8140, + .boot_mode = 1, + .shared_size = 0xde000000, + .fw_restore = true, + .deferred_start = true, + .setfile_stride_fixup = true, +}; + static const struct of_device_id apple_isp_of_match[] = { + { .compatible = "apple,t6050-isp", .data = &apple_isp_hw_t6050 }, { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, { .compatible = "apple,t8122-isp", .data = &apple_isp_hw_t8122 }, diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 9056c3d7af143..66fa3e1264576 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -162,6 +162,7 @@ struct apple_isp_hw { u64 pmu_base; int dsid_count; + const u64 *dsid_clr_bases; /* windows of the H17 multi broadcast-clear command, the unused all ones */ u8 dsid_slots; u64 dsid_clr_base0; @@ -194,6 +195,7 @@ struct apple_isp_hw { u32 mbox2_offset; /* further interrupt enable and routing words (ISP17a) */ bool mbox_irq_route; + bool mbox_extra_enables; /* coprocessor control register, 0 for ISP_COPROC_CONTROL */ u32 coproc_control; /* coprocessor status register, 0 for ISP_COPROC_STATUS */ @@ -216,6 +218,7 @@ struct apple_isp_hw { */ const struct isp_dapf_window *dapf; unsigned int num_dapf; + u32 dapf_sidmask; /* ISP_GPIO_6 boot mode */ u32 boot_mode; diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 28727ef24fbb6..302b18cd189c6 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -251,7 +251,7 @@ static irqreturn_t apple_isp_isr_thread(int irq, void *dev) static void isp_disable_irq(struct apple_isp *isp) { isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0x0); - if (isp->hw->mbox_irq_route) { + if (isp->hw->mbox_irq_route || isp->hw->mbox_extra_enables) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, 0x0); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, 0x0); } @@ -273,13 +273,14 @@ static int isp_enable_irq(struct apple_isp *isp) isp_dbg(isp, "about to enable interrupts...\n"); isp_mbox_write32(isp, isp->hw->mbox_irq_enable, 0xf); - if (isp->hw->mbox_irq_route) { + if (isp->hw->mbox_irq_route || isp->hw->mbox_extra_enables) { isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE1_T8140, ISP_MBOX_IRQ_ENABLE1_T8140_VAL); isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE2_T8140, ISP_MBOX_IRQ_ENABLE2_T8140_VAL); - isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, - ISP_MBOX_IRQ_ROUTE_T8140_VAL); + if (isp->hw->mbox_irq_route) + isp_mbox_write32(isp, ISP_MBOX_IRQ_ROUTE_T8140, + ISP_MBOX_IRQ_ROUTE_T8140_VAL); } return 0; @@ -934,12 +935,13 @@ static int isp_set_dsid_clr(struct apple_isp *isp) unsigned int slots = max_t(unsigned int, hw->dsid_count, hw->dsid_slots); - if (WARN_ON(hw->dsid_count > ARRAY_SIZE(used) || + if (WARN_ON((!hw->dsid_clr_bases && hw->dsid_count > ARRAY_SIZE(used)) || slots > ARRAY_SIZE(bases))) return -EINVAL; for (unsigned int i = 0; i < slots; i++) - bases[i] = i < hw->dsid_count ? used[i] : U64_MAX; + bases[i] = i < hw->dsid_count ? + (hw->dsid_clr_bases ? hw->dsid_clr_bases[i] : used[i]) : U64_MAX; return isp_cmd_set_dsid_clr_multi_bc(isp, bases, slots, hw->dsid_clr_range0); From ae6b06fd7893410afdb4504cdf03c681828240b0 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 14:49:09 -0500 Subject: [PATCH 102/392] dt-bindings: iommu: apple,dart: Add the T6050 ISP compatible Identify the ISP DARTs separately so the driver can reset host-owned streams without disturbing the secure core's translation roots. Developed with Claude Code. Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/iommu/apple,dart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..4d33293082a62 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -48,6 +48,7 @@ properties: - apple,t6031-dart - apple,t6041-dart - apple,t6050-dart + - apple,t6050-isp-dart - apple,t8122-dart - apple,t8132-dart - apple,t8140-dart From b6e1439872e61d8981954b4fbdecdea4640af6ce Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 14:49:09 -0500 Subject: [PATCH 103/392] iommu: apple-dart: Limit T6050 ISP reset to host streams The T6050 ISP shares its DARTs with a secure core. A full reset clears translation roots outside the host's ownership. Reset and invalidate only the host streams, admit client mappings on streams 0 and 12, and preserve the retained translation state across system sleep. Refuse a locked controller or a pending fault at reset. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/iommu/apple-dart.c | 46 +++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..5e675e0416a37 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -299,6 +299,7 @@ struct apple_dart { u32 locked : 1; u32 tunneled : 1; u32 per_stream : 1; + u32 isp_host : 1; u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; @@ -2001,11 +2002,45 @@ static void apple_dart_managed_quarantine(struct apple_dart *dart) spin_unlock_irqrestore(&dart->managed_lock, flags); } +/* ISP streams 4/5/8/9 belong to its secure core. Never reset those roots. */ +static int apple_dart_isp_host_reset(struct apple_dart *dart) +{ + const u32 host = BIT(0) | BIT(1) | BIT(6) | BIT(7) | + BIT(10) | BIT(11) | BIT(12); + struct apple_dart_stream_map map = { .dart = dart }; + unsigned int sid; + int ret; + + if (dart->hw->type != DART_T8110 || dart->num_streams != 16 || + dart->locked || (apple_dart_readl(dart, DART_T8110_ERROR) & DART_T8110_ERROR_FLAG)) + return -EBUSY; + for (sid = 0; sid < 16; sid++) + if (host & BIT(sid)) + __set_bit(sid, map.sidmap); + apple_dart_hw_disable_dma(&map); + apple_dart_hw_clear_all_ttbrs(&map); + apple_dart_hw_apply_remaps(dart); + apple_dart_writel(dart, host, DART_T8110_ENABLE_STREAMS); + for (sid = 0; sid < 16; sid++) { + if (!(host & BIT(sid))) + continue; + bitmap_zero(map.sidmap, DART_MAX_STREAMS); + __set_bit(sid, map.sidmap); + ret = dart->hw->invalidate_tlb(&map); + if (ret) + return ret; + } + apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + return 0; +} + static int apple_dart_hw_reset(struct apple_dart *dart) { struct apple_dart_stream_map stream_map; int i; + if (dart->isp_host) + return apple_dart_isp_host_reset(dart); if (dart->managed_sids || dart->cold_ane) return -EBUSY; stream_map.dart = dart; @@ -2990,6 +3025,8 @@ static int apple_dart_of_xlate(struct device *dev, return i; sid = args->args[0]; + if (dart->isp_host && sid != 0 && sid != 12) + return -EINVAL; if (dart->cold_ane && sid != 0) return -EPERM; if (dart->managed_sids && @@ -4243,7 +4280,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->cold_ane = of_device_is_compatible(dev->of_node, "apple,t8140-ane-dart"); if (dart->cold_ane && !IS_BUILTIN(CONFIG_APPLE_DART)) return -EOPNOTSUPP; - dart->per_stream = of_device_is_compatible(dev->of_node, "apple,t8140-dart") || + dart->isp_host = of_device_is_compatible(dev->of_node, "apple,t6050-isp-dart"); + dart->power_retained = dart->isp_host; + dart->per_stream = dart->isp_host || + of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart"); if (of_property_present(dev->of_node, "apple,managed-sids")) { u32 ids[2]; @@ -4534,6 +4574,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); + if (dart->isp_host && dart->locked) { + ret = -EBUSY; + goto err_clk_disable; + } /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); /* Preserve our retained-root path when no reserved-table handoff exists. */ From ec5c067ab2eec0f3b57fb31e154627f2b212be9a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:17:40 +1100 Subject: [PATCH 104/392] dmaengine: apple-sio: add the T6041 SIO The M4 Pro/Max (T6041) SIO carries the DisplayPort/HDMI audio streams of the external display processors. Its firmware reports protocol 10, as T6030's does, but its boot contract differs, so the compatible selects it and protocol 10 alone does not: - the boot loader's control tables go to the firmware in a fixed order (panic buffer, map ranges, wrapper tunables, the optional register tables, a power-gating scalar 0x2a = 0, DMA shims, device types, performance buffer), each as its address and then its exact size; - the driver owns the segment array (0xaaa entries of 12 bytes) and the response array (0x1cc0 entries of 16 bytes, one per endpoint and tag) and publishes both as entry counts; DMA is registered only after the response count is acknowledged; - a channel configuration is six words: shape, timeout, FIFO size, refill threshold 0x600, the host default timeout 5 and a zero word; - recycled period descriptors are cleared before reuse. The probe refuses tables that are not exactly that contract (a missing, repeated, unknown or wrongly sized table, the DMA arrays or the scalar from the loader, or a table outside the windows the loader mapped) before it starts the coprocessor, and refuses firmware that reports another protocol. The firmware itself is preloaded by iBoot; nothing is loaded. KUnit: apple-sio-t6041 covers admission, order, optional tables, every ACK failure, the array counts and the six-word configuration; the existing protocol 9 and T6030 suites are unchanged and pass. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/dma/Kconfig | 10 + drivers/dma/apple-sio-t6041-test.c | 276 +++++++++++++++++++++++++ drivers/dma/apple-sio.c | 313 ++++++++++++++++++++++++++++- 3 files changed, 588 insertions(+), 11 deletions(-) create mode 100644 drivers/dma/apple-sio-t6041-test.c diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 46eb147157185..12b59fc898a29 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -125,6 +125,16 @@ config APPLE_SIO_T6030_KUNIT_TEST against tables a T6030-aware boot loader builds, an earlier loader's tables, and malformed, truncated and out-of-window ones. +config APPLE_SIO_T6041_KUNIT_TEST + bool "KUnit tests for the T6041 SIO boot contract" if !KUNIT_ALL_TESTS + depends on APPLE_SIO=y && KUNIT=y + default KUNIT_ALL_TESTS + help + Check the T6041 (M4 Pro/Max) SIO boot: which boot-loader tables + are admitted and at which sizes, the order the firmware takes them + in with the power-gating scalar, the DMA array counts, and the + six-word channel configuration. + config ARM_DMA350 tristate "Arm DMA-350 support" depends on ARM || ARM64 || COMPILE_TEST diff --git a/drivers/dma/apple-sio-t6041-test.c b/drivers/dma/apple-sio-t6041-test.c new file mode 100644 index 0000000000000..1c9b1a56f8ae7 --- /dev/null +++ b/drivers/dma/apple-sio-t6041-test.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by apple-sio.c: the T6041 SIO boot contract. */ +#include + +struct sio_t6041_test { + struct device dev; + struct sio_data sio; + u32 keys[32]; + u32 values[32]; + unsigned int count; + u32 fail_key; +}; + +#define T6041_WIN 0x10000118000ULL /* a window the loader mapped */ + +/* What a T6041-aware loader passes: J616s has no optional tables. */ +static const u32 sio_t6041_loader[] = { + 0x1e, 0x10000118, 0x1f, 0x30, + 0x0f, 0x1000011c, 0x10, 0x1e000, + 0x30d, 0x1000013c, 0x30e, 0x4000, + 0x1a, 0x10000140, 0x1b, 0x40, + 0x22, 0x10000141, 0x23, 0xd8, + 0x1c, 0x10000142, 0x1d, 0x48, +}; + +/* The firmware's order, the scalar after the register tables, before the shims. */ +static const u32 sio_t6041_expected[][2] = { + { 0x0f, 0x1000011c }, { 0x10, 0x1e000 }, + { 0x1a, 0x10000140 }, { 0x1b, 0x40 }, + { 0x1e, 0x10000118 }, { 0x1f, 0x30 }, + { 0x2a, 0 }, + { 0x22, 0x10000141 }, { 0x23, 0xd8 }, + { 0x1c, 0x10000142 }, { 0x1d, 0x48 }, + { 0x30d, 0x1000013c }, { 0x30e, 0x4000 }, +}; + +static const struct sio_t6030_window sio_t6041_windows[] = { + { T6041_WIN, 0x2d000 }, +}; + +static struct sio_t6041_test *sio_t6041_alloc(struct kunit *test) +{ + struct sio_t6041_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + t->sio.dev = &t->dev; + t->sio.hw = &sio_hw_t6041; + t->fail_key = U32_MAX; + return t; +} + +static int sio_t6041_set_param(struct sio_data *sio, u32 key, u32 val) +{ + struct sio_t6041_test *t = container_of(sio, struct sio_t6041_test, sio); + + if (t->count >= ARRAY_SIZE(t->keys)) + return -E2BIG; + t->keys[t->count] = key; + t->values[t->count++] = val; + return key == t->fail_key ? -ETIME : 0; +} + +static int sio_t6041_run(struct sio_t6041_test *t, const u32 *params, unsigned int n) +{ + u32 addr[ARRAY_SIZE(sio_t6041_params)] = {}; + u32 size[ARRAY_SIZE(sio_t6041_params)] = {}; + int err; + + t->count = 0; + err = sio_t6041_collect(params, n, sio_t6041_windows, ARRAY_SIZE(sio_t6041_windows), + addr, size); + if (err) + return err; + return sio_t6041_send_params(&t->sio, addr, size, sio_t6041_set_param); +} + +static void sio_t6041_order_test(struct kunit *test) +{ + struct sio_t6041_test *t = sio_t6041_alloc(test); + u32 params[ARRAY_SIZE(sio_t6041_loader)]; + unsigned int i, n = ARRAY_SIZE(sio_t6041_loader), rev; + + for (rev = 0; rev < 2; rev++) { + for (i = 0; i < n; i += 2) { + unsigned int src = rev ? n - i - 2 : i; + + params[i] = sio_t6041_loader[src]; + params[i + 1] = sio_t6041_loader[src + 1]; + } + KUNIT_ASSERT_EQ(test, sio_t6041_run(t, params, n), 0); + KUNIT_ASSERT_EQ(test, t->count, ARRAY_SIZE(sio_t6041_expected)); + for (i = 0; i < t->count; i++) { + KUNIT_EXPECT_EQ(test, t->keys[i], sio_t6041_expected[i][0]); + KUNIT_EXPECT_EQ(test, t->values[i], sio_t6041_expected[i][1]); + } + } +} + +static void sio_t6041_optional_test(struct kunit *test) +{ + struct sio_t6041_test *t = sio_t6041_alloc(test); + u32 params[ARRAY_SIZE(sio_t6041_loader) + 8]; + unsigned int n = ARRAY_SIZE(sio_t6041_loader); + + memcpy(params, sio_t6041_loader, sizeof(sio_t6041_loader)); + /* SIO registers, then misc tunables: both go before the scalar, in table order */ + params[n++] = 0x28; params[n++] = 0x10000143; + params[n++] = 0x29; params[n++] = 0x18; + params[n++] = 0x20; params[n++] = 0x10000144; + params[n++] = 0x21; params[n++] = 0x30; + KUNIT_ASSERT_EQ(test, sio_t6041_run(t, params, n), 0); + KUNIT_ASSERT_EQ(test, t->count, ARRAY_SIZE(sio_t6041_expected) + 4); + KUNIT_EXPECT_EQ(test, t->keys[6], 0x20U); + KUNIT_EXPECT_EQ(test, t->keys[7], 0x21U); + KUNIT_EXPECT_EQ(test, t->keys[8], 0x28U); + KUNIT_EXPECT_EQ(test, t->values[9], 0x18U); + KUNIT_EXPECT_EQ(test, t->keys[10], 0x2aU); + KUNIT_EXPECT_EQ(test, t->keys[11], 0x22U); + /* Half an optional table */ + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n - 2), -EINVAL); + KUNIT_EXPECT_EQ(test, t->count, 0U); +} + +static void sio_t6041_reject_test(struct kunit *test) +{ + struct sio_t6041_test *t = sio_t6041_alloc(test); + u32 params[ARRAY_SIZE(sio_t6041_loader) + 2]; + unsigned int n = ARRAY_SIZE(sio_t6041_loader), i; + /* Each required size, one off: protocol-9 shims, eight device types... */ + static const u32 wrong[][2] = { + { 0x23, 0xa0 }, { 0x23, 6 * 32 }, { 0x1d, 0x40 }, { 0x1b, 0x50 }, + { 0x10, 0x1d000 }, { 0x1f, 0x18 }, { 0x30e, 0x8000 }, + }; + /* Keys the loader must not pass: arrays, the scalar, unknown */ + static const u32 foreign[] = { 0x0b, 0x0c, 0x01, 0x02, 0x2a, 0x1234, 0x10b }; + + for (i = 0; i < ARRAY_SIZE(wrong); i++) { + unsigned int j; + + memcpy(params, sio_t6041_loader, sizeof(sio_t6041_loader)); + for (j = 0; j < n; j += 2) + if (params[j] == wrong[i][0]) + params[j + 1] = wrong[i][1]; + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n), -EINVAL); + } + for (i = 0; i < ARRAY_SIZE(foreign); i++) { + memcpy(params, sio_t6041_loader, sizeof(sio_t6041_loader)); + params[n] = foreign[i]; + params[n + 1] = 0x10000145; + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n + 2), -EINVAL); + } + /* A duplicate, a missing required table, an odd count, nothing */ + memcpy(params, sio_t6041_loader, sizeof(sio_t6041_loader)); + params[n] = 0x1e; + params[n + 1] = 0x10000118; + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n + 2), -EINVAL); + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n - 2), -EINVAL); + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n - 1), -EINVAL); + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, 0), -EINVAL); + /* Outside the mapped window: below it, and running past its end */ + params[1] = 0x10000100; + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n), -EINVAL); + params[1] = 0x10000143; /* 0x30 bytes from the window's last page: fits */ + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n), 0); + params[5] = 0x10000130; /* the panic buffer would end past the window */ + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, params, n), -EINVAL); + KUNIT_EXPECT_EQ(test, t->count, 0U); +} + +static void sio_t6041_ack_failure_test(struct kunit *test) +{ + struct sio_t6041_test *t = sio_t6041_alloc(test); + unsigned int fail; + + for (fail = 0; fail < ARRAY_SIZE(sio_t6041_expected); fail++) { + t->fail_key = sio_t6041_expected[fail][0]; + KUNIT_EXPECT_EQ(test, sio_t6041_run(t, sio_t6041_loader, + ARRAY_SIZE(sio_t6041_loader)), -ETIME); + /* Nothing follows a refused parameter, a table's size included. */ + KUNIT_EXPECT_EQ(test, t->count, fail + 1); + } +} + +static void sio_t6041_arrays_test(struct kunit *test) +{ + struct sio_t6041_test *t = sio_t6041_alloc(test); + static const u32 keys[] = { 1, 2, 0x0b, 0x0c }; + static const u32 values[] = { 0x10200000, 0xaaa, 0x10300000, 0x1cc0 }; + unsigned int i; + + KUNIT_EXPECT_EQ(test, sio_t6041_arrays.segments * sio_t6041_arrays.segment_size, + 0x7ff8U); + KUNIT_EXPECT_EQ(test, sio_t6041_arrays.responses * sio_t6041_arrays.response_size, + 0x1cc00U); + /* Every endpoint and tag has its response: index endpoint * 32 + tag - 1. */ + KUNIT_EXPECT_LT(test, (NCHANNELS_MAX - 1) * 32 + SIO_NTAGS - 2, + (int)sio_t6041_arrays.responses); + t->sio.arrays = &sio_t6041_arrays; + t->sio.shmem_iova = 0x10200000000ULL; + t->sio.responses_iova = 0x10300000000ULL; + KUNIT_ASSERT_EQ(test, sio_send_v9_shmem(&t->sio, sio_t6041_set_param), 0); + KUNIT_ASSERT_EQ(test, t->count, ARRAY_SIZE(keys)); + for (i = 0; i < t->count; i++) { + KUNIT_EXPECT_EQ(test, t->keys[i], keys[i]); + KUNIT_EXPECT_EQ(test, t->values[i], values[i]); + } + /* The response count's ACK is the last step: a refusal there fails the boot. */ + t->count = 0; + t->fail_key = 0x0c; + KUNIT_EXPECT_EQ(test, sio_send_v9_shmem(&t->sio, sio_t6041_set_param), -ETIME); + KUNIT_EXPECT_EQ(test, t->count, 4U); +} + +static void sio_t6041_config_test(struct kunit *test) +{ + struct sio_shmem_chan_config cfg = { + .datashape = 2, .timeout = 0, .fifo = 0x800, + .threshold = sio_hw_t6041.fifo_threshold, .limit = sio_hw_t6041.cfg_word4, + }; + static const u32 t6041[] = { 2, 0, 0x800, 0x600, 5, 0 }; + u32 words[8]; + + memset(words, 0xa5, sizeof(words)); + sio_write_config(&sio_hw_t6041, words, &cfg); + KUNIT_EXPECT_MEMEQ(test, words, t6041, sizeof(t6041)); + KUNIT_EXPECT_EQ(test, words[6], 0xa5a5a5a5U); + + /* The other layouts keep their five words and leave the sixth alone. */ + memset(words, 0xa5, sizeof(words)); + cfg.threshold = sio_hw_t6030.fifo_threshold; + cfg.limit = sio_hw_t6030.cfg_word4; + sio_write_config(&sio_hw_t6030, words, &cfg); + KUNIT_EXPECT_EQ(test, words[3], 0x600U); + KUNIT_EXPECT_EQ(test, words[4], 0x800U); + KUNIT_EXPECT_EQ(test, words[5], 0xa5a5a5a5U); +} + +static void sio_t6041_match_test(struct kunit *test) +{ + const struct of_device_id *id; + + for (id = sio_of_match; id->compatible[0]; id++) + if (!strcmp(id->compatible, "apple,t6041-sio")) + break; + KUNIT_ASSERT_TRUE(test, id->compatible[0] != '\0'); + KUNIT_EXPECT_PTR_EQ(test, id->data, (const void *)&sio_hw_t6041); + KUNIT_EXPECT_EQ(test, sio_hw_t6041.protocol, 10U); + KUNIT_EXPECT_TRUE(test, sio_hw_t6041.clear_period_desc); + KUNIT_EXPECT_PTR_EQ(test, sio_hw_t6041.arrays, &sio_t6041_arrays); + /* Protocol 10 alone is T6030's too, which keeps its own contract. */ + KUNIT_EXPECT_EQ(test, sio_hw_t6030.protocol, 0U); + KUNIT_EXPECT_NULL(test, sio_hw_t6030.arrays); + /* The one-segment descriptor overlay: handle slot * 4, IOVA at +72, length at +80 */ + KUNIT_EXPECT_EQ(test, 56 + offsetof(struct sio_coproc_desc, flag), 60U); + KUNIT_EXPECT_EQ(test, 56 + offsetof(struct sio_coproc_desc, iova), 72U); + KUNIT_EXPECT_EQ(test, 56 + offsetof(struct sio_coproc_desc, size), 80U); + KUNIT_EXPECT_EQ(test, sizeof(struct sio_coproc_desc), 4U * SIO_T6041_SEGMENT_SIZE); +} + +static struct kunit_case sio_t6041_test_cases[] = { + KUNIT_CASE(sio_t6041_order_test), + KUNIT_CASE(sio_t6041_optional_test), + KUNIT_CASE(sio_t6041_reject_test), + KUNIT_CASE(sio_t6041_ack_failure_test), + KUNIT_CASE(sio_t6041_arrays_test), + KUNIT_CASE(sio_t6041_config_test), + KUNIT_CASE(sio_t6041_match_test), + {} +}; + +static struct kunit_suite sio_t6041_test_suite = { + .name = "apple-sio-t6041", + .test_cases = sio_t6041_test_cases, +}; + +kunit_test_suite(sio_t6041_test_suite); diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index f2d3afaaf9bc3..94c48dffb983b 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -151,10 +151,20 @@ struct sio_chan { typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); +struct sio_data; +struct sio_dma_arrays; + struct sio_hw_data { + u32 protocol; /* the only protocol STARTED may report, or 0 */ u32 fifo_threshold; + u32 cfg_word4; /* configuration word 4 */ + bool cfg_six_words; /* the configuration ends in a zero sixth word */ bool clear_period_desc; + /* DMA arrays owned by the driver, if not protocol 9's */ + const struct sio_dma_arrays *arrays; int (*check_boot_tables)(struct device *dev); + /* The boot loader's tables in this firmware's order, if not as listed */ + int (*send_dt_params)(struct sio_data *sio); }; struct sio_data { @@ -168,6 +178,7 @@ struct sio_data { dma_addr_t shmem_iova; void *responses; dma_addr_t responses_iova; + const struct sio_dma_arrays *arrays; /* when the driver owns them */ struct sio_coproc_desc *shmem_desc_base; unsigned long *desc_allocated; @@ -206,6 +217,7 @@ static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, sio_ack_callback ack_callback, void *cookie); static int sio_call(struct sio_data *sio, u64 msg); +static int sio_set_param(struct sio_data *sio, u32 key, u32 val); /* * Boot tables of the T6030 SIO firmware, protocol 10. The boot loader builds them from the ADT @@ -363,17 +375,265 @@ static int sio_t6030_check_boot_tables(struct device *dev) return 0; } +/* + * The T6041 (M4 Pro/Max) SIO firmware. It reports protocol 10 as T6030's + * does, but its boot is a different contract, so the protocol number does + * not select it; the compatible does: + * + * - the control tables go in this order, a table's address (IOVA page) + * then its size in bytes, which also completes it. Optional tables the + * ADT does not describe are left out whole. Key 0x2a, after the register + * tables, is a scalar: 0 leaves the SIO CPU's power gating off. The shim + * size (0x23) comes before the device types and lets the firmware set up + * its shims. + * - the segment array (keys 1, 2) and the response array (0x0b, 0x0c) are + * published last, as entry counts: 0xaaa segments of 12 bytes and 0x1cc0 + * responses of 16 bytes (one per endpoint and tag). The ACK of 0x0c + * brings up the DMA service. The driver owns both arrays. + * - a channel configuration is six words: data shape, timeout, FIFO size, + * refill threshold, the host's default timeout (5) and a zero word. + * + * The boot loader builds the control tables from the ADT and maps them + * into the windows it lists in memory-region. + */ +#define SIO_T6041_SEGMENTS 0xaaa +#define SIO_T6041_SEGMENT_SIZE 12 +#define SIO_T6041_RESPONSES 0x1cc0 +#define SIO_T6041_RESPONSE_SIZE 16 +#define SIO_T6041_KEY_CPU_POWER_GATE 0x2a + +enum sio_t6041_kind { + SIO_T6041_REQUIRED, /* an address and its exact size */ + SIO_T6041_OPTIONAL, /* both or neither, any non-zero size */ + SIO_T6041_SCALAR, /* the driver's own value, never the loader's */ +}; + +static const struct sio_t6041_param { + u16 key; + u8 kind; + u32 value; /* the size of a required table, or a scalar */ +} sio_t6041_params[] = { + { 0x0f, SIO_T6041_REQUIRED, 0x1e000 }, /* panic buffer */ + { 0x1a, SIO_T6041_REQUIRED, 4 * 16 }, /* map ranges */ + { 0x1e, SIO_T6041_REQUIRED, 0x30 }, /* ASC-wrapper tunables */ + { 0x20, SIO_T6041_OPTIONAL }, /* miscellaneous tunables */ + { 0x24, SIO_T6041_OPTIONAL }, /* power-state registers */ + { 0x28, SIO_T6041_OPTIONAL }, /* SIO registers */ + { SIO_T6041_KEY_CPU_POWER_GATE, SIO_T6041_SCALAR, 0 }, + { 0x22, SIO_T6041_REQUIRED, 6 * 36 }, /* DMA shims */ + { 0x1c, SIO_T6041_REQUIRED, 9 * 8 }, /* device types */ + { 0x30d, SIO_T6041_REQUIRED, 0x4000 }, /* performance endpoint */ +}; + +/* Where a loader key goes in sio_t6041_params, or -1 */ +static int sio_t6041_param_index(u32 key, bool *size) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sio_t6041_params); i++) { + const struct sio_t6041_param *p = &sio_t6041_params[i]; + + if (p->kind == SIO_T6041_SCALAR) + continue; + if (key == p->key || key == p->key + 1) { + *size = key == p->key + 1; + return i; + } + } + return -1; +} + +/* + * Collect the loader's T6041 control tables into addr[]/size[] (indexed as + * sio_t6041_params), refusing anything not exactly the contract above: an + * unknown or repeated key, a scalar or a DMA array from the loader, a + * required table missing or of another size, half an optional table, or a + * table outside every window the loader mapped for this SIO. + */ +static int sio_t6041_collect(const u32 *params, unsigned int nparams, + const struct sio_t6030_window *windows, unsigned int nwindows, + u32 *addr, u32 *size) +{ + unsigned long seen = 0; + unsigned int i, j; + + static_assert(2 * ARRAY_SIZE(sio_t6041_params) <= BITS_PER_LONG); + if (!nparams || nparams % 2 || nparams > 2 * SIO_T6030_PARAMS_MAX) + return -EINVAL; + for (i = 0; i < nparams; i += 2) { + bool is_size; + int idx = sio_t6041_param_index(params[i], &is_size); + unsigned int bit; + + if (idx < 0) + return -EINVAL; + bit = 2 * idx + is_size; + if (seen & BIT(bit)) + return -EINVAL; + seen |= BIT(bit); + (is_size ? size : addr)[idx] = params[i + 1]; + } + + for (i = 0; i < ARRAY_SIZE(sio_t6041_params); i++) { + const struct sio_t6041_param *p = &sio_t6041_params[i]; + bool have_addr = seen & BIT(2 * i), have_size = seen & BIT(2 * i + 1); + bool inside = false; + u64 start, end; + + if (p->kind == SIO_T6041_SCALAR) + continue; + if (have_addr != have_size) + return -EINVAL; + if (!have_addr) { + if (p->kind == SIO_T6041_REQUIRED) + return -EINVAL; + continue; + } + if (!size[i] || (p->kind == SIO_T6041_REQUIRED && size[i] != p->value)) + return -EINVAL; + start = (u64)addr[i] << 12; + if (check_add_overflow(start, (u64)size[i], &end) || end > SIO_T6030_IOVA_LIMIT) + return -EINVAL; + for (j = 0; j < nwindows && !inside; j++) { + u64 window_end; + + if (check_add_overflow(windows[j].start, windows[j].size, &window_end)) + continue; + inside = start >= windows[j].start && end <= window_end; + } + if (!inside) + return -EINVAL; + } + return 0; +} + +static int sio_t6041_read_params(struct device_node *np, u32 *params, unsigned int max) +{ + int nparams = of_property_count_u32_elems(np, "apple,sio-firmware-params"); + + if (nparams <= 0 || nparams > max || + of_property_read_u32_array(np, "apple,sio-firmware-params", params, nparams)) + return -EINVAL; + return nparams; +} + +static int sio_t6041_check_node(struct device_node *np, u32 *addr, u32 *size) +{ + struct sio_t6030_window windows[SIO_T6030_WINDOWS_MAX]; + u32 params[2 * SIO_T6030_PARAMS_MAX]; + int nparams, nwindows; + + nparams = sio_t6041_read_params(np, params, ARRAY_SIZE(params)); + if (nparams < 0) + return nparams; + nwindows = sio_t6030_windows(np, windows, ARRAY_SIZE(windows)); + if (nwindows < 0) + return nwindows; + return sio_t6041_collect(params, nparams, windows, nwindows, addr, size); +} + +static int sio_t6041_check_boot_tables(struct device *dev) +{ + u32 addr[ARRAY_SIZE(sio_t6041_params)] = {}; + u32 size[ARRAY_SIZE(sio_t6041_params)] = {}; + + if (sio_t6041_check_node(dev->of_node, addr, size)) { + dev_err(dev, "boot tables are not T6041 SIO tables; leaving the SIO stopped\n"); + return -ENODEV; + } + return 0; +} + +/* Publish the collected control tables, and the scalar, in firmware order. */ +static int sio_t6041_send_params(struct sio_data *sio, const u32 *addr, const u32 *size, + int (*set_param)(struct sio_data *, u32, u32)) +{ + unsigned int i; + int err; + + for (i = 0; i < ARRAY_SIZE(sio_t6041_params); i++) { + const struct sio_t6041_param *p = &sio_t6041_params[i]; + + if (p->kind == SIO_T6041_SCALAR) { + err = set_param(sio, p->key, p->value); + if (err) + return err; + continue; + } + if (!size[i]) + continue; /* an optional table the ADT does not have */ + err = set_param(sio, p->key, addr[i]); + if (!err) + err = set_param(sio, p->key + 1, size[i]); + if (err) + return err; + } + return 0; +} + +static int sio_t6041_send_dt_params(struct sio_data *sio) +{ + u32 addr[ARRAY_SIZE(sio_t6041_params)] = {}; + u32 size[ARRAY_SIZE(sio_t6041_params)] = {}; + int err; + + /* Checked before the boot already; the property cannot have changed. */ + err = sio_t6041_check_node(sio->dev->of_node, addr, size); + if (err) + return err; + return sio_t6041_send_params(sio, addr, size, sio_set_param); +} + +/* The DMA arrays a protocol publishes as entry counts, after its control tables. */ +struct sio_dma_arrays { + u32 segments; + u32 segment_size; + u32 responses; + u32 response_size; +}; + +static const struct sio_dma_arrays sio_v9_arrays = { + .segments = SIO_V9_SEGMENTS, + .segment_size = SIO_V9_SEGMENT_SIZE, + .responses = SIO_V9_RESPONSES, + .response_size = SIO_V9_RESPONSE_SIZE, +}; + +static const struct sio_dma_arrays sio_t6041_arrays = { + .segments = SIO_T6041_SEGMENTS, + .segment_size = SIO_T6041_SEGMENT_SIZE, + .responses = SIO_T6041_RESPONSES, + .response_size = SIO_T6041_RESPONSE_SIZE, +}; + +static_assert(56 + SIO_NO_DESC_SLOTS * sizeof(struct sio_coproc_desc) <= + SIO_T6041_SEGMENTS * SIO_T6041_SEGMENT_SIZE); + static const struct sio_hw_data sio_hw_generic = { .fifo_threshold = 0x800, + .cfg_word4 = 0x800, }; static const struct sio_hw_data sio_hw_t6030 = { .fifo_threshold = 0x600, + .cfg_word4 = 0x800, .clear_period_desc = true, .check_boot_tables = sio_t6030_check_boot_tables, }; +static const struct sio_hw_data sio_hw_t6041 = { + .protocol = 10, + .fifo_threshold = 0x600, + .cfg_word4 = 5, /* the host's default timeout */ + .cfg_six_words = true, + .clear_period_desc = true, + .arrays = &sio_t6041_arrays, + .check_boot_tables = sio_t6041_check_boot_tables, + .send_dt_params = sio_t6041_send_dt_params, +}; + static const struct of_device_id sio_of_match[] = { + { .compatible = "apple,t6041-sio", .data = &sio_hw_t6041 }, { .compatible = "apple,t6030-sio", .data = &sio_hw_t6030 }, { .compatible = "apple,sio", .data = &sio_hw_generic }, { } @@ -1323,6 +1583,16 @@ static const struct apple_rtkit_ops sio_rtkit_ops = { .recv_message = sio_recv_msg, }; +/* The configuration record CONFIGURE points the firmware at (handle 0) */ +static void sio_write_config(const struct sio_hw_data *hw, void *shmem, + const struct sio_shmem_chan_config *cfg) +{ + memcpy(shmem, cfg, sizeof(*cfg)); + /* T6041 takes a sixth configuration word, zero. */ + if (hw->cfg_six_words) + memset(shmem + sizeof(*cfg), 0, sizeof(u32)); +} + static int sio_device_config(struct dma_chan *chan, struct dma_slave_config *config) { @@ -1355,7 +1625,7 @@ static int sio_device_config(struct dma_chan *chan, cfg.timeout = 0; cfg.fifo = 0x800; - cfg.limit = 0x800; + cfg.limit = sio->hw->cfg_word4; /* * The ADT records each client's channel configuration with it (the * dp-audioN dma-channels: channel, data shape, timeout, FIFO size, @@ -1390,7 +1660,7 @@ static int sio_device_config(struct dma_chan *chan, if (siochan->configured && memcmp(&siochan->cfg, &cfg, sizeof(cfg))) return -EBUSY; - *cfg_shmem = cfg; + sio_write_config(sio->hw, cfg_shmem, &cfg); dma_wmb(); ret = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_CONFIGURE) | @@ -1497,9 +1767,9 @@ static void sio_free_v9_shmem(void *data) dev_err(sio->dev, "SIO shutdown failed: %d; retaining DMA buffers\n", err); return; } - dma_free_coherent(sio->dev, SIO_V9_RESPONSES * SIO_V9_RESPONSE_SIZE, + dma_free_coherent(sio->dev, sio->arrays->responses * sio->arrays->response_size, sio->responses, sio->responses_iova); - dma_free_coherent(sio->dev, SIO_V9_SEGMENTS * SIO_V9_SEGMENT_SIZE, + dma_free_coherent(sio->dev, sio->arrays->segments * sio->arrays->segment_size, sio->shmem, sio->shmem_iova); } @@ -1511,9 +1781,9 @@ static int sio_send_v9_shmem(struct sio_data *sio, u32 value; } params[] = { { 1, sio->shmem_iova >> 12 }, - { 2, SIO_V9_SEGMENTS }, + { 2, (sio->arrays ?: &sio_v9_arrays)->segments }, { 0x0b, sio->responses_iova >> 12 }, - { 0x0c, SIO_V9_RESPONSES }, + { 0x0c, (sio->arrays ?: &sio_v9_arrays)->responses }, }; unsigned int i; int err; @@ -1528,10 +1798,21 @@ static int sio_send_v9_shmem(struct sio_data *sio, static int sio_alloc_shmem(struct sio_data *sio) { - bool v9 = sio->protocol == 9; - size_t size = v9 ? SIO_V9_SEGMENTS * SIO_V9_SEGMENT_SIZE : SIO_SHMEM_SIZE; + const struct sio_dma_arrays *arrays = sio->hw->arrays; + size_t size, responses_size = 0; int err; + if (!arrays && sio->protocol == 9) + arrays = &sio_v9_arrays; + sio->arrays = arrays; + if (arrays && + (check_mul_overflow((size_t)arrays->segments, (size_t)arrays->segment_size, &size) || + check_mul_overflow((size_t)arrays->responses, (size_t)arrays->response_size, + &responses_size))) + return -EOVERFLOW; + if (!arrays) + size = SIO_SHMEM_SIZE; + sio->shmem = dma_alloc_coherent(sio->dev, size, &sio->shmem_iova, GFP_KERNEL | __GFP_ZERO); if (!sio->shmem) @@ -1545,9 +1826,8 @@ static int sio_alloc_shmem(struct sio_data *sio) goto free_shmem; } - if (v9) { - sio->responses = dma_alloc_coherent(sio->dev, - SIO_V9_RESPONSES * SIO_V9_RESPONSE_SIZE, + if (arrays) { + sio->responses = dma_alloc_coherent(sio->dev, responses_size, &sio->responses_iova, GFP_KERNEL | __GFP_ZERO); if (!sio->responses) { err = -ENOMEM; @@ -1577,6 +1857,9 @@ static int sio_send_dt_params(struct sio_data *sio) const char *propname = "apple,sio-firmware-params"; int nparams, err, i; + if (sio->hw->send_dt_params) + return sio->hw->send_dt_params(sio); + nparams = of_property_count_u32_elems(np, propname); if (nparams < 0) { err = nparams; @@ -1719,6 +2002,10 @@ static int sio_probe(struct platform_device *pdev) err = -ENXIO; return dev_err_probe(&pdev->dev, err, "starting SIO service\n"); } + if (hw->protocol && sio->protocol != hw->protocol) + return dev_err_probe(&pdev->dev, -ENODEV, + "SIO firmware protocol %u, this SoC's is %u\n", + sio->protocol, hw->protocol); err = sio_send_dt_params(sio); if (err < 0) @@ -1786,6 +2073,10 @@ MODULE_LICENSE("Dual MIT/GPL"); #include "apple-sio-test.c" #endif +#if IS_ENABLED(CONFIG_APPLE_SIO_T6041_KUNIT_TEST) +#include "apple-sio-t6041-test.c" +#endif + #if IS_ENABLED(CONFIG_APPLE_SIO_KUNIT_TEST) #include "apple-sio-probe-test.c" #include "apple-sio-boot-test.c" From 733268011b9f9e3abda84b2ac219d2515efebbbd Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:17:41 +1100 Subject: [PATCH 105/392] drm/apple: start desktop external audio on the T6041 DCPEXTs The desktop 26.x audio binding (endpoint 0x2b) could not start on any Mac: avep_init(), its admission and the native-link check all required dcp->external, which is set only for apple,t6030-dcpext. The M4 Pro/Max external processors are apple,t6041-dcpext, so appledrm.hdmi_audio=1 on the J616s logged "endpoint 0x2b (AUDIO) not started: audio init failed (-95)" for both. Record the desktop external role in its own flag and use it on those paths. The audio interface those processors publish is dcpav-audio-interface-epic, unit 0, not the provisional dcpav-audio-epic. With its role in front ("dispext0:") the compact announcement is longer than the 32-byte field and arrives cut to 31 characters, without its unit. Let an ops entry name its full ":" (compact_full) and match a cut of it of at least 20 characters, only when the announcement filled its field; the admission takes such a cut as unit 0 and nothing else. The binding stays behind appledrm.hdmi_audio (off by default). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/afk.c | 24 ++++++++++++++++++++++-- drivers/gpu/drm/apple/afk.h | 15 +++++++++++++++ drivers/gpu/drm/apple/av-v14-test.c | 25 ++++++++++++++++++++++++- drivers/gpu/drm/apple/av.c | 26 +++++++++++++++++++------- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 1 + 6 files changed, 83 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c988e566c5344..9ac50fb8b6576 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -778,9 +778,29 @@ afk_compact_match_service(struct apple_dcp_afkep *ep, const char *name) return ops; } + /* An announcement that filled its field may have lost its end. */ + if (strlen(name) < AFK_COMPACT_NAME_LEN - 1) + return NULL; + for (ops = ep->ops; ops->name[0]; ops++) + if (afk_compact_cut_of(ops, service_name)) + return ops; + return NULL; } +/* + * Whether @service_name, the part after the role of an announcement that + * filled its field, is the firmware's cut of @ops's long name. + */ +bool afk_compact_cut_of(const struct apple_epic_service_ops *ops, const char *service_name) +{ + size_t len = strlen(service_name); + + return ops->compact_full && len >= AFK_COMPACT_CUT_MIN && + len < strlen(ops->compact_full) && + !strncmp(ops->compact_full, service_name, len); +} + /* A free slot for a newly announced service, or the one it replaces. */ static struct apple_epic_service * afk_compact_slot(struct apple_dcp_afkep *ep, @@ -867,8 +887,8 @@ static void afk_recv_handle_compact_init(struct apple_dcp_afkep *ep, ep->endpoint, channel); return; } - memcpy(wire_name, payload, 32); - wire_name[32] = '\0'; + memcpy(wire_name, payload, AFK_COMPACT_NAME_LEN); + wire_name[AFK_COMPACT_NAME_LEN] = '\0'; name = strchr(wire_name, ':'); name = name ? name + 1 : wire_name; strscpy(service_name, name, sizeof(service_name)); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d9fb811252f87..1e0a3fae1138c 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -89,8 +89,20 @@ struct apple_epic_service { enum epic_subtype; +/* A compact service announcement's name field, with its NUL */ +#define AFK_COMPACT_NAME_LEN 32 +/* The shortest cut of a long compact service name that still identifies it */ +#define AFK_COMPACT_CUT_MIN 20 + struct apple_epic_service_ops { const char name[32]; + /* + * A compact service whose announcement does not fit the 32-byte field: + * its full ":". The firmware cuts the announcement to 31 + * characters, losing the end of the name and the unit; a cut of this + * string, at least AFK_COMPACT_CUT_MIN characters of it, matches. + */ + const char *compact_full; /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; /* EOPNOTSUPP from prepare means intentional absence only for this role. */ @@ -117,6 +129,9 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; +bool afk_compact_cut_of(const struct apple_epic_service_ops *ops, const char *service_name); + + static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) { u32 header, block; diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index 433530763af7f..4718b61c27c00 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -128,6 +128,7 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) DCP_FIRMWARE_V_26_6_DESKTOP : DCP_FIRMWARE_V_14_7; t->dcp->external = true; t->dcp->external_native = true; + t->dcp->desktop_external = true; init_rwsem(&asrv->srv_rwsem); mutex_init(&asrv->plug_lock); @@ -429,8 +430,9 @@ static void av_native_link_payload_status(struct kunit *test) static void av_desktop_service_admission(struct kunit *test) { struct av_v14_test *t = av_v14_setup(test); - const char *name = "dcpav-audio-epic"; + const char *name = "dcpav-audio-interface-epic"; bool desktop = t->dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP; + const struct apple_epic_service_ops *ops; KUNIT_EXPECT_TRUE(test, dcp_audiosrv_native_link(t->dcp->dev)); KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), @@ -443,6 +445,27 @@ static void av_desktop_service_admission(struct kunit *test) -EINVAL); KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, NULL, name, 0), -EINVAL); KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 1), -EINVAL); + /* The provisional pre-capture name is no longer admitted. */ + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-audio-epic", + "dcpav-audio-epic", 0), -EINVAL); + /* "dispext0:dcpav-audio-interface-epic:0" cut to its 31-character field */ + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-audio-interface-", + "dcpav-audio-interface-", -1), 0); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-audio-interface-", + "dcpav-audio-interface-", 1), -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-audio-i", + "dcpav-audio-i", -1), -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-audio-interface-epix", + "dcpav-audio-interface-epix", -1), -EINVAL); + ops = &av_desktop_ops[0]; + KUNIT_EXPECT_TRUE(test, afk_compact_cut_of(ops, "dcpav-audio-interface-")); + KUNIT_EXPECT_TRUE(test, afk_compact_cut_of(ops, "dcpav-audio-interface-epic:")); + KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(ops, "dcpav-audio-interface-epic:0")); + KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(ops, "dcpav-audio-epic")); + KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(&avep_ops[0], "DCPAVSimpleVideoInter")); + /* Only the desktop external processor's own flag admits it. */ + t->dcp->desktop_external = false; + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); t->dcp->external = false; KUNIT_EXPECT_FALSE(test, dcp_audiosrv_native_link(t->dcp->dev)); KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 5e11c704a86ef..567b03156d163 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -260,7 +260,7 @@ bool dcp_audiosrv_native_link(struct device *dev) struct apple_dcp *dcp = dev_get_drvdata(dev); return dcp_is_v14_7_external(dcp) || - (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->external); + (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->desktop_external); } /* The 14.7 external role's audio service calls; KUnit stubs them. */ @@ -664,21 +664,33 @@ static void av_cec_teardown(struct apple_epic_service *service) #endif #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +static const struct apple_epic_service_ops av_desktop_ops[]; + +/* + * The desktop 26.x external processors (T6041 DCPEXT) publish their audio + * interface on endpoint 0x2b as dcpav-audio-interface-epic, unit 0. With + * the role in front ("dispext0:") the announcement is longer than its + * field, so it arrives cut, without the unit: admit the cut too, as unit 0. + */ static int av_desktop_audio_prepare(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - if (dcp->fw_compat != DCP_FIRMWARE_V_26_6_DESKTOP || !dcp->external || - service->ep->endpoint != AV_AUDIO_ENDPOINT_DESKTOP || - !name || strcmp(name, "dcpav-audio-epic") || unit != 0) + if (dcp->fw_compat != DCP_FIRMWARE_V_26_6_DESKTOP || !dcp->desktop_external || + service->ep->endpoint != AV_AUDIO_ENDPOINT_DESKTOP || !name) return -EINVAL; - return 0; + if (!strcmp(name, av_desktop_ops[0].name) && unit == 0) + return 0; + if (unit == -1 && afk_compact_cut_of(&av_desktop_ops[0], name)) + return 0; + return -EINVAL; } static const struct apple_epic_service_ops av_desktop_ops[] = { { - .name = "dcpav-audio-epic", + .name = "dcpav-audio-interface-epic", + .compact_full = "dcpav-audio-interface-epic:0", .reusable = true, .prepare = av_desktop_audio_prepare, .init = av_audiosrv_init, @@ -1032,7 +1044,7 @@ int avep_init(struct apple_dcp *dcp) if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) - if (!dcp->external) + if (!dcp->desktop_external) return -EOPNOTSUPP; ops = av_desktop_ops; av_endpoint = AV_AUDIO_ENDPOINT_DESKTOP; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5b9f50b60d300..e7d3d9891aa4f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -447,6 +447,8 @@ struct apple_dcp { /* Bumped by every iomfb poweron; see afk_compact_slot(). */ u32 afk_power_gen; bool external; + /* A desktop 26.x external processor (apple,t6041-dcpext) */ + bool desktop_external; /* * T8122 dcpext: the panel's DRM device never waits on this processor. * A probe or bind failure leaves it in the device as an unusable pipe diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f56c4423ea247..a724f318a778e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5888,6 +5888,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext") || of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); + dcp->desktop_external = of_device_is_compatible(dev->of_node, "apple,t6041-dcpext"); dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); dcp_external_audio_init(dcp); dcp->dev = dev; From ec8e1fa90f9994d6f9f40c38b9716deedb10a1f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:17:41 +1100 Subject: [PATCH 106/392] arm64: dts: apple: t6040-j616s: enable the DPA1 audio chain Enable what the first DisplayPort/HDMI audio stream needs: the SIO CPU power domain, the SIO DART and mailbox, the DPA power domains and the DPA1 client of DCPEXT0 (SIO channel 0x66). The SIO node itself stays disabled: the boot loader enables it once it has reserved the firmware iBoot preloaded and published the T6041 boot tables, and finds it through the new "sio" alias. Without such a loader DPA1 waits for its channel. DPA2 (DCPEXT1, channel 0x68) stays disabled until it is qualified. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../boot/dts/apple/t6040-j616s-dpaudio.dtsi | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi index 8acd2d04aea8d..a43d5df5683c3 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -1,18 +1,28 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT /* * DPA1 and DPA2 belong to DCPEXT0 and DCPEXT1, respectively. SIO supplies - * their sample streams; neither interface has AP-side registers. Keep the - * clients disabled until the T6041 SIO firmware backend and loader handoff - * are available and the compact audio protocol has been qualified. + * their sample streams; neither interface has AP-side registers. + * + * The first chain is enabled: the SIO DART, mailbox and CPU domain, DPA1 + * and its client. The SIO itself stays disabled here: iBoot has loaded its + * firmware, and the boot loader enables the node once it has reserved that + * firmware and published the T6041 boot tables (it finds it by the alias). + * Without a T6041-aware loader DPA1 waits for its DMA channel and nothing + * else changes. The DCP side starts only with appledrm.hdmi_audio=1. + * DPA2 (DCPEXT1, SIO channel 0x68) stays disabled until it is qualified. */ / { + aliases { + sio = &sio; + }; + dpaudio1: audio-controller-1 { compatible = "apple,t6041-dpaudio", "apple,dpaudio"; dmas = <&sio 0x66>; dma-names = "tx"; power-domains = <&ps_dpa1>; - status = "disabled"; + status = "okay"; ports { #address-cells = <1>; @@ -75,3 +85,23 @@ }; }; }; + +&ps_sio_cpu { + status = "okay"; +}; + +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + +&ps_dpa_p { + status = "okay"; +}; + +&ps_dpa1 { + status = "okay"; +}; From dd39d21e83e6fadccf6e97ee298e5fc0ba1c4051 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:35:39 +1100 Subject: [PATCH 107/392] Documentation: gpu: apple-dcp: T6041 desktop audio service and DPA1 chain Name the published desktop audio service, describe how its cut compact announcement is admitted, and say which part of the J616s DPA chain the device tree enables and what the boot loader enables. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- Documentation/gpu/apple-dcp-t6041-panel.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6041-panel.rst b/Documentation/gpu/apple-dcp-t6041-panel.rst index b75e6584bc1d2..d8834d7e1a877 100644 --- a/Documentation/gpu/apple-dcp-t6041-panel.rst +++ b/Documentation/gpu/apple-dcp-t6041-panel.rst @@ -190,9 +190,13 @@ Desktop external audio (draft) The desktop audio integration uses the existing DCP PCM and native link lifecycle, with a separate compact service binding on endpoint 0x2b. Endpoint -0x29 retains AV device control. Only the external firmware role admits the -``dcpav-audio-epic`` unit-zero service; the internal panel does not acquire an -audio client. The existing ``hdmi_audio`` opt-in still controls service startup. +0x29 retains AV device control. Only the external processors +(``apple,t6041-dcpext``) admit the ``dcpav-audio-interface-epic`` unit-zero +service; the internal panel does not acquire an audio client. With its role +prefix the compact announcement does not fit its 32-byte field and arrives cut +to 31 characters without the unit; such a cut of at least 20 characters is +admitted as unit zero. The existing ``hdmi_audio`` opt-in still controls +service startup. A PCM opens and pins one service instance. Prepare, Start, Stop and Complete use that instance's element link data. Sink termination disconnects the card, @@ -202,10 +206,12 @@ method replies must have the expected body length; serialized property replies must contain their declared used bytes within the actual received body. J616s DPA1 and DPA2 are described with SIO channels 0x66 and 0x68, connected -bidirectionally to DCPEXT0 and DCPEXT1 port zero, respectively. Both clients -remain disabled. This description does not enable SIO, its DART, its mailbox -or any DPA power domain. A generation-specific SIO firmware backend and loader -handoff are still required before a candidate may enable the clients. +bidirectionally to DCPEXT0 and DCPEXT1 port zero, respectively. DPA1 is +enabled with the SIO DART, mailbox, CPU domain and the DPA power domains. The +SIO node stays disabled in the device tree: the boot loader reserves the +firmware iBoot preloaded, publishes the T6041 boot tables and enables the node +(``apple,t6041-sio``, see the apple-sio driver). DPA2 stays disabled until it +is qualified the same way. The desktop service name/unit, shared 13.5 command numbers and native 14.7 link and element layouts are provisional integration choices. Build and synthetic From db2036536223b3f446a9fada2820782ef95e77b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:44:58 +1100 Subject: [PATCH 108/392] drm/apple: take the T6041 DCPEXT CEC interface on endpoint 0x2b Endpoint 0x2b of the M4 Pro/Max external processors also publishes their CEC interface, which arrives cut to "dispext0:dcpav-cec-interface-ep". Unmatched, every boot with appledrm.hdmi_audio=1 logged "unable to match compact service". The host makes no use of CEC: take it with the same no-op service the T6030 HDMI port uses and drop its reports. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/av-v14-test.c | 3 +++ drivers/gpu/drm/apple/av.c | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index 4718b61c27c00..f0dbe8f079dd0 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -463,6 +463,9 @@ static void av_desktop_service_admission(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(ops, "dcpav-audio-interface-epic:0")); KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(ops, "dcpav-audio-epic")); KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(&avep_ops[0], "DCPAVSimpleVideoInter")); + /* The CEC interface the same endpoint publishes, as it arrives on the J616s */ + KUNIT_EXPECT_TRUE(test, afk_compact_cut_of(&av_desktop_ops[1], "dcpav-cec-interface-ep")); + KUNIT_EXPECT_FALSE(test, afk_compact_cut_of(&av_desktop_ops[0], "dcpav-cec-interface-ep")); /* Only the desktop external processor's own flag admits it. */ t->dcp->desktop_external = false; KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 567b03156d163..7b1e308890c02 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -697,6 +697,18 @@ static const struct apple_epic_service_ops av_desktop_ops[] = { .report = av_audiosrv_report, .teardown = av_audiosrv_teardown, }, + { + /* + * The same endpoint also publishes the processor's CEC + * interface, cut to "dcpav-cec-interface-ep" as above. The + * host makes no use of CEC: take it and drop its reports. + */ + .name = "dcpav-cec-interface-epic", + .compact_full = "dcpav-cec-interface-epic:0", + .init = av_cec_init, + .report = av_cec_report, + .teardown = av_cec_teardown, + }, {} }; #endif From 64ad8cbd2cea11af17c0175060a38d7b32ccd76a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:12:50 +1100 Subject: [PATCH 109/392] wifi: brcmfmac: fail a join whose BSS is gone instead of reporting success On the J616s (BCM4388), a reconnect that is still in flight when the system enters sleep logs, in 3 of 6 fleet suites: brcmf_inform_single_bss: Invalid Bss info record. Discarding WARNING: net/wireless/sme.c:845 at __cfg80211_connect_result WARNING: net/wireless/sme.c:1302 at __cfg80211_port_authorized cfg80211's suspend disconnects the interface (reason 3, locally generated) while the firmware completes the association. The link-up event is then processed against a torn-down join: GET_BSS_INFO returns no valid record, brcmf_update_bss_info()'s error is ignored, and the driver reports WLAN_STATUS_SUCCESS for a BSS that cfg80211 does not know. cfg80211 rejects that with the first WARN. For firmware-supplicant (PSK/SAE) joins it then gets a port-authorized event for a connection it never made, which is the second WARN. When the firmware cannot describe the BSS and cfg80211 has no entry for it either, report the join as failed and disassociate the firmware, so the two sides agree. The supplicant retries as for any failed join. A BSS that cfg80211 already knows (from a scan) is still reported as connected, as before. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index b415aaeb73479..75e2a3dd16059 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6846,6 +6846,21 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, return err; } +/* Whether cfg80211 has a BSS entry for @bssid (the firmware's may be gone). */ +static bool brcmf_bss_known(struct brcmf_cfg80211_info *cfg, const u8 *bssid) +{ + struct cfg80211_bss *bss; + + if (is_zero_ether_addr(bssid)) + return false; + bss = cfg80211_get_bss(cfg_to_wiphy(cfg), NULL, bssid, NULL, 0, + IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ANY); + if (!bss) + return false; + cfg80211_put_bss(cfg_to_wiphy(cfg), bss); + return true; +} + static s32 brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, const struct brcmf_event_msg *e, @@ -6864,7 +6879,25 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, memset(&conn_params, 0, sizeof(conn_params)); if (completed) { brcmf_get_assoc_ies(cfg, ifp); - brcmf_update_bss_info(cfg, ifp); + if (brcmf_update_bss_info(cfg, ifp) && + !brcmf_bss_known(cfg, profile->bssid)) { + /* + * A link-up that raced the join's teardown (a + * disconnect for system sleep, say) leaves the + * firmware with no BSS to report, and cfg80211 + * has none either: it would reject the success + * with a WARN in __cfg80211_connect_result and + * another in __cfg80211_port_authorized. Report + * the join as failed and drop the firmware's + * half-made association instead. + */ + brcmf_dbg(CONN, "connected BSS %pM unknown, join failed\n", + profile->bssid); + brcmf_fil_cmd_data_set(ifp, BRCMF_C_DISASSOC, NULL, 0); + completed = false; + } + } + if (completed) { set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); conn_params.status = WLAN_STATUS_SUCCESS; From 691e2fbdbe8e9bd661f3c512491f650ceedd3004 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:30:30 +1100 Subject: [PATCH 110/392] tools/asahi: test a join whose BSS was lost before its link-up brcmf-join-bssid-tests.py stubs brcmf_update_bss_info() with an error return now, plus brcmf_bss_known() and the DISASSOC command. Two new controls cover it. With no BSS from the firmware or cfg80211, every fwsup mode reports a failed join, is never authorized, and disassociates once. When the firmware lost the record but cfg80211 knows the BSS, the join stays a success. The new 'lost-bss-success' mutation, which keeps reporting success, must fail the controls. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- tools/asahi/brcmf-join-bssid-tests.py | 30 ++++++++++++++++++++++++++- tools/asahi/run-host-tests.py | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tools/asahi/brcmf-join-bssid-tests.py b/tools/asahi/brcmf-join-bssid-tests.py index ae29767becdc3..770b0af9764f7 100755 --- a/tools/asahi/brcmf-join-bssid-tests.py +++ b/tools/asahi/brcmf-join-bssid-tests.py @@ -24,6 +24,7 @@ '\t\t\tconn_params.links[0].bssid = profile->bssid;\n', '\t\tconn_params.links[0].bssid = profile->bssid;\n'), 'no-call': ('\tbrcmf_connect_set_target(profile, params->bssid);\n', ''), + 'lost-bss-success': ('\t\t\t\tcompleted = false;\n', '\t\t\t\t;\n'), } PREFIX = r''' @@ -67,7 +68,14 @@ static u8 reported_bssid[ETH_ALEN]; static bool reported_none; static void brcmf_get_assoc_ies(struct brcmf_cfg80211_info *c, struct brcmf_if *i) { (void)c; (void)i; } -static void brcmf_update_bss_info(struct brcmf_cfg80211_info *c, struct brcmf_if *i) { (void)c; (void)i; } +#define BRCMF_C_DISASSOC 52 +static s32 bss_info_err; +static bool bss_known = true; +static int disassocs; +static s32 brcmf_update_bss_info(struct brcmf_cfg80211_info *c, struct brcmf_if *i) { (void)c; (void)i; return bss_info_err; } +static bool brcmf_bss_known(struct brcmf_cfg80211_info *c, const u8 *b) { (void)c; (void)b; return bss_known; } +static s32 brcmf_fil_cmd_data_set(struct brcmf_if *i, u32 cmd, void *d, u32 l) +{ (void)i; (void)d; (void)l; if (cmd == BRCMF_C_DISASSOC) disassocs++; return 0; } static void cfg80211_connect_done(struct net_device *n, struct cfg80211_connect_resp_params *p, int gfp) { (void)n; (void)gfp; reports++; reported_status = p->status; @@ -141,6 +149,26 @@ assert(reports == 1 && !memcmp(reported_bssid, TRY24, ETH_ALEN) && authorized == 1); controls++; + /* A link-up after the join was torn down: no BSS from the firmware or cfg80211. */ + for (int fwsup = 0; fwsup <= BRCMF_PROFILE_FWSUP_SAE; fwsup++) { + start(TRY24, fwsup); + bss_info_err = -22; bss_known = false; disassocs = 0; + brcmf_bss_connect_done(&cfg, &ndev, &ssid, true); + assert(reports == 1 && reported_status == WLAN_STATUS_AUTH_TIMEOUT); + assert(!authorized && !test_bit(BRCMF_VIF_STATUS_CONNECTED, &vif.sme_state)); + assert(disassocs == 1); + controls++; + } + + /* The firmware lost the record but cfg80211 knows the BSS: still connected. */ + start(TRY24, BRCMF_PROFILE_FWSUP_PSK); + bss_info_err = -22; bss_known = true; disassocs = 0; + brcmf_bss_connect_done(&cfg, &ndev, &psk, true); + assert(reports == 1 && reported_status == WLAN_STATUS_SUCCESS && authorized == 1); + assert(test_bit(BRCMF_VIF_STATUS_CONNECTED, &vif.sme_state) && !disassocs); + bss_info_err = 0; + controls++; + /* Only one report per join. */ brcmf_bss_connect_done(&cfg, &ndev, NULL, false); assert(reports == 1); diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 84ced6b0f27d5..e472c9d74cc10 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -87,7 +87,7 @@ def mutations(flag, names, base, expect=KILL): 'brcmf-bss-tests.py': [(out(), PASS)], # The old revision reports a failed join against the last associated BSS. 'brcmf-join-bssid-tests.py': [(out(), PASS)] - + mutations('--mutation', ('stale-target', 'zero-named', 'no-call'), out()), + + mutations('--mutation', ('stale-target', 'zero-named', 'no-call', 'lost-bss-success'), out()), 'brcmf-pcie-dma-tests.py': [(out('--output'), PASS)] + mutations('--mutation', ('narrow-stream', 'ignore-aperture', 'revision-wide'), out('--output')), 'brcmf-roam-tests.py': [(out(), PASS)], From 85193a07c0d71d34a175971ef6d22df30abd17bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 09:32:16 +1100 Subject: [PATCH 111/392] ASoC: apple: mca: Bound the PCM buffer size again Commit 79af0bae4d0d ("ASoC: apple: mca: Make the capture buffers cacheable") stopped preallocating the capture buffers, so they are now allocated at hw_params to the buffer's own size. The 3 MiB preallocation was also what bounded the capture buffer: the core sets a substream's buffer_bytes_max to its preallocated size, and the runtime hardware leaves it at SIZE_MAX. Without a preallocation the sense PCM offers buffers of up to 4 GiB. speakersafetyd sets no buffer size on the speaker sense PCM, so alsa-lib picks the largest buffer and the NONCOHERENT allocation fails: snd_pcm_hw_params failed with error 'Out of memory (12)' speakersafetyd then panics and restarts in a loop. It never takes the speaker volume lock, and the speakers stay at the kernel's floor, with no sound on every MCA Mac it protects (seen on a J316 on 7.3.0-26.fc45.aurora, the kernel with the legacy unlock interface). Bound the PCM buffer at 3 MiB, the playback preallocation's size and the limit capture had before 79af0bae4d0d. Clients that size their own buffer (speakerguardd, PipeWire) ask for far less and are unaffected. Fixes: 79af0bae4d0d ("ASoC: apple: mca: Make the capture buffers cacheable") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/mca.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 69abfcda7e813..18d73b50af861 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -127,6 +127,9 @@ SNDRV_PCM_FMTBIT_S24_LE | \ SNDRV_PCM_FMTBIT_S32_LE) +/* Largest PCM buffer, and the playback preallocation: 3 MiB. */ +#define MCA_BUFFER_BYTES_MAX (512 * 1024 * 6) + struct mca_cluster { int no; __iomem void *base; @@ -962,7 +965,15 @@ static int mca_set_runtime_hwparams(struct snd_soc_component *component, hw.periods_max = UINT_MAX; hw.period_bytes_min = 256; hw.period_bytes_max = dma_get_max_seg_size(dma_dev); - hw.buffer_bytes_max = SIZE_MAX; + /* + * The capture buffer is not preallocated (see mca_pcm_new()), so the + * core does not bound it by a preallocation: bound it here, at the size + * the playback preallocation (and formerly the capture one) allows. An + * unbounded maximum lets a client that sets no buffer size, such as + * speakersafetyd on the speaker sense PCM, ask for gigabytes and fail + * hw_params with -ENOMEM. + */ + hw.buffer_bytes_max = MCA_BUFFER_BYTES_MAX; hw.fifo_size = 16; ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data, @@ -1245,7 +1256,7 @@ static int mca_pcm_new(struct snd_soc_component *component, chan->device->dev, 0, SIZE_MAX); else snd_pcm_set_managed_buffer(substream, SNDRV_DMA_TYPE_DEV_IRAM, - chan->device->dev, 512 * 1024 * 6, + chan->device->dev, MCA_BUFFER_BYTES_MAX, SIZE_MAX); } From f8c7547ef29eae71436808226b86601f7d112f80 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:47:08 +1100 Subject: [PATCH 112/392] arm64: dts: apple: j616s: enable USB4/Thunderbolt on the left front and right ports Enable ACIO1 (typec1, left front) and ACIO2 (typec2, right) the same way as ACIO0: router, its DART with the ACIO0 DVA window, NHI, mailbox and power domains, PCIe tunnel off. The routers' resets, LSTX pins, ring interrupts and Type-C graph links are already described. The loader hands every router its tuning and DROM. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 77 ++++++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 0e29b6291ed9f..0d7c60454d480 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -167,9 +167,10 @@ }; /* - * USB4/Thunderbolt on the left back port (typec0, ACIO0). The loader hands - * the router, adapter and NHI tuning and the DROM over. No PCIe tunnel yet: - * host-to-host (XDomain, thunderbolt-net) and USB4 links only. + * USB4/Thunderbolt on all three Type-C ports: typec0 (left back, ACIO0), + * typec1 (left front, ACIO1) and typec2 (right, ACIO2). The loader hands + * each router its router, adapter and NHI tuning and its DROM over. No PCIe + * tunnels yet: host-to-host (XDomain, thunderbolt-net) and USB4 links only. */ &pmgr_cio { status = "okay"; @@ -209,3 +210,73 @@ &usb4_0_nhi { status = "okay"; }; + +&ps_atc1_cio { + status = "okay"; +}; + +&ps_atc1_pcie { + status = "okay"; +}; + +&ps_atc1_cio_pcie { + status = "okay"; +}; + +&ps_atc1_cio_usb { + status = "okay"; +}; + +&usb4_1_mbox { + status = "okay"; +}; + +&usb4_1_acio { + /delete-property/ apple,pcie-tunnel; + status = "okay"; +}; + +&usb4_1_dart { + /* The same DVA window as the ACIO0 router DART. */ + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; + status = "okay"; +}; + +&usb4_1_nhi { + status = "okay"; +}; + +&ps_atc2_cio { + status = "okay"; +}; + +&ps_atc2_pcie { + status = "okay"; +}; + +&ps_atc2_cio_pcie { + status = "okay"; +}; + +&ps_atc2_cio_usb { + status = "okay"; +}; + +&usb4_2_mbox { + status = "okay"; +}; + +&usb4_2_acio { + /delete-property/ apple,pcie-tunnel; + status = "okay"; +}; + +&usb4_2_dart { + /* The same DVA window as the ACIO0 router DART. */ + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; + status = "okay"; +}; + +&usb4_2_nhi { + status = "okay"; +}; From 2e23899af9db961603b0dece672d22d2cfcfa1ba Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:00 +1100 Subject: [PATCH 113/392] drm/apple: keep display sound cards after the built-in ones The DCPs probe before the built-in audio, so a display audio card took the first free slot and moved the speakers, the headphone jack and the microphones up by one (on the J616s macaudio went from card 1 to 2). Anything that names the built-in cards by number then picks the wrong one. Register display cards from slot 8 up. The built-in cards keep the numbers they have without display audio, whatever the probe order. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/audio.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 44c228deb2a89..48539904efa91 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -1323,6 +1323,17 @@ static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *nam extern bool hdmi_audio; +/* + * The DCPs probe before the built-in audio (speakers, headphone jack, + * microphones), and a display card in the first free slot would move + * those. Display cards take the slots from DCPAUD_FIRST_SLOT up, so the + * built-in cards keep their numbers whether or not display audio is on. + * A negative index is a mask of acceptable slots, see snd_card_new(). + */ +#define DCPAUD_FIRST_SLOT 8 +#define DCPAUD_CARD_SLOTS (SNDRV_CARDS > DCPAUD_FIRST_SLOT ? \ + (int)~(BIT(DCPAUD_FIRST_SLOT) - 1) : SNDRV_DEFAULT_IDX1) + static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) { int ret; @@ -1330,7 +1341,7 @@ static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) return -ENODEV; - ret = snd_card_new(dcpaud->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + ret = snd_card_new(dcpaud->dev, DCPAUD_CARD_SLOTS, SNDRV_DEFAULT_STR1, THIS_MODULE, 0, &dcpaud->card); if (ret) return ret; From 333ea91c896b566bfa686e1dd3d5fc62c14ea4d4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:01 +1100 Subject: [PATCH 114/392] drm/apple: turn DP/HDMI audio on by default On macOS a display that takes audio is an output as soon as it is connected, over HDMI and over DisplayPort, USB-C alt mode and docks included. Here it took appledrm.hdmi_audio=1, so nobody had it. Make hdmi_audio default to on. Every processor whose port@0 links an audio device starts its audio endpoint and registers its card; one without stays as it is. hdmi_audio=0 still turns it all off. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- Documentation/gpu/apple-dcp-t6041-panel.rst | 2 +- arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi | 2 +- drivers/gpu/drm/apple/av.c | 2 +- drivers/gpu/drm/apple/av.h | 2 +- drivers/gpu/drm/apple/dcp.c | 12 +++++++++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Documentation/gpu/apple-dcp-t6041-panel.rst b/Documentation/gpu/apple-dcp-t6041-panel.rst index d8834d7e1a877..c131fb02a5683 100644 --- a/Documentation/gpu/apple-dcp-t6041-panel.rst +++ b/Documentation/gpu/apple-dcp-t6041-panel.rst @@ -195,7 +195,7 @@ lifecycle, with a separate compact service binding on endpoint 0x2b. Endpoint service; the internal panel does not acquire an audio client. With its role prefix the compact announcement does not fit its 32-byte field and arrives cut to 31 characters without the unit; such a cut of at least 20 characters is -admitted as unit zero. The existing ``hdmi_audio`` opt-in still controls +admitted as unit zero. ``hdmi_audio`` (on by default; 0 turns it off) controls service startup. A PCM opens and pins one service instance. Prepare, Start, Stop and Complete diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi index a43d5df5683c3..1f6e862285bc2 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -8,7 +8,7 @@ * firmware, and the boot loader enables the node once it has reserved that * firmware and published the T6041 boot tables (it finds it by the alias). * Without a T6041-aware loader DPA1 waits for its DMA channel and nothing - * else changes. The DCP side starts only with appledrm.hdmi_audio=1. + * else changes. The DCP side starts unless appledrm.hdmi_audio=0. * DPA2 (DCPEXT1, SIO channel 0x68) stays disabled until it is qualified. */ diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 7b1e308890c02..23d963992cbdd 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -1069,7 +1069,7 @@ int avep_init(struct apple_dcp *dcp) * Only a processor whose port@0 links an audio device has DP/HDMI * audio to start. Look before the firmware: a processor without one * (an internal panel's, or one whose firmware has no audio service - * here, such as H17P) stays as it is without hdmi_audio, quietly. + * here, such as H17P) stays as it is, quietly. */ endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); if (endpoint) { diff --git a/drivers/gpu/drm/apple/av.h b/drivers/gpu/drm/apple/av.h index 1d710b0f706f3..6d25d83d78d79 100644 --- a/drivers/gpu/drm/apple/av.h +++ b/drivers/gpu/drm/apple/av.h @@ -6,7 +6,7 @@ //int avep_audiosrv_startlink(struct apple_dcp *dcp, struct dcp_sound_cookie *cookie); //int avep_audiosrv_stoplink(struct apple_dcp *dcp); -/* dcp.c: the opt-in to DP/HDMI audio (appledrm.hdmi_audio). */ +/* dcp.c: DP/HDMI audio, on unless appledrm.hdmi_audio=0. */ extern bool hdmi_audio; #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a724f318a778e..f233c079997cc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -204,9 +204,15 @@ static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and show the notch (also when /chosen/asahi,show-notch is \"1\")"); -bool hdmi_audio; +/* + * DP/HDMI audio of external displays, on by default as on macOS: a display + * that takes audio, over HDMI or DisplayPort (USB-C alt mode included), + * is an audio output while it is connected. hdmi_audio=0 leaves the audio + * endpoints stopped and registers no sound cards. + */ +bool hdmi_audio = true; module_param(hdmi_audio, bool, 0644); -MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); +MODULE_PARM_DESC(hdmi_audio, "DP/HDMI audio of external displays (default: on)"); static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); @@ -3919,7 +3925,7 @@ int dcp_start(struct platform_device *pdev) /* * A native T6030 external processor: its DP/HDMI audio endpoint (EP 0x29) - * on the driver's opt-in (hdmi_audio), as the other DCPs start it, for a + * unless hdmi_audio is off, as the other DCPs start it, for a * processor whose port@0 links an audio device; once per session, and after * the first display's EDID was read: with the endpoint's interfaces * published, an EDID copy (dcpavserv_copy_edid()) has the firmware rebuild From 9532c57d63be076a77ad97b9a3f0aa681e2ecc5f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:51 +1100 Subject: [PATCH 115/392] arm64: dts: apple: t6040-j616s: enable DP audio on DCPEXT1 DCPEXT1 drives USB-C DisplayPort on the J616s, next to DCPEXT0 (HDMI, and USB-C when DCPEXT1 is taken). Its audio interface is DPA2 on SIO channel 0x68, as macOS has it (dp-audio2, DCPAVAudioDMADelegate). With only DPA1 enabled a display on the port DCPEXT1 drives had no audio. Enable DPA2 and its power domain, as DPA1 already is. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../boot/dts/apple/t6040-j616s-dpaudio.dtsi | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi index 1f6e862285bc2..597feaa71ac51 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -3,13 +3,13 @@ * DPA1 and DPA2 belong to DCPEXT0 and DCPEXT1, respectively. SIO supplies * their sample streams; neither interface has AP-side registers. * - * The first chain is enabled: the SIO DART, mailbox and CPU domain, DPA1 - * and its client. The SIO itself stays disabled here: iBoot has loaded its - * firmware, and the boot loader enables the node once it has reserved that - * firmware and published the T6041 boot tables (it finds it by the alias). - * Without a T6041-aware loader DPA1 waits for its DMA channel and nothing - * else changes. The DCP side starts unless appledrm.hdmi_audio=0. - * DPA2 (DCPEXT1, SIO channel 0x68) stays disabled until it is qualified. + * Both chains are enabled: the SIO DART, mailbox and CPU domain, DPA1 and + * DPA2 and their clients. The SIO itself stays disabled here: iBoot has + * loaded its firmware, and the boot loader enables the node once it has + * reserved that firmware and published the T6041 boot tables (it finds it + * by the alias). Without a T6041-aware loader DPA1 and DPA2 wait for their + * DMA channels and nothing else changes. The DCP side starts unless + * appledrm.hdmi_audio=0. */ / { @@ -42,7 +42,7 @@ dmas = <&sio 0x68>; dma-names = "tx"; power-domains = <&ps_dpa2>; - status = "disabled"; + status = "okay"; ports { #address-cells = <1>; @@ -105,3 +105,7 @@ &ps_dpa1 { status = "okay"; }; + +&ps_dpa2 { + status = "okay"; +}; From ae7bd1043f2fc90df3a241885e4ae5d4a64c868a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:43:15 +1100 Subject: [PATCH 116/392] drm/apple: give display audio an ELD control Userspace names a display's audio output after the display, as macOS does, from the ELD control HDMI codecs have: PipeWire and PulseAudio read the monitor name, the channel count and the speaker layout from it. The DCP audio card had none, so every display was "Apple DisplayPort" in GNOME and KDE. Add the ELD control (PCM device 0, read-only, volatile) and fill it on connect from the ELD drm builds for the connector the processor drives. The firmware may hand out an EDID rebuilt from its display metadata, with no CTA-861 audio block: a sink the firmware reports as taking audio then gets the basic audio CTA-861 requires of every sink with audio, two-channel LPCM at 32, 44.1 and 48 kHz. The control is cleared on disconnect and changes before the jack does, so the name is there when the output becomes available. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/audio.c | 97 +++++++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/audio.h | 1 + drivers/gpu/drm/apple/av.c | 24 +++++++++ 3 files changed, 119 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 48539904efa91..64496a827c776 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -23,6 +23,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -36,6 +40,7 @@ #define DCPAUD_ELEMENTS_MAXSIZE 16384 #define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 +#define DCPAUD_ELD_SIZE MAX_ELD_BYTES /* The native element catalog capacity (14.7 external role). */ #define DCPAUD_ELEMENTS_V14_7 SZ_256K @@ -70,6 +75,7 @@ struct dcp_audio { struct dma_chan *chan; struct snd_card *card; struct snd_jack *jack; + struct snd_kcontrol *eld_ctl; struct snd_pcm_substream *substream; unsigned int open_cookie; bool open_unplugged; @@ -78,6 +84,8 @@ struct dcp_audio { bool dcp_connected; /// dcp status keep for delayed initialization bool connected; unsigned int connection_cookie; + /* The connected sink's ELD, zero while none is; data_lock */ + u8 eld[DCPAUD_ELD_SIZE]; /* When the sink was last connected */ ktime_t connected_at; @@ -1261,8 +1269,39 @@ static int dcpaud_create_pcm(struct dcp_audio *dcpaud) return 0; } +/* + * The sink's ELD, so that userspace names the output after the display, as + * macOS does, and sees its formats. drm builds it from the EDID. An EDID + * without a CTA-861 audio block (the firmware may provide one rebuilt from + * its display metadata, which has none) gets the basic audio CTA-861 makes + * every sink with audio take: two-channel LPCM at 32, 44.1 and 48 kHz. A + * sink is connected here only once the firmware reports it takes audio. + */ +static void dcpaud_build_eld(struct dcp_audio *dcpaud, u8 *eld) +{ + u8 *sad; + + if (dcp_audio_sink_eld(dcpaud->dcp_dev, eld, DCPAUD_ELD_SIZE) || + !eld[DRM_ELD_VER]) { + memset(eld, 0, DCPAUD_ELD_SIZE); + eld[DRM_ELD_VER] = DRM_ELD_VER_CEA861D; + } + if (!drm_eld_sad_count(eld)) { + sad = eld + DRM_ELD_CEA_SAD(drm_eld_mnl(eld), 0); + sad[0] = (HDMI_AUDIO_CODING_TYPE_PCM << 3) | (2 - 1); + sad[1] = BIT(0) | BIT(1) | BIT(2); /* 32, 44.1, 48 kHz */ + sad[2] = BIT(0) | BIT(1) | BIT(2); /* 16, 20, 24 bits */ + eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= 1 << DRM_ELD_SAD_COUNT_SHIFT; + if (!(eld[DRM_ELD_SPEAKER] & DRM_ELD_SPEAKER_MASK)) + eld[DRM_ELD_SPEAKER] = DRM_ELD_SPEAKER_FLR; + } + eld[DRM_ELD_BASELINE_ELD_LEN] = + DIV_ROUND_UP(drm_eld_calc_baseline_block_size(eld), 4); +} + /* expects to be called with data_lock locked and unlocks it */ -static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) +static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected, + const u8 *eld) { struct snd_pcm_substream *substream = dcpaud->substream; @@ -1276,8 +1315,14 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) dcpaud->connection_cookie++; dcpaud->connected_at = ktime_get(); } + if (connected && eld) + memcpy(dcpaud->eld, eld, DCPAUD_ELD_SIZE); + else + memset(dcpaud->eld, 0, DCPAUD_ELD_SIZE); mutex_unlock(&dcpaud->data_lock); + /* The name first: the jack makes the output available. */ + snd_ctl_notify(dcpaud->card, SNDRV_CTL_EVENT_MASK_VALUE, &dcpaud->eld_ctl->id); snd_jack_report(dcpaud->jack, connected ? SND_JACK_AVOUT : 0); if (!connected) { @@ -1289,6 +1334,42 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) } } +static int dcpaud_eld_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_BYTES; + uinfo->count = DCPAUD_ELD_SIZE; + return 0; +} + +static int dcpaud_eld_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct dcp_audio *dcpaud = snd_kcontrol_chip(kcontrol); + + guard(mutex)(&dcpaud->data_lock); + memcpy(ucontrol->value.bytes.data, dcpaud->eld, DCPAUD_ELD_SIZE); + return 0; +} + +/* The ELD control of HDMI codecs, which PipeWire and PulseAudio read. */ +static int dcpaud_create_eld_ctl(struct dcp_audio *dcpaud) +{ + struct snd_kcontrol_new tmpl = { + .access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .iface = SNDRV_CTL_ELEM_IFACE_PCM, + .name = "ELD", + .device = dcpaud->substream->pcm->device, + .info = dcpaud_eld_info, + .get = dcpaud_eld_get, + }; + + dcpaud->eld_ctl = snd_ctl_new1(&tmpl, dcpaud); + if (!dcpaud->eld_ctl) + return -ENOMEM; + return snd_ctl_add(dcpaud->card, dcpaud->eld_ctl); +} + static int dcpaud_create_jack(struct dcp_audio *dcpaud) { struct snd_card *card = dcpaud->card; @@ -1356,6 +1437,10 @@ static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) if (ret) goto err_free_card; + ret = dcpaud_create_eld_ctl(dcpaud); + if (ret) + goto err_free_card; + ret = dcpaud_create_jack(dcpaud); if (ret) goto err_free_card; @@ -1375,10 +1460,16 @@ static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) void dcpaud_connect(struct platform_device *pdev, bool connected) { struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + u8 eld[DCPAUD_ELD_SIZE]; + + if (connected && dcpaud->dcp_dev) + dcpaud_build_eld(dcpaud, eld); + else + memset(eld, 0, sizeof(eld)); mutex_lock(&dcpaud->data_lock); - dcpaud_report_hotplug(dcpaud, connected); + dcpaud_report_hotplug(dcpaud, connected, eld); } void dcpaud_disconnect(struct platform_device *pdev) @@ -1389,7 +1480,7 @@ void dcpaud_disconnect(struct platform_device *pdev) dcpaud = platform_get_drvdata(pdev); mutex_lock(&dcpaud->data_lock); - dcpaud_report_hotplug(dcpaud, false); + dcpaud_report_hotplug(dcpaud, false, NULL); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) /* The service may go without a release, and a held link with it. */ if (dcpaud->native_link) diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h index 79bee44bcefbc..6cda3f38c9308 100644 --- a/drivers/gpu/drm/apple/audio.h +++ b/drivers/gpu/drm/apple/audio.h @@ -13,6 +13,7 @@ int dcp_audiosrv_stoplink(struct device *dev); int dcp_audiosrv_unprepare(struct device *dev); int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize); int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize); +int dcp_audio_sink_eld(struct device *dev, u8 *eld, size_t size); void dcpaud_connect(struct platform_device *pdev, bool connected); void dcpaud_disconnect(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 23d963992cbdd..81b7389f9f99f 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -591,6 +591,30 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi return ret; } +/* + * The ELD of the display this processor drives, as drm builds it from the + * connector's EDID: monitor name, vendor and product codes, connection type + * and the audio descriptors of a CTA-861 audio block, if there is one. + * Zeroes without a connector or EDID. + */ +int dcp_audio_sink_eld(struct device *dev, u8 *eld, size_t size) +{ + struct apple_dcp *dcp; + struct apple_connector *connector; + + KUNIT_STATIC_STUB_REDIRECT(dcp_audio_sink_eld, dev, eld, size); + memset(eld, 0, size); + dcp = dev_get_drvdata(dev); + if (!dcp) + return -ENODEV; + connector = READ_ONCE(dcp->connector); + if (!connector) + return -ENODEV; + guard(mutex)(&connector->base.eld_mutex); + memcpy(eld, connector->base.eld, min(size, sizeof(connector->base.eld))); + return 0; +} + #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) /* A DCPAV report (group 2) with command 1: Terminate. */ static bool av_report_terminate(const void *data, size_t size) From deb91317b986add207307ccde2498d78beda4229 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:46:00 +1100 Subject: [PATCH 117/392] soc: apple: clpc: hold the T6040 DRAM floor for ANE work On the M4 Pro the ANE votes only the PMC SoC-max floor (fabric 2) while an EXECUTE is in flight. Nothing raises the DRAM controller: the DCS stays at state 0 unless a CPU cluster happens to be busy. ResNet-50 streams about 50 MB of weights per inference, so it runs DRAM bound: host p50 1.43 ms with a 0.93-1.98 ms spread, the fast runs being the ones where CLPC had raised the DCS for an unrelated CPU burst. Give the T6040 CLPC the ANE input the T8140 already has (DCS 5, fabric 2: the states a busy CPU cluster gets), let apple_clpc_ane_get() find the T6041 CLPC, and publish H16 EXECUTEs in flight from the H16 backend, next to the existing perf floor vote. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/accel/ane/ane_h16_backend.c | 10 ++++++++++ drivers/soc/apple/clpc.c | 22 +++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index e44e55e6ac510..5b8cd939d1a5f 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,8 @@ struct ane_h16_device { /* Optional "perf" floor, voted only while an EXECUTE is in flight. */ struct device *perf; bool perf_held; + /* Optional CLPC input: DRAM and fabric floors while an EXECUTE is in flight. */ + struct apple_clpc_ane *clpc; struct device *dart[3]; u32 powered; u32 darts_powered; @@ -1171,6 +1174,8 @@ static void ane_h16_release(struct ane_h16_device *h) dev_pm_domain_detach(h->perf, false); h->perf = NULL; } + apple_clpc_ane_put(h->clpc); + h->clpc = NULL; while (h->powered) { struct device *pd = h->power[--h->powered]; @@ -1216,6 +1221,7 @@ static void ane_h16_busy(void *ctx, bool busy) { struct ane_h16_device *h = ctx; + apple_clpc_ane_set_active(h->clpc, busy); if (!h->perf) return; if (busy) { @@ -1292,6 +1298,10 @@ static int ane_h16_acquire(struct platform_device *pdev, struct ane_h16_device * if (ret) return ret; } + /* Optional: success with no input when the SoC's CLPC takes none. */ + ret = apple_clpc_ane_get(h->dev, &h->clpc); + if (ret) + return ret; for (i = 0; i < ARRAY_SIZE(names); i++) { struct resource *r = platform_get_resource_byname(pdev, IORESOURCE_MEM, names[i]); diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index fa94e1d522937..26aa54fffa2be 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -334,6 +334,13 @@ void apple_clpc_ane_put(struct apple_clpc_ane *ane) } EXPORT_SYMBOL_GPL(apple_clpc_ane_put); +/* SoCs whose CLPC takes ANE input; each SoC has exactly one CLPC. */ +static const struct of_device_id apple_clpc_ane_providers[] = { + { .compatible = "apple,t6041-clpc" }, + { .compatible = "apple,t8140-clpc" }, + {} +}; + int apple_clpc_ane_get(struct device *dev, struct apple_clpc_ane **out) { struct device_node *np; @@ -345,10 +352,10 @@ int apple_clpc_ane_get(struct device *dev, struct apple_clpc_ane **out) return 0; np = of_parse_phandle(dev->of_node, "apple,clpc", 0); if (!np && !of_find_property(dev->of_node, "apple,clpc", NULL)) - np = of_find_compatible_node(NULL, NULL, "apple,t8140-clpc"); + np = of_find_matching_node(NULL, apple_clpc_ane_providers); if (!np) return 0; - if (!of_device_is_compatible(np, "apple,t8140-clpc")) { + if (!of_match_node(apple_clpc_ane_providers, np)) { of_node_put(np); return 0; } @@ -1181,6 +1188,13 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { .sustain = 3, .gpu_demand = false, .gpu_input = true, + /* + * The ANE's own floor (PMC SoC-max) holds only the fabric. ResNet-50 + * streams about 50 MB of weights per inference, and with DRAM left at + * DCS 0 an EXECUTE takes 0.9-2.0 ms (p50 1.43 ms) instead of 0.9 ms. + * Hold the same qualified DCS 5 / fabric 2 a busy CPU cluster gets. + */ + .ane = { [APPLE_CLPC_DCS] = 5, [APPLE_CLPC_SOC] = 2 }, }; /* @@ -1393,6 +1407,9 @@ static void apple_clpc_ane_demand_hold_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_clpc_ane_input_demand(clpc, demand)); KUNIT_EXPECT_GT(test, ane.updated_ns, 0ull); KUNIT_EXPECT_EQ(test, ane.sequence, 4ull); + /* T6040 holds its top qualified DCS and fabric states for the ANE. */ + KUNIT_EXPECT_EQ(test, apple_clpc_hw_t6040.ane[APPLE_CLPC_DCS], 5); + KUNIT_EXPECT_EQ(test, apple_clpc_hw_t6040.ane[APPLE_CLPC_SOC], 2); } static void apple_clpc_ane_states_test(struct kunit *test) @@ -1435,7 +1452,6 @@ static void apple_clpc_ane_no_provider_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_clpc_ane_input_demand(clpc, demand)); KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_DCS], 0u); KUNIT_EXPECT_EQ(test, demand[APPLE_CLPC_SOC], 0u); - KUNIT_EXPECT_EQ(test, apple_clpc_hw_t6040.ane[APPLE_CLPC_DCS], 0); KUNIT_EXPECT_EQ(test, apple_clpc_hw_t8132.ane[APPLE_CLPC_DCS], 0); KUNIT_EXPECT_EQ(test, apple_clpc_hw_t8142.ane[APPLE_CLPC_DCS], 0); } From 3241d0c726a4eef94148dba7b399b8c88ab9d37e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:46:20 +1100 Subject: [PATCH 118/392] drm/apple: retry an audio open the firmware gave to a replaced instance When a display wakes up (DPMS on), the DCP reports its sink on the audio interface instance it had and replaces that instance some 200 ms later. PipeWire opens the PCM as soon as the jack turns on, the Open goes to the old instance, the firmware answers it with a reply that matches no call, and the Open times out a second later. PipeWire then drops the output for good: on the J616s, with the Acer QG270 S3 on a USB-C dock, every DPMS cycle left the display without an audio output ("playback open failed: Connection timed out"). Retry the Open once when the published instance changed while it ran. The second Open goes to the new instance and is answered. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/av.c | 68 ++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 81b7389f9f99f..fae38c8468ad3 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -295,7 +295,8 @@ int dcp_audiosrv_open(struct device *dev) struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; struct apple_epic_service *srv; - u8 reply[32] = {}; + u8 reply[32]; + bool replaced; u64 gen; int ret; @@ -303,32 +304,51 @@ int dcp_audiosrv_open(struct device *dev) return -ENODEV; guard(mutex)(&asrv->open_lock); - scoped_guard(rwsem_read, &asrv->srv_rwsem) { - srv = afk_service_get(asrv->srv); - gen = asrv->gen; - } - if (!srv) - return -ENODEV; - if (asrv->open_srv) { - if (asrv->open_gen == gen) { - afk_service_put(srv); - return 0; + /* + * A display that wakes up (DPMS on) reports its sink on the audio + * interface it had, and the firmware replaces that instance a few + * hundred milliseconds later: an Open sent in between is refused + * with a reply that matches no call, and times out. The Open that + * follows the replacement is answered, so try the instance published + * by then once more. Userspace drops an output whose first open + * fails, so this is the difference between a display with and + * without audio after it wakes up. + */ + for (int attempt = 0; ; attempt++) { + scoped_guard(rwsem_read, &asrv->srv_rwsem) { + srv = afk_service_get(asrv->srv); + gen = asrv->gen; + } + if (!srv) + return -ENODEV; + if (asrv->open_srv) { + if (asrv->open_gen == gen) { + afk_service_put(srv); + return 0; + } + /* An earlier instance's open went with that instance. */ + afk_service_put(asrv->open_srv); + asrv->open_srv = NULL; } - /* An earlier instance's open went with that instance. */ - afk_service_put(asrv->open_srv); - asrv->open_srv = NULL; - } - /* Open: a zero body; status at +0, the opened Boolean at +16. */ - ret = av_audiosrv_call(srv, asrv->cmds.open, NULL, 0, 32, reply, sizeof(reply)); - if (!ret && (get_unaligned_le32(reply) || !get_unaligned_le64(reply + 16))) { - dev_err(dev, "audiosrv: open answered %#x, opened %llu\n", - get_unaligned_le32(reply), get_unaligned_le64(reply + 16)); - ret = -EIO; - } - if (ret) { + /* Open: a zero body; status at +0, the opened Boolean at +16. */ + memset(reply, 0, sizeof(reply)); + ret = av_audiosrv_call(srv, asrv->cmds.open, NULL, 0, 32, reply, sizeof(reply)); + if (!ret && (get_unaligned_le32(reply) || !get_unaligned_le64(reply + 16))) { + dev_err(dev, "audiosrv: open answered %#x, opened %llu\n", + get_unaligned_le32(reply), get_unaligned_le64(reply + 16)); + ret = -EIO; + } + if (!ret) + break; afk_service_put(srv); - return ret; + if (attempt) + return ret; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + replaced = asrv->srv && asrv->gen != gen; + if (!replaced) + return ret; + dev_dbg(dev, "audiosrv: open on a replaced instance failed (%d), retrying\n", ret); } asrv->open_srv = srv; asrv->open_gen = gen; From 44f79bc3863431b9674ae599732d787dfb99d455 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:19:13 +1100 Subject: [PATCH 119/392] drm/apple: take the T6041 DCPEXT video interface on endpoint 0x2b With display audio on, every power-on of a display on a T6041 DCPEXT logged "unable to match compact service" for its DCPAV video interface, which endpoint 0x2b publishes cut to "dispext0:dcpav-video-interface-". The video side runs through IOMFB: take the interface with the no-op service the CEC interface already uses. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/av.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index fae38c8468ad3..550e695e2485c 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -753,6 +753,18 @@ static const struct apple_epic_service_ops av_desktop_ops[] = { .report = av_cec_report, .teardown = av_cec_teardown, }, + { + /* + * And its video interface, cut to "dcpav-video-interface-" on + * every display power-on. The video side runs through the + * IOMFB endpoint; take it like the CEC interface. + */ + .name = "dcpav-video-interface-epic", + .compact_full = "dcpav-video-interface-epic:0", + .init = av_cec_init, + .report = av_cec_report, + .teardown = av_cec_teardown, + }, {} }; #endif From f20d2ce0230c0ef7cd5e71aff6e222c03f569d4b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:46:00 +1100 Subject: [PATCH 120/392] soc: apple: clpc: hold the T6040 ANE cluster frequency floor On the M4 Pro the ANE engine runs at a speed set by the frequency of the second P cluster (cpu-map cluster2), whether or not anything runs on it. With that cluster at its idle frequency MobileNetV1 takes 65-375 us of engine time per EXECUTE (mean ~165 us) and ResNet-50 0.8-1.7 ms; with it at 3.3 GHz or above, 88-91 us and 0.70-0.97 ms. Raising the first P cluster or the E cluster changes nothing; above 3.756 GHz nothing more is gained. While ANE work is in flight, and for the vote hold after it, add a cpufreq minimum-frequency request of 3.756 GHz on that cluster, next to the DCS and fabric floors. The request is added on first use, since cpufreq may register after the CLPC, and dropped on halt. debugfs ane_cluster_khz sets the floor for A/B runs (0 = off). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/soc/apple/clpc.c | 131 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 26aa54fffa2be..f1194055f6e0a 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -21,6 +21,7 @@ */ #include +#include #include #include #include @@ -80,6 +81,10 @@ enum apple_clpc_rail { * @gpu_dense: submissions per sampling period that make light GPU work * dense, see apple_clpc_gpu_dense(); 0 for none * @ane: state of each rail while ANE work is in flight; all zero disables input + * @ane_cluster: /cpus/cpu-map cluster whose CPUs' frequency the ANE engine + * speed follows + * @ane_cluster_khz: minimum frequency of that cluster while ANE work is in + * flight; 0 for none * @gpu_input: the GPU driver reports its work in flight through * apple_clpc_gpu_get(); while there is some, both rails go to the top * of @map @@ -96,6 +101,8 @@ struct apple_clpc_hw { unsigned int gpu_dense; bool gpu_input; u8 ane[APPLE_CLPC_RAILS]; + unsigned int ane_cluster; + unsigned int ane_cluster_khz; }; struct apple_clpc_cluster { @@ -161,6 +168,12 @@ struct apple_clpc { struct apple_clpc_gpu *gpu; struct apple_clpc_ane *ane; unsigned int ane_states[APPLE_CLPC_RAILS]; + /* Minimum frequency of hw->ane_cluster while ANE work is in flight. */ + u32 ane_cluster_khz; + int ane_cpu; + struct freq_qos_request ane_qos; + unsigned int ane_qos_khz; + unsigned long ane_until; u32 last_floor; u32 last_reason; u64 last_vote_ns; @@ -678,6 +691,38 @@ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, return true; } +/* + * Hold the ANE's cluster frequency floor from the first sample that sees ANE + * work until the vote hold after the last one. The request is added on first + * use: cpufreq may register its policies after the CLPC probes. + */ +static void apple_clpc_ane_cluster(struct apple_clpc *clpc, bool active, + unsigned long now) +{ + unsigned int khz = 0; + + if (active) + clpc->ane_until = now + msecs_to_jiffies(APPLE_CLPC_HOLD_MS); + if (time_before(now, clpc->ane_until)) + khz = clpc->ane_cluster_khz; + if (khz && !freq_qos_request_active(&clpc->ane_qos) && clpc->ane_cpu >= 0) { + struct cpufreq_policy *policy = cpufreq_cpu_get(clpc->ane_cpu); + + if (policy) { + if (freq_qos_add_request(&policy->constraints, &clpc->ane_qos, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE) < 0) + clpc->update_errors++; + cpufreq_cpu_put(policy); + } + } + if (khz == clpc->ane_qos_khz || !freq_qos_request_active(&clpc->ane_qos)) + return; + if (freq_qos_update_request(&clpc->ane_qos, khz ?: FREQ_QOS_MIN_DEFAULT_VALUE) < 0) + clpc->update_errors++; + else + clpc->ane_qos_khz = khz; +} + static void apple_clpc_sample(struct apple_clpc *clpc) { unsigned int demand[APPLE_CLPC_RAILS] = {}; @@ -707,6 +752,7 @@ static void apple_clpc_sample(struct apple_clpc *clpc) reason |= 2; if (apple_clpc_ane_input_demand(clpc, demand)) reason |= 32; + apple_clpc_ane_cluster(clpc, reason & 32, now); if (clpc->gpu_pmgr) { /* Both run every sample: each tracks state across samples. */ bool late = time_after(now, clpc->gpu_sampled + @@ -746,7 +792,8 @@ static void apple_clpc_sample(struct apple_clpc *clpc) clpc->update_errors++; /* Sample on time while graphics is powered, to see it power off. */ - apple_clpc_queue(clpc, !apple_clpc_floor(clpc, state) && !clpc->gpu_on); + apple_clpc_queue(clpc, !apple_clpc_floor(clpc, state) && !clpc->gpu_on && + !clpc->ane_qos_khz); } static void apple_clpc_work(struct work_struct *work) @@ -921,6 +968,10 @@ static void apple_clpc_halt(struct apple_clpc *clpc) clpc->saved_floor); for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) clpc->vote[rail].state = 0; + if (clpc->ane_qos_khz && + freq_qos_update_request(&clpc->ane_qos, FREQ_QOS_MIN_DEFAULT_VALUE) >= 0) + clpc->ane_qos_khz = 0; + clpc->ane_until = jiffies; } static void apple_clpc_stop(void *data) @@ -940,9 +991,39 @@ static void apple_clpc_stop(void *data) apple_clpc_ane_detach(clpc->ane); debugfs_remove_recursive(clpc->debugfs); apple_clpc_halt(clpc); + if (freq_qos_request_active(&clpc->ane_qos)) + freq_qos_remove_request(&clpc->ane_qos); apple_clpc_free_clusters(clpc); } +/* The first CPU of /cpus/cpu-map/cluster@index, or -ENODEV. */ +static int apple_clpc_cluster_cpu(unsigned int index) +{ + struct device_node *map, *cluster, *core, *cpu; + char name[16]; + int id = -ENODEV; + + map = of_find_node_by_path("/cpus/cpu-map"); + if (!map) + return id; + snprintf(name, sizeof(name), "cluster%u", index); + cluster = of_get_child_by_name(map, name); + of_node_put(map); + if (!cluster) + return id; + core = of_get_child_by_name(cluster, "core0"); + of_node_put(cluster); + if (!core) + return id; + cpu = of_parse_phandle(core, "cpu", 0); + of_node_put(core); + if (cpu) { + id = of_cpu_node_to_id(cpu); + of_node_put(cpu); + } + return id; +} + static int apple_clpc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -985,6 +1066,15 @@ static int apple_clpc_probe(struct platform_device *pdev) for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) clpc->ane_states[rail] = clpc->hw->ane[rail]; } + clpc->ane_cpu = -ENODEV; + if (clpc->ane && clpc->hw->ane_cluster_khz) { + clpc->ane_cpu = apple_clpc_cluster_cpu(clpc->hw->ane_cluster); + if (clpc->ane_cpu >= 0) + clpc->ane_cluster_khz = clpc->hw->ane_cluster_khz; + else + dev_warn(dev, "no CPU in cpu-map cluster%u: no ANE cluster floor\n", + clpc->hw->ane_cluster); + } /* The property is optional: without it, CPU demand alone sets the floors. */ if (clpc->hw->gpu_demand) gpu = of_parse_phandle(dev->of_node, "apple,gpu-power-state", 0); @@ -1059,6 +1149,9 @@ static int apple_clpc_probe(struct platform_device *pdev) if (clpc->ane) debugfs_create_file("ane_states", 0600, clpc->debugfs, clpc, &apple_clpc_ane_states_fops); + if (clpc->ane && clpc->ane_cpu >= 0) + debugfs_create_u32("ane_cluster_khz", 0600, clpc->debugfs, + &clpc->ane_cluster_khz); ret = devm_add_action_or_reset(dev, apple_clpc_stop, clpc); if (ret) @@ -1189,12 +1282,19 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { .gpu_demand = false, .gpu_input = true, /* - * The ANE's own floor (PMC SoC-max) holds only the fabric. ResNet-50 - * streams about 50 MB of weights per inference, and with DRAM left at - * DCS 0 an EXECUTE takes 0.9-2.0 ms (p50 1.43 ms) instead of 0.9 ms. - * Hold the same qualified DCS 5 / fabric 2 a busy CPU cluster gets. + * ANE work needs two things its own floor (PMC SoC-max: fabric 2) + * does not give it. ResNet-50 streams about 50 MB of weights per + * inference and runs DRAM bound at DCS 0, so hold the DCS 5 / fabric + * 2 a busy CPU cluster gets. And the engine runs at a speed set by + * the frequency of the second P cluster (cpu-map cluster2), whatever + * runs on it: MobileNetV1 takes 65-375 us of engine time (mean 165) + * with that cluster at its idle frequency and 88-91 us with it at + * 3.3 GHz or above (no gain above 3.756 GHz, measured at 1.8 to 4.5); + * the first P cluster and the E cluster make no difference. */ .ane = { [APPLE_CLPC_DCS] = 5, [APPLE_CLPC_SOC] = 2 }, + .ane_cluster = 2, + .ane_cluster_khz = 3756000, }; /* @@ -1412,6 +1512,26 @@ static void apple_clpc_ane_demand_hold_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_clpc_hw_t6040.ane[APPLE_CLPC_SOC], 2); } +static void apple_clpc_ane_cluster_test(struct kunit *test) +{ + struct apple_clpc *clpc = kunit_kzalloc(test, sizeof(*clpc), GFP_KERNEL); + unsigned long now = 1000; + + KUNIT_ASSERT_NOT_NULL(test, clpc); + /* No CPU for the cluster: the floor is never requested. */ + clpc->ane_cpu = -ENODEV; + clpc->ane_cluster_khz = apple_clpc_hw_t6040.ane_cluster_khz; + apple_clpc_ane_cluster(clpc, true, now); + KUNIT_EXPECT_FALSE(test, freq_qos_request_active(&clpc->ane_qos)); + KUNIT_EXPECT_EQ(test, clpc->ane_qos_khz, 0u); + /* The hold runs from the last active sample. */ + KUNIT_EXPECT_TRUE(test, time_before(now, clpc->ane_until)); + KUNIT_EXPECT_FALSE(test, time_before(now + msecs_to_jiffies(APPLE_CLPC_HOLD_MS), + clpc->ane_until)); + KUNIT_EXPECT_EQ(test, apple_clpc_hw_t6040.ane_cluster, 2u); + KUNIT_EXPECT_EQ(test, apple_clpc_hw_t8140.ane_cluster_khz, 0u); +} + static void apple_clpc_ane_states_test(struct kunit *test) { struct apple_clpc *clpc = kunit_kzalloc(test, sizeof(*clpc), GFP_KERNEL); @@ -1485,6 +1605,7 @@ static void apple_clpc_ane_wake_removal_test(struct kunit *test) static struct kunit_case apple_clpc_cases[] = { KUNIT_CASE(apple_clpc_ane_demand_hold_test), KUNIT_CASE(apple_clpc_ane_states_test), + KUNIT_CASE(apple_clpc_ane_cluster_test), KUNIT_CASE(apple_clpc_ane_no_provider_test), KUNIT_CASE(apple_clpc_ane_wake_removal_test), KUNIT_CASE(apple_clpc_t8132_gpu_input_test), From 88b4acd769a0d01bfca40280226216a197c54516 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:47:49 +1100 Subject: [PATCH 121/392] tools/asahi: cover the T6040 ANE floors in the CLPC host demand test Extract apple_clpc_ane_cluster() into the host fixture with cpufreq QoS stand-ins and check that T6040 ANE work publishes DCS 5 / fabric 2, adds the cluster minimum-frequency request once, keeps it through the vote hold and drops it after. Two negative controls: the request never made, and the floor dropped without the hold. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- tools/asahi/clpc-gpu-demand-tests.py | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index c8c2c9d9ad361..57e3d43fc135e 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -38,6 +38,7 @@ def main(): 'static void apple_clpc_gpu_floor_count(', 'static bool apple_clpc_gpu_input_demand(', 'static bool apple_clpc_ane_input_demand(', + 'static void apple_clpc_ane_cluster(', 'static void apple_clpc_sample(']) code = r''' #include @@ -69,6 +70,20 @@ def main(): #define spin_lock_irqsave(l, f) ((void)(l), (f) = 0) #define spin_unlock_irqrestore(l, f) ((void)(l), (void)(f)) struct apple_clpc_cluster { unsigned int request[APPLE_CLPC_RAILS]; }; +// cpufreq QoS stand-ins: one policy, the last requested minimum is recorded. +#define FREQ_QOS_MIN 1 +#define FREQ_QOS_MIN_DEFAULT_VALUE 0 +struct freq_qos_request { bool active; int value; }; +struct cpufreq_policy { int constraints; }; +static struct cpufreq_policy policy; +static int qos_min = -1, qos_adds; +static bool freq_qos_request_active(struct freq_qos_request *r) { return r->active; } +static struct cpufreq_policy *cpufreq_cpu_get(int cpu) { (void)cpu; return &policy; } +static void cpufreq_cpu_put(struct cpufreq_policy *p) { (void)p; } +static int freq_qos_add_request(int *c, struct freq_qos_request *r, int type, int v) { + (void)c; assert(type==FREQ_QOS_MIN && !r->active); r->active=true; r->value=v; qos_min=v; qos_adds++; return 1; +} +static int freq_qos_update_request(struct freq_qos_request *r, int v) { assert(r->active); r->value=v; qos_min=v; return 1; } struct apple_clpc { const struct apple_clpc_hw *hw; struct apple_clpc_cluster clusters[1]; @@ -88,6 +103,11 @@ def main(): struct apple_clpc_gpu *gpu; struct apple_clpc_ane *ane; unsigned int ane_states[APPLE_CLPC_RAILS]; + u32 ane_cluster_khz; + int ane_cpu; + struct freq_qos_request ane_qos; + unsigned int ane_qos_khz; + unsigned long ane_until; }; static unsigned long jiffies; static u64 ktime_get_mono_fast_ns(void) { return jiffies * 1000000ull; } @@ -127,6 +147,25 @@ def main(): jiffies=1040; apple_clpc_sample(&c); assert(published==0 && queued_idle); } +// T6040 ANE work also holds the second P cluster at its qualified frequency, +// added on first use, for the vote hold after the last request, then drops it. +static void check_t6040_ane(void) { + struct apple_clpc_ane ane={0}; + struct apple_clpc c={.hw=&apple_clpc_hw_t6040,.agent=1,.mask=3,.ane=&ane}; + c.ane_states[APPLE_CLPC_DCS]=c.hw->ane[APPLE_CLPC_DCS]; + c.ane_states[APPLE_CLPC_SOC]=c.hw->ane[APPLE_CLPC_SOC]; + c.ane_cpu=9; c.ane_cluster_khz=c.hw->ane_cluster_khz; + assert(c.hw->ane_cluster==2 && c.ane_cluster_khz==3756000); + read_error=vote_error=0; pmgr_status=0; pmp_floor=0; jiffies=2000; published=~0u; qos_min=-1; qos_adds=0; + apple_clpc_sample(&c); assert(published==0 && qos_adds==0 && queued_idle); + ane.active=true; jiffies=2004; apple_clpc_sample(&c); + assert(published==0x25 && c.last_reason==32 && qos_adds==1 && qos_min==3756000); + ane.active=false; jiffies=2008; apple_clpc_sample(&c); + assert(qos_min==3756000 && !queued_idle); + jiffies=2036; apple_clpc_sample(&c); assert(published==0 && qos_min==0 && c.ane_qos_khz==0 && queued_idle); + ane.active=true; jiffies=2040; apple_clpc_sample(&c); assert(qos_adds==1 && qos_min==3756000); +} + // T8132's G16 firmware keeps graphics powered while the GPU idles: the // M5/F3 floor follows the work the GPU driver reports, as on T6040. static void check_t8132(void) { @@ -229,6 +268,7 @@ def main(): check_t8140(); check_t8140_dense(); check_t8140_ane(); + check_t6040_ane(); puts("PASS: T6040 and T8132 host demand; T8140 ANE, streaming, dense work, read errors and PMP backoff; idle release"); } ''' @@ -240,6 +280,10 @@ def main(): ('omit-host-demand', code.replace('if (apple_clpc_gpu_input_demand(clpc->hw, clpc->gpu, demand))', 'if (false && apple_clpc_gpu_input_demand(clpc->hw, clpc->gpu, demand))')), ('omit-ane-demand', code.replace('if (apple_clpc_ane_input_demand(clpc, demand))', 'if (false && apple_clpc_ane_input_demand(clpc, demand))')), + ('omit-ane-cluster', code.replace('apple_clpc_ane_cluster(clpc, reason & 32, now);', + '(void)apple_clpc_ane_cluster;')), + ('ane-cluster-no-hold', code.replace('if (time_before(now, clpc->ane_until))', + 'if (active)')), ('omit-streaming-demand', code.replace('bool streaming = apple_clpc_gpu_streaming(clpc, now);', 'bool streaming = (apple_clpc_gpu_streaming(clpc, now), false);')), ('omit-dense-demand', code.replace('if (streaming || dense) {', 'if (streaming || (dense && false)) {')), From 285908af68b66cb274884000b8aeb7746055f16d Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:01:01 +1100 Subject: [PATCH 122/392] soc: apple: clpc: withdraw the ANE cluster floor with its cpufreq policy Review follow-ups for the T6040 ANE floors: - The cluster floor is a QoS request on a cpufreq policy, and the Apple cpufreq driver is a module: withdraw the request when that policy is removed (CPUFREQ_REMOVE_POLICY) instead of updating freed memory, and let the next ANE work add it again. - The floor was measured on the M4 Pro only. The M4 Max and T6050 share this CLPC table, so apply the cluster floor only on machines compatible with apple,t6040, and say so at debug level elsewhere. - Start the hold expired, and restart it from the current time once it has run out, so that jiffies wrap can never re-arm it. - The H16 backend looks the CLPC up again before its node appears, in case the CLPC bound after acquire, and says when the input is attached. - Drop an unneeded include. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/accel/ane/ane_h16_backend.c | 7 ++-- drivers/soc/apple/clpc.c | 52 +++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 5b8cd939d1a5f..eb1de39a24f09 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -1458,6 +1458,9 @@ static int ane_h16_program_stage(struct ane_h16_device *h) { int ret; + /* Before the node exists: the CLPC may have bound after acquire. */ + if (!h->clpc && apple_clpc_ane_get(h->dev, &h->clpc)) + h->clpc = NULL; ret = ane_h16_session_init(&h->session, &h->boot.transport, &ane_h16_session_ops, h, h->boot.build->command_timeout_ns); if (!ret) { @@ -1470,8 +1473,8 @@ static int ane_h16_program_stage(struct ane_h16_device *h) return ret; } h->program_ready = true; - dev_info(h->dev, "H16 stage 5 program window %llx..%llx\n", h->program_next, - h->program_end - 1); + dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_next, + h->program_end - 1, h->clpc ? ", CLPC input attached" : ""); return 0; } diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index f1194055f6e0a..6dfd277463a13 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -21,7 +21,6 @@ */ #include -#include #include #include #include @@ -85,6 +84,8 @@ enum apple_clpc_rail { * speed follows * @ane_cluster_khz: minimum frequency of that cluster while ANE work is in * flight; 0 for none + * @ane_cluster_machine: the machine compatible the cluster floor was + * measured on; SoCs sharing this CLPC without it get no cluster floor * @gpu_input: the GPU driver reports its work in flight through * apple_clpc_gpu_get(); while there is some, both rails go to the top * of @map @@ -103,6 +104,7 @@ struct apple_clpc_hw { u8 ane[APPLE_CLPC_RAILS]; unsigned int ane_cluster; unsigned int ane_cluster_khz; + const char *ane_cluster_machine; }; struct apple_clpc_cluster { @@ -174,6 +176,8 @@ struct apple_clpc { struct freq_qos_request ane_qos; unsigned int ane_qos_khz; unsigned long ane_until; + struct notifier_block policy_nb; + bool policy_nb_added; u32 last_floor; u32 last_reason; u64 last_vote_ns; @@ -705,6 +709,8 @@ static void apple_clpc_ane_cluster(struct apple_clpc *clpc, bool active, clpc->ane_until = now + msecs_to_jiffies(APPLE_CLPC_HOLD_MS); if (time_before(now, clpc->ane_until)) khz = clpc->ane_cluster_khz; + else + clpc->ane_until = now; if (khz && !freq_qos_request_active(&clpc->ane_qos) && clpc->ane_cpu >= 0) { struct cpufreq_policy *policy = cpufreq_cpu_get(clpc->ane_cpu); @@ -991,11 +997,34 @@ static void apple_clpc_stop(void *data) apple_clpc_ane_detach(clpc->ane); debugfs_remove_recursive(clpc->debugfs); apple_clpc_halt(clpc); + if (clpc->policy_nb_added) + cpufreq_unregister_notifier(&clpc->policy_nb, CPUFREQ_POLICY_NOTIFIER); if (freq_qos_request_active(&clpc->ane_qos)) freq_qos_remove_request(&clpc->ane_qos); apple_clpc_free_clusters(clpc); } +/* + * The cluster floor is a request on a cpufreq policy, which goes away with + * the cpufreq driver (a module). Withdraw it then; it is added again on the + * next ANE work once a policy covers the CPU. + */ +static int apple_clpc_policy_notify(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct apple_clpc *clpc = container_of(nb, struct apple_clpc, policy_nb); + struct cpufreq_policy *policy = data; + + if (event != CPUFREQ_REMOVE_POLICY || + !cpumask_test_cpu(clpc->ane_cpu, policy->related_cpus)) + return NOTIFY_DONE; + guard(mutex)(&clpc->sample_lock); + if (freq_qos_request_active(&clpc->ane_qos)) + freq_qos_remove_request(&clpc->ane_qos); + clpc->ane_qos_khz = 0; + return NOTIFY_OK; +} + /* The first CPU of /cpus/cpu-map/cluster@index, or -ENODEV. */ static int apple_clpc_cluster_cpu(unsigned int index) { @@ -1067,13 +1096,21 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->ane_states[rail] = clpc->hw->ane[rail]; } clpc->ane_cpu = -ENODEV; - if (clpc->ane && clpc->hw->ane_cluster_khz) { + clpc->ane_until = jiffies; + if (clpc->ane && clpc->hw->ane_cluster_khz && + of_machine_is_compatible(clpc->hw->ane_cluster_machine)) { clpc->ane_cpu = apple_clpc_cluster_cpu(clpc->hw->ane_cluster); - if (clpc->ane_cpu >= 0) - clpc->ane_cluster_khz = clpc->hw->ane_cluster_khz; - else - dev_warn(dev, "no CPU in cpu-map cluster%u: no ANE cluster floor\n", - clpc->hw->ane_cluster); + if (clpc->ane_cpu >= 0) { + clpc->policy_nb.notifier_call = apple_clpc_policy_notify; + if (!cpufreq_register_notifier(&clpc->policy_nb, + CPUFREQ_POLICY_NOTIFIER)) { + clpc->policy_nb_added = true; + clpc->ane_cluster_khz = clpc->hw->ane_cluster_khz; + } + } + if (!clpc->ane_cluster_khz) + dev_dbg(dev, "no ANE cluster floor (cpu-map cluster%u)\n", + clpc->hw->ane_cluster); } /* The property is optional: without it, CPU demand alone sets the floors. */ if (clpc->hw->gpu_demand) @@ -1295,6 +1332,7 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { .ane = { [APPLE_CLPC_DCS] = 5, [APPLE_CLPC_SOC] = 2 }, .ane_cluster = 2, .ane_cluster_khz = 3756000, + .ane_cluster_machine = "apple,t6040", }; /* From d3ab97e972b0a88c48e546104599f24a6a53f6d7 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Sun, 8 Nov 2026 21:29:07 -0500 Subject: [PATCH 123/392] dt-bindings: mailbox: Add the physical T6050 GPU profile Describe the ASCWrap v6 compatible for the physical GPU aperture shared by J775D and J716c, distinct from the existing T6050 alias profiles. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/mailbox/apple,ascwrap-v6.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index 289c29ba752fb..a08c3224e7b1e 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -14,7 +14,9 @@ description: wrapper resource contains the coprocessor control registers and mailbox FIFOs. A separate resource provides the coprocessor vector-base register. The firmware role identifies which GPU firmware instance uses the mailbox; - the T6040 runs a single one. + the T6040 runs a single one. The T6050 J775D profile describes the physical + GPU aperture shared by J775D and J716c; the other T6050 profiles describe + alias windows. properties: compatible: @@ -25,6 +27,7 @@ properties: - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 + - apple,t6050-j775d-ascwrap-v6 reg: items: From 174cd0f4c2fe90451130dd7155b7f1614df40267 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Sun, 8 Nov 2026 21:29:07 -0500 Subject: [PATCH 124/392] soc: apple: mailbox: Add the physical T6050 GPU windows The existing T6050 profiles validate alias addresses and reject the physical GFX/GFX1 resources used on J716c. Add the shared J775D/J716c profile for 0x2302600000 and 0x2302e00000, including their vector-base registers. Reuse the ASCWrap v6 transport and retain all existing address profiles. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/soc/apple/mailbox.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index c13c03f9d92ab..999dec63b3b29 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -144,6 +144,11 @@ static bool ascwrap_v8_rx_irq; module_param(ascwrap_v8_rx_irq, bool, 0444); MODULE_PARM_DESC(ascwrap_v8_rx_irq, "Service T8152 GPU ASCWrap v8 receive with the recv-not-empty interrupt (default: poll, no GPU mailbox interrupt is enabled)"); +/* Physical T6050 GPU windows, shared by J775D and J716c. */ +#define APPLE_J775D_GFX_WRAPPER_BASE 0x2302600000ULL +#define APPLE_J775D_GFX_IOP_VBAR_BASE 0x2302050000ULL +#define APPLE_J775D_GFX1_WRAPPER_BASE 0x2302e00000ULL +#define APPLE_J775D_GFX1_IOP_VBAR_BASE 0x2302850000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -1636,8 +1641,7 @@ static const struct apple_mbox_hw apple_mbox_t8152_gfx_ascwrap_v8_hw = { static_assert(0x20 + sizeof(u32) <= APPLE_ASCWRAP_V8_MBOX_SIZE); -/* T6050's native ASCWrap v6 uses the same RUN/stop sequence. Aura grants - * these complete wrapper/VBAR windows in the retained GPU aperture. */ +/* Retained T6050 wrapper/VBAR alias profile. */ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, APPLE_T8140_GFX_WRAPPER_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA, @@ -1647,6 +1651,17 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_ascwrap_v6_ APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA }, }; +/* + * T6050 at its physical addresses, with both roles in the GPU aperture at + * 0x2300000000 on die 0. The other T6050 profiles retain their alias windows. + */ +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_j775d_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_J775D_GFX_WRAPPER_BASE, APPLE_J775D_GFX_IOP_VBAR_BASE }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_J775D_GFX1_WRAPPER_BASE, APPLE_J775D_GFX1_IOP_VBAR_BASE }, +}; + static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_apif_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, APPLE_T8140_GFX_WRAPPER_BASE, @@ -1672,6 +1687,22 @@ static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { .has_irq_controls = false, }; +static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t6050_j775d_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t6050_j775d_ascwrap_v6_roles), + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + .has_irq_controls = false, +}; + /* Retained compatibility profile: unshifted wrapper aliases, native VBAR. */ static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, @@ -1717,6 +1748,8 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, + { .compatible = "apple,t6050-j775d-ascwrap-v6", + .data = &apple_mbox_t6050_j775d_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /* From 8f84cdd51be83496b61ff233c7713fb1cf94737c Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:43:53 +1100 Subject: [PATCH 125/392] drm/apple: take a refused compact call's bare descriptor as its answer A desktop 26.x DCP refuses a service call by answering with the call's descriptor alone: the request's status, and an IOReturn where the length goes. The reply matcher wanted the call header back for every service call, so a refusal matched nothing: it logged "AFK[ep:2b]: unexpected compact reply c0" and the call waited out its full timeout. On the J616s this happens every time an external display powers on. The display audio sink is reported on the audio interface instance the DCP had, PipeWire opens it within some 25 ms, and that instance, which the firmware replaces about 160 ms later, refuses the Open with kIOReturnOffline (0xe00002d7). Captured on the J616s with the Acer QG270 S3 on a USB-C dock, three DPMS cycles: each Open (group 0, method 4, status tag 0xc00) was answered at once with the 8-byte descriptor 00 0c 00 00 d7 02 00 e0. Match a bare descriptor by its status alone and complete the call with the refusal: -ENODEV for an interface that is offline, not ready or not there, -EREMOTEIO for anything else, at debug level. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/afk.c | 45 ++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9ac50fb8b6576..302932f3b31a0 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -960,6 +960,27 @@ static void afk_recv_handle_compact_init(struct apple_dcp_afkep *ep, } } +/* IOReturn values of a refused call (IOKit's sys_iokit error space) */ +#define AFK_IORETURN_NO_DEVICE 0xe00002c0 +#define AFK_IORETURN_OFFLINE 0xe00002d7 +#define AFK_IORETURN_NOT_READY 0xe00002d8 + +/* + * An interface that is offline or not there (yet) is -ENODEV, as a call to + * an interface already gone is; anything else the firmware refused, -EREMOTEIO. + */ +static int afk_compact_refusal_errno(u32 ioreturn) +{ + switch (ioreturn) { + case AFK_IORETURN_NO_DEVICE: + case AFK_IORETURN_OFFLINE: + case AFK_IORETURN_NOT_READY: + return -ENODEV; + default: + return -EREMOTEIO; + } +} + static void afk_recv_handle_compact_reply(struct apple_dcp_afkep *ep, u16 channel, u8 type, const void *payload, @@ -970,6 +991,7 @@ static void afk_recv_handle_compact_reply(struct apple_dcp_afkep *ep, struct afk_compact_cmd *pending, *match = NULL; unsigned long flags; size_t copy_size; + bool refused; if (payload_size < sizeof(*desc)) { dev_err(ep->dcp->dev, @@ -984,13 +1006,22 @@ static void afk_recv_handle_compact_reply(struct apple_dcp_afkep *ep, call = NULL; } + /* + * A call the firmware refuses is answered with its descriptor alone: + * the request's status, and an IOReturn where the length goes (an + * audio interface the firmware is replacing answers an Open with + * kIOReturnOffline). It carries no call header, so it matches by its + * status only. + */ + refused = payload_size == sizeof(*desc); + spin_lock_irqsave(&ep->lock, flags); list_for_each_entry(pending, &ep->compact_cmds, node) { if (pending->channel != channel || pending->type != type) continue; if (pending->request_status != le32_to_cpu(desc->status)) continue; - if (pending->service_call && + if (pending->service_call && !refused && (!call || pending->group != le16_to_cpu(call->group) || pending->command != le32_to_cpu(call->command))) continue; @@ -1018,6 +1049,18 @@ static void afk_recv_handle_compact_reply(struct apple_dcp_afkep *ep, kfree(match); return; } + if (refused && match->service_call) { + u32 ioreturn = le32_to_cpu(desc->length); + + match->retcode = ioreturn; + match->status = afk_compact_refusal_errno(ioreturn); + complete(&match->done); + spin_unlock_irqrestore(&ep->lock, flags); + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: call group %u cmd %u on interface %u refused: %#x\n", + ep->endpoint, match->group, match->command, channel, ioreturn); + return; + } match->retcode = le32_to_cpu(desc->status); match->status = 0; if (call && le32_to_cpu(call->data_len) > From be7df3364d7f7bc20a556855dc198740b5dae68a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:43:53 +1100 Subject: [PATCH 126/392] drm/apple: reopen display audio on the instance that replaces a refused one An Open that the dying audio interface refuses (kIOReturnOffline) now fails at once instead of after its 1 s timeout. The retry looked for the replacement instance right away, but the firmware publishes it some 160 ms later, so the Open would have failed and PipeWire would have dropped the output. On -ENODEV, wait up to 1 s for the next instance to be published, then open that one. After a display wakes up, its audio output is back about 200 ms after the jack turns on instead of a second later. KUnit: apple-av-native-audio gains a refused Open with a successor (reopened on the successor) and without one (fails after the wait). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/av-lifetime-test.c | 3 ++ drivers/gpu/drm/apple/av-v14-test.c | 42 ++++++++++++++++++++++++ drivers/gpu/drm/apple/av.c | 27 ++++++++++++--- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/av-lifetime-test.c b/drivers/gpu/drm/apple/av-lifetime-test.c index 193358ce2bfba..f75be1a5bb3a8 100644 --- a/drivers/gpu/drm/apple/av-lifetime-test.c +++ b/drivers/gpu/drm/apple/av-lifetime-test.c @@ -157,6 +157,9 @@ static struct av_life_test *av_life_setup(struct kunit *test, work_func_t owner_ spin_lock_init(&t->ep->lock); INIT_LIST_HEAD(&t->ep->compact_cmds); init_rwsem(&t->asrv->srv_rwsem); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + init_waitqueue_head(&t->asrv->gen_wq); +#endif INIT_WORK(&t->asrv->start_av_service_wq, owner_fn); INIT_WORK(&t->asrv->terminate_wq, owner_fn); INIT_WORK(&t->dcp->external_audio_work, av_life_owner_fn); diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index f0dbe8f079dd0..f0e5ee63d9d0a 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -30,6 +30,9 @@ struct av_v14_test { u32 link_status; u8 link_body[64]; bool replace_during_prepare; + /* The next Open is refused as offline; with a successor published first */ + bool refuse_open; + bool successor_with_refusal; }; static struct av_v14_test *av_v14_current; @@ -46,6 +49,12 @@ static int av_v14_call(struct apple_epic_service *srv, u32 command, const void * } t->ncalls++; memset(reply, 0, reply_len); + if (command == t->asrv->cmds.open && t->refuse_open) { + t->refuse_open = false; + if (t->successor_with_refusal) + av_v14_replace(t); + return -ENODEV; /* afk: the instance answered kIOReturnOffline */ + } /* Open: status 0 at +0, opened at +16 */ if (command == t->asrv->cmds.open && reply_len >= 24) put_unaligned_le64(1, reply + 16); @@ -133,6 +142,7 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) init_rwsem(&asrv->srv_rwsem); mutex_init(&asrv->plug_lock); mutex_init(&asrv->open_lock); + init_waitqueue_head(&asrv->gen_wq); asrv->cmds = dcp_av_audio_cmds_v13_5; INIT_WORK(&asrv->start_av_service_wq, av_work_service_start); disable_work(&asrv->start_av_service_wq); @@ -234,6 +244,36 @@ static void av_v14_old_terminate_vs_successor(struct kunit *test) KUNIT_EXPECT_EQ(test, t->svc.users, 1U); } +/* + * A display waking up: the Open goes to the instance the firmware is + * replacing, which refuses it as offline; the Open waits for the successor + * and goes to it. + */ +static void av_v14_refused_open_reopens_successor(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + t->refuse_open = true; + t->successor_with_refusal = true; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], t->asrv->cmds.open); + KUNIT_EXPECT_NE(test, t->call_gens[1], t->call_gens[0]); + KUNIT_EXPECT_EQ(test, t->asrv->open_gen, t->asrv->gen); + KUNIT_EXPECT_PTR_EQ(test, t->asrv->open_srv, &t->svc); +} + +/* Refused, and no successor comes: the Open fails once its wait is over. */ +static void av_v14_refused_open_without_successor(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + t->refuse_open = true; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_NULL(test, t->asrv->open_srv); +} + /* An instance published over the open one, without a teardown, is opened too. */ static void av_v14_republished(struct kunit *test) { @@ -490,6 +530,8 @@ static struct kunit_case av_v14_cases[] = { KUNIT_CASE_PARAM(av_v14_old_close_not_to_successor, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_old_terminate_vs_successor, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_republished, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_refused_open_reopens_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_refused_open_without_successor, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_successor_during_release, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_terminate_after_successor, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_link_instance, av_native_gen_params), diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 550e695e2485c..f32d9343439c5 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -92,6 +92,8 @@ struct audiosrv_data { struct mutex open_lock; struct apple_epic_service *open_srv; u64 open_gen; + /* Woken when gen changes: an Open waits there for a replacement. */ + wait_queue_head_t gen_wq; /* The instance the link was last prepared on (open_lock) */ u64 link_gen; #endif @@ -155,6 +157,9 @@ static void av_audiosrv_init(struct apple_epic_service *service, const char *nam #endif queue_start = !asrv->reports_stopped; up_write(&asrv->srv_rwsem); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + wake_up_all(&asrv->gen_wq); +#endif if (old) afk_service_put(old); @@ -280,6 +285,9 @@ static int av_audiosrv_call(struct apple_epic_service *srv, u32 command, const v return afk_service_call(srv, 0, command, body, body_len, body_pad, reply, reply_len, 0); } +/* How long an Open refused by an instance being replaced waits for the next one */ +#define AV_AUDIOSRV_REPLACE_MS 1000 + /* * The 14.7 external role: the service is open while a PCM is (see * av_service_connect()). The instance the PCM opened is kept under open_lock, @@ -308,11 +316,11 @@ int dcp_audiosrv_open(struct device *dev) * A display that wakes up (DPMS on) reports its sink on the audio * interface it had, and the firmware replaces that instance a few * hundred milliseconds later: an Open sent in between is refused - * with a reply that matches no call, and times out. The Open that - * follows the replacement is answered, so try the instance published - * by then once more. Userspace drops an output whose first open - * fails, so this is the difference between a display with and - * without audio after it wakes up. + * (kIOReturnOffline). The Open that follows the replacement is + * answered, so wait for that instance and try it once more. + * Userspace drops an output whose first open fails, so this is the + * difference between a display with and without audio after it + * wakes up. */ for (int attempt = 0; ; attempt++) { scoped_guard(rwsem_read, &asrv->srv_rwsem) { @@ -344,6 +352,14 @@ int dcp_audiosrv_open(struct device *dev) afk_service_put(srv); if (attempt) return ret; + /* + * The instance refused it as offline: its successor is on the + * way (published within some 200 ms on the J616s), wait for it. + */ + if (ret == -ENODEV) + wait_event_timeout(asrv->gen_wq, + READ_ONCE(asrv->srv) && READ_ONCE(asrv->gen) != gen, + msecs_to_jiffies(AV_AUDIOSRV_REPLACE_MS)); scoped_guard(rwsem_read, &asrv->srv_rwsem) replaced = asrv->srv && asrv->gen != gen; if (!replaced) @@ -1176,6 +1192,7 @@ int avep_init(struct apple_dcp *dcp) #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) INIT_WORK(&audiosrv_data->terminate_wq, av_work_service_terminate); mutex_init(&audiosrv_data->open_lock); + init_waitqueue_head(&audiosrv_data->gen_wq); #endif audiosrv_data->works_live = true; dcp->audiosrv = audiosrv_data; From 11319ea8e3740fc4dab909b9b1e9437807277aa4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:43:53 +1100 Subject: [PATCH 127/392] drm/apple: update the display audio ELD when the sink's EDID arrives The firmware reports a newly attached sink before the DCP has read its EDID, and reports it again once the mode is set. The card took the ELD only from the first report, so a display plugged in after boot, or one moved to another display processor, kept an ELD with no monitor name and the basic-audio fallback: on the J616s, the Acer QG270 S3 moved to DCPEXT1 showed up as an unnamed HDMI sink while the same display on DCPEXT0 had its name. When a connected sink is reported again with a different ELD, store it and notify the ELD control, as HDMI codecs do when the ELD changes. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/apple/audio.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 64496a827c776..de8fc565a11dd 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -1305,10 +1305,26 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected, { struct snd_pcm_substream *substream = dcpaud->substream; - if (!dcpaud->card || dcpaud->connected == connected) { + if (!dcpaud->card) { mutex_unlock(&dcpaud->data_lock); return; } + /* + * A sink the firmware reports before its EDID is read (a display + * plugged in, or brought up on another processor) is connected again + * once the mode is set: its ELD then has the monitor name. + */ + if (dcpaud->connected == connected) { + bool changed = connected && eld && memcmp(dcpaud->eld, eld, DCPAUD_ELD_SIZE); + + if (changed) + memcpy(dcpaud->eld, eld, DCPAUD_ELD_SIZE); + mutex_unlock(&dcpaud->data_lock); + if (changed) + snd_ctl_notify(dcpaud->card, SNDRV_CTL_EVENT_MASK_VALUE, + &dcpaud->eld_ctl->id); + return; + } dcpaud->connected = connected; if (connected) { From d95db69304a5d405989721972191e8fb8208d41c Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:38:36 +1100 Subject: [PATCH 128/392] accel/ane: wake H16 firmware waits on the channel IRQ Every H16 wait polls the rings: 2 us busy steps for the first 30 us, then 20-50 us sleeps. On the M4 Pro an EXECUTE transport reply arrives about 90 us after the doorbell, so it lands inside a sleep and is seen up to 50 us late; the completion report of a MobileNet request is quantised the same way. Busy-polling instead does not help: polling the engine's mailbox and fault registers every 2 us while the request runs slows the engine itself (MobileNet report->done 88 -> 158 us). macOS takes the same replies from the channel interrupt. Do the same once the firmware is up: before sleeping, the wait enables the channel IRQ, and the handler disables it again and wakes the wait. The transport still reads, acknowledges and drains every ring from the waiting thread, exactly as before; the IRQ only ends the sleep early, and the old sleep length stays as the timeout. The report wait drops the session lock around its pause, so the waiting half of relax() becomes its own transport op, pause(), which needs no lock; waits without it keep ane_fw_poll_pause(). The IRQ stays off during the firmware startup handshake (at boot and on the cold restart after system sleep), so that sequence keeps its polled timing, and only platforms whose IRQ drains the channels (ANE_FW_IRQ_CHANNELS, H16) use it. debugfs ane-h16//irq_wakeups and irq_timeouts count how each sleep ended. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/accel/ane/ane_fw_transport.h | 6 ++ drivers/accel/ane/ane_h16_backend.c | 112 ++++++++++++++++++++++++--- drivers/accel/ane/ane_h16_program.c | 5 +- 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/drivers/accel/ane/ane_fw_transport.h b/drivers/accel/ane/ane_fw_transport.h index d327118204120..527dff351aabf 100644 --- a/drivers/accel/ane/ane_fw_transport.h +++ b/drivers/accel/ane/ane_fw_transport.h @@ -92,6 +92,12 @@ struct ane_fw_transport_ops { * their lock around the pause call it first, with the lock held. */ void (*service)(void *ctx); + /* + * Optional: the waiting half of relax(), safe without the caller's + * lock (it may end early on the channel IRQ). Default + * ane_fw_poll_pause(). + */ + void (*pause)(void *ctx, u64 waited_ns); int (*alloc)(void *ctx, size_t size, u32 tag, struct ane_fw_surface *surface); void (*refused)(void *ctx, u64 word0, u64 word1, u64 word2, int error); /* Optional: a shared allocation was freed ("FREE") or served again ("reuse"). */ diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index eb1de39a24f09..010dca71d8c07 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include #include #include "ane_fw_sleep.h" @@ -66,6 +68,19 @@ struct ane_h16_device { /* While pinned: refuse system sleep (ane_fw_sleep.h). */ struct notifier_block pm_notifier; int irq; + /* + * The channel IRQ (ANE_FW_IRQ_CHANNELS) as a wakeup for the polled + * waits once the firmware is up: relax() arms it before sleeping and + * the handler disables it again. The transport still reads, + * acknowledges and drains every ring itself, as before. + */ + spinlock_t irq_lock; + wait_queue_head_t irq_wq; + bool irq_wait; + bool irq_armed; + bool irq_fired; + u64 irq_wakeups; + u64 irq_timeouts; struct ane_fw_rtk rtk; struct { u64 word0; @@ -188,6 +203,28 @@ static void ane_h16_quarantine(void *ctx, int error) static void ane_h16_busy(void *ctx, bool busy); static void ane_h16_sleep_trace(struct ane_h16_device *h, const char *when); +/* + * Let the channel IRQ wake the waits, or go back to pure polling. Off + * across power down and the firmware startup that follows it, so the + * startup handshake keeps its proven polled timing. + */ +static void ane_h16_irq_wait_set(struct ane_h16_device *h, bool on) +{ + bool disarm = false; + + if (h->irq < 0 || h->boot.soc->irq_role != ANE_FW_IRQ_CHANNELS) + return; + scoped_guard(spinlock_irqsave, &h->irq_lock) { + WRITE_ONCE(h->irq_wait, on); + if (!on && h->irq_armed) { + h->irq_armed = false; + disarm = true; + } + } + if (disarm) + disable_irq(h->irq); +} + static int ane_h16_suspend_firmware(struct ane_h16_device *h) { struct ane_fw_transport *t = &h->boot.transport; @@ -263,6 +300,7 @@ static void ane_h16_power_down(struct ane_h16_device *h) unsigned int i; ane_h16_busy(h, false); + ane_h16_irq_wait_set(h, false); for (i = ARRAY_SIZE(h->dart); i-- > 0; ) pm_runtime_put_sync(h->dart[i]); for (i = h->powered; i-- > 0; ) @@ -503,6 +541,7 @@ static int ane_h16_cold_boot(struct ane_h16_device *h) if (ret) return ret; h->resume_ready_ns = ktime_get_ns(); + ane_h16_irq_wait_set(h, true); dev_info(h->dev, "H16 ANE: booted again from the preload after system sleep\n"); ane_h16_session_restart(&h->session); ret = ane_h16_session_replay(&h->session, &h->resume_replayed, &h->resume_first_ns); @@ -975,11 +1014,50 @@ static u64 ane_h16_now(void *ctx) } /* - * The channel IRQ stays masked, so every wait polls, with the shared pacing - * (see ane_fw_poll_sleep_us()): busy steps for the first 30 us, then short - * sleeps. A fixed 1 ms sleep made each of the four waits of an EXECUTE cost - * at least a millisecond. + * Every wait polls, with the shared pacing (see ane_fw_poll_sleep_us()): + * busy steps for the first 30 us, then short sleeps. A fixed 1 ms sleep + * made each of the four waits of an EXECUTE cost at least a millisecond. + * + * After the busy window an EXECUTE reply (typically 40-50 us) landed in a + * 20-50 us sleep and was seen 40-50 us late. Once the firmware is up, the + * sleep therefore also ends on the channel IRQ, the way macOS drains the + * rings from IRQ 1054; the timeout keeps the polled cadence as a bound. */ +static void ane_h16_irq_pause(struct ane_h16_device *h, u64 waited_ns) +{ + unsigned int min_us, max_us; + long ret; + + min_us = ane_fw_poll_sleep_us(waited_ns, &max_us); + scoped_guard(spinlock_irqsave, &h->irq_lock) { + h->irq_fired = false; + if (!h->irq_armed) { + h->irq_armed = true; + enable_irq(h->irq); + } + } + ret = wait_event_hrtimeout(h->irq_wq, READ_ONCE(h->irq_fired), + us_to_ktime(min_us)); + scoped_guard(spinlock_irqsave, &h->irq_lock) { + if (ret) + h->irq_timeouts++; + else + h->irq_wakeups++; + } +} + +/* The waiting half of relax(); callers may hold no lock. */ +static void ane_h16_pause(void *ctx, u64 waited_ns) +{ + struct ane_h16_device *h = ctx; + + if (READ_ONCE(h->irq_wait) && waited_ns >= ANE_FW_POLL_SPIN_NS && + !READ_ONCE(h->boot.transport.error)) + ane_h16_irq_pause(h, waited_ns); + else + ane_fw_poll_pause(waited_ns); +} + static void ane_h16_service(void *ctx) { struct ane_h16_device *h = ctx; @@ -994,7 +1072,7 @@ static void ane_h16_service(void *ctx) static void ane_h16_relax(void *ctx, u64 waited_ns) { ane_h16_service(ctx); - ane_fw_poll_pause(waited_ns); + ane_h16_pause(ctx, waited_ns); } static int ane_h16_alloc_shared(void *ctx, size_t size, u32 tag, struct ane_fw_surface *out) @@ -1114,6 +1192,7 @@ static const struct ane_fw_transport_ops ane_h16_transport_ops = { .write32 = ane_h16_engine_write, .now_ns = ane_h16_now, .relax = ane_h16_relax, + .pause = ane_h16_pause, .service = ane_h16_service, .alloc = ane_h16_alloc_shared, .refused = ane_h16_refused, @@ -1189,9 +1268,20 @@ static void ane_h16_release(struct ane_h16_device *h) dev_info(h->dev, "H16 pre-RUN rollback complete; no CPU release\n"); } -static irqreturn_t ane_h16_masked_irq(int irq, void *data) +/* Wake the sleeping wait and stay disabled until relax() arms it again. */ +static irqreturn_t ane_h16_irq(int irq, void *data) { - return IRQ_NONE; + struct ane_h16_device *h = data; + + scoped_guard(spinlock_irqsave, &h->irq_lock) { + if (h->irq_armed) { + disable_irq_nosync(irq); + h->irq_armed = false; + } + WRITE_ONCE(h->irq_fired, true); + } + wake_up(&h->irq_wq); + return IRQ_HANDLED; } /* @@ -1357,11 +1447,10 @@ static int ane_h16_acquire(struct platform_device *pdev, struct ane_h16_device * irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; - ret = request_irq(irq, ane_h16_masked_irq, IRQF_NO_AUTOEN, "ane-h16-polled", h); + ret = request_irq(irq, ane_h16_irq, IRQF_NO_AUTOEN, "ane-h16", h); if (ret) return ret; h->irq = irq; - dev_info(h->dev, "H16 polled mode; channel IRQ remains masked\n"); return 0; } @@ -1473,6 +1562,7 @@ static int ane_h16_program_stage(struct ane_h16_device *h) return ret; } h->program_ready = true; + ane_h16_irq_wait_set(h, true); dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_next, h->program_end - 1, h->clpc ? ", CLPC input attached" : ""); return 0; @@ -1669,6 +1759,8 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat } h->dev = &pdev->dev; h->irq = -1; + spin_lock_init(&h->irq_lock); + init_waitqueue_head(&h->irq_wq); INIT_LIST_HEAD(&h->mappings); h->boot.soc = soc; /* The build, and with it the DVA layout, is chosen from the preload. */ @@ -1695,6 +1787,8 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat h->debugfs = debugfs_create_dir(dev_name(h->dev), ane_h16_debugfs); debugfs_create_file("stage", 0600, h->debugfs, h, &ane_h16_stage_fops); debugfs_create_file("execute", 0600, h->debugfs, h, &ane_h16_execute_fops); + debugfs_create_u64("irq_wakeups", 0400, h->debugfs, &h->irq_wakeups); + debugfs_create_u64("irq_timeouts", 0400, h->debugfs, &h->irq_timeouts); platform_set_drvdata(pdev, h); INIT_WORK(&h->start, ane_h16_start_work); schedule_work(&h->start); diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index eb860a3d4799d..7a47f842e7b7d 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -1160,7 +1160,10 @@ static int h16_wait_report(struct ane_h16_session *s, struct ane_h16_pending *re if (s->lock && t->ops->service) { t->ops->service(t->ctx); mutex_unlock(s->lock); - ane_fw_poll_pause(now - start); + if (t->ops->pause) + t->ops->pause(t->ctx, now - start); + else + ane_fw_poll_pause(now - start); mutex_lock(s->lock); } else if (s->lock) { mutex_unlock(s->lock); From aad9099efc7486e10e2633d43d12111142870e1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 13:57:12 +1100 Subject: [PATCH 129/392] ASoC: apple: macaudio: Serve both speaker protection interfaces The tree carries only the "Speaker Protection" lease, which a protection engine in the signal path (speakerguardd v2) owns. The distribution kernels revert it to get "Speaker Volume Unlock" back, the volume lock that speakersafetyd and speakerguardd 0.1 use, so the fleet and distribution kernels differ, and the lease cannot return on the boards it is qualified for without locking every other Mac out. Carry both. "Speaker Protection Interface" says which one owns the speakers: - "Speaker Volume Unlock", at every card bind: the volume lock exactly as on a lock-only kernel ("Speaker Sample Rate", the ELEM_LOCK and ping lease, its ceiling, the system-sleep handling, the J700 software-gain epoch, "Amp Gain Volume" limited but the amp's to set). An owner may hold "Speaker Protection", but nothing is granted, and its locks and releases touch no amp. - "Speaker Protection", after a CAP_SYS_ADMIN write only. The handover passes through both floors: the lock's ceiling first, then no speaker start and the running streams stopped, the lease's safe level, every amp gain fixed at its limit, and only then the ceiling lifted with every volume still at the safe level. While leased the lock's ping fails with -EBUSY and userspace writes to the speaker volumes and amp gains with -EPERM. Back to "Speaker Volume Unlock" by a write, or after 30 s without an owner of "Speaker Protection" (the engine died, or never came): the lease's floor, then the lock's ceiling over the safe level. The speakers go up only once a daemon holds the lock and pings it, as after a cold bind, and a daemon still holding it learns once, with -ETIMEDOUT, that it lost it. An engine that restarts within the grace keeps the lease. The switch is state for the boot: INACTIVE keeps alsactl off it, and whatever makes it persist writes it again at every boot. On the J700 the software gain's table top follows the interface through a new register of the AOP component, which no control exposes: half the amplifier's full scale under the lock, the scale its daemon was calibrated against, and the full scale only while leased. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/aop_t8140.rs | 91 +- sound/soc/apple/macaudio-iface-test.h | 264 +++ sound/soc/apple/macaudio-pm-volumes-test.h | 268 ++++ sound/soc/apple/macaudio-protection-test.h | 67 +- sound/soc/apple/macaudio.c | 1428 +++++++++++++++-- .../selftests/alsa/macaudio-fixed-controls.py | 15 +- 6 files changed, 1993 insertions(+), 140 deletions(-) create mode 100644 sound/soc/apple/macaudio-iface-test.h create mode 100644 sound/soc/apple/macaudio-pm-volumes-test.h diff --git a/sound/soc/apple/aop_t8140.rs b/sound/soc/apple/aop_t8140.rs index ff3bd53591276..aa97deda3b5c3 100644 --- a/sound/soc/apple/aop_t8140.rs +++ b/sound/soc/apple/aop_t8140.rs @@ -27,8 +27,10 @@ //! //! The speaker front-end is copy-only: the LEAP consumes float32 and the //! driver converts each S32 sample through the amplifier's full scale and -//! the "Speaker Playback Volume" control, which macaudio holds at full scale -//! while a protection engine owns the speakers and 20 dB down otherwise. +//! the "Speaker Playback Volume" control, below one of two ceilings that +//! macaudio selects with the protection interface: half the amplifier's full +//! scale under the volume lock of a protection daemon, the full scale under +//! the lease of a protection engine in the signal path. //! //! Closing the lid disconnects both internal microphones in hardware. The //! array then reads back as digital silence, but the low-power microphone's @@ -492,6 +494,9 @@ struct SndSocT8140AopData { hpai_want_run: AtomicFlag, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, + /// REG_SPKR_FULL_SCALE: the volume is below SPKR_CEILING_Q31, not the + /// legacy ceiling. + spkr_full_scale: AtomicFlag, /// The sleeping firmware starts queued from the atomic triggers. #[pin] spkr_start_work: Work, @@ -1614,9 +1619,10 @@ const FE_DMA_NAMES: [[Option<&core::ffi::CStr>; 2]; FE_COUNT] = [ /// "Secondary" PCM): its data goes through `spkr_copy` below. const FE_SPKR: usize = 1; -/// Digital ceiling of the speaker path, as a Q31 gain applied to every -/// sample userspace writes: unity, so a full-scale S32 sample is a float of -/// 1.0, the amplifier's full scale. +/// Digital ceiling of the speaker path under the lease of a protection engine +/// ("Speaker Protection"), as a Q31 gain applied to every sample userspace +/// writes: unity, so a full-scale S32 sample is a float of 1.0, the +/// amplifier's full scale. /// /// The LEAP consumes float32 with 1.0 as the I2S full scale. Measured /// from outside the machine, the same program through the same speaker @@ -1636,6 +1642,13 @@ const FE_SPKR: usize = 1; /// down with speaker playback refused. const SPKR_CEILING_Q31: i64 = 1 << 31; +/// Digital ceiling under the volume lock of a protection daemon ("Speaker +/// Volume Unlock", speakerguardd 0.1): 0.5, the scale that daemon was +/// calibrated against, unchanged from the kernels that had only that +/// interface. It is the default: macaudio selects the full scale only +/// while the lease is the interface (REG_SPKR_FULL_SCALE). +const SPKR_LEGACY_CEILING_Q31: i64 = 1 << 30; + /// "Speaker Playback Volume": the software gain below the ceiling that /// macaudio sets (full scale or its safe level), 0.5 dB per /// step from -63.0 dB (1) to 0 dB (127) relative to SPKR_CEILING_Q31, 0 = @@ -1659,8 +1672,33 @@ const SPKR_VOLUME_Q31: [u32; 128] = [ 1140065663, 1207618800, 1279174713, 1354970580, 1435257634, 1520301996, 1610385552, 1705806895, 1806882308, 1913946816, 2027355295, 2147483648, ]; -/// The component's only "register": the speaker volume. +/// The volume table under the volume lock: entry i = +/// round(SPKR_LEGACY_CEILING_Q31 * 10^(-(127 - i) / 40)). +const _: () = assert!( + SPKR_LEGACY_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64 == SPKR_LEGACY_CEILING_Q31 +); +const SPKR_LEGACY_VOLUME_Q31: [u32; 128] = [ + 0, 760151, 805193, 852903, 903441, 956973, 1013678, 1073742, 1137365, 1204758, 1276145, + 1351761, 1431858, 1516701, 1606571, 1701766, 1802602, 1909413, 2022553, 2142397, 2269342, + 2403809, 2546243, 2697118, 2856932, 3026216, 3205530, 3395470, 3596664, 3809780, 4035523, + 4274643, 4527932, 4796229, 5080423, 5381457, 5700328, 6038094, 6395874, 6774853, 7176288, + 7601510, 8051928, 8529034, 9034412, 9569734, 10136776, 10737418, 11373650, 12047581, 12761445, + 13517609, 14318577, 15167006, 16065708, 17017661, 18026021, 19094130, 20225528, 21423966, + 22693416, 24038085, 25462431, 26971175, 28569318, 30262156, 32055302, 33954698, 35966640, + 38097798, 40355234, 42746432, 45279317, 47962285, 50804230, 53814569, 57003283, 60380940, + 63958736, 67748529, 71762882, 76015100, 80519278, 85290345, 90344115, 95697341, 101367765, + 107374182, 113736503, 120475814, 127614455, 135176087, 143185773, 151670064, 160657080, + 170176611, 180260209, 190941298, 202255281, 214239660, 226934158, 240380852, 254624313, + 269711752, 285693178, 302621563, 320553018, 339546978, 359666402, 380977976, 403552340, + 427464319, 452793173, 479622855, 508042296, 538145694, 570032831, 603809400, 639587356, + 677485290, 717628817, 760150998, 805192776, 852903448, 903441154, 956973408, 1013677647, + 1073741824, +]; +/// The component's "registers": the speaker volume, and which ceiling it +/// is below (0: SPKR_LEGACY_CEILING_Q31, 1: SPKR_CEILING_Q31), which +/// macaudio writes (MACAUDIO_SW_GAIN_REG_FULL_SCALE) and no control exposes. const REG_SPKR_VOLUME: u32 = 0; +const REG_SPKR_FULL_SCALE: u32 = 1; /// DECLARE_TLV_DB_SCALE(-6350, 50, 1): a dB scale from -63.50 dB in 0.50 dB /// steps whose lowest value mutes. static SPKR_VOLUME_TLV: [u32; 4] = [ @@ -1972,6 +2010,15 @@ fn f32_bits_of_q31(v: i32) -> u32 { /// One S32 sample -> the float32 word the LEAP consumes, through the volume /// table (whose top entry is the ceiling). +/// The Q31 gain of @volume below the selected ceiling. +fn spkr_gain_q31(full_scale: bool, volume: usize) -> u32 { + if full_scale { + SPKR_VOLUME_Q31[volume] + } else { + SPKR_LEGACY_VOLUME_Q31[volume] + } +} + fn spkr_f32_bits(sample: i32, gain_q31: u32) -> u32 { let scaled = (i64::from(sample) * i64::from(gain_q31)) >> 31; // |scaled| <= SPKR_CEILING_Q31 f32_bits_of_q31(scaled as i32) @@ -2038,7 +2085,7 @@ unsafe extern "C" fn asoc_pcm_copy( // throughout registration, use and unregister. let data = unsafe { component_data(component) }; let volume = core::cmp::min(data.spkr_volume.load(Relaxed), SPKR_VOLUME_MAX) as usize; - let gain = SPKR_VOLUME_Q31[volume]; + let gain = spkr_gain_q31(data.spkr_full_scale.load(Relaxed), volume); let mut chunk = [0u8; 512]; let mut off = 0; while off < bytes { @@ -2135,6 +2182,7 @@ unsafe extern "C" fn asoc_component_read( let data = unsafe { component_data(component) }; match reg { REG_SPKR_VOLUME => data.spkr_volume.load(Relaxed), + REG_SPKR_FULL_SCALE => u32::from(data.spkr_full_scale.load(Relaxed)), _ => 0, } } @@ -2157,6 +2205,10 @@ unsafe extern "C" fn asoc_component_write( .store(core::cmp::min(val, SPKR_VOLUME_MAX), Relaxed); 0 } + REG_SPKR_FULL_SCALE => { + data.spkr_full_scale.store(val != 0, Relaxed); + 0 + } _ => EINVAL.to_errno(), } } @@ -2889,6 +2941,7 @@ impl SndSocT8140AopDriver { spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), + spkr_full_scale: AtomicFlag::new(false), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), hpai_start_work <- new_work!("SndSocT8140AopData::hpai_start_work"), mic, @@ -2929,6 +2982,30 @@ mod tests { /// 1.0 as a float32: the amplifier's full scale on the wire. const F32_ONE: u32 = 0x3f80_0000; + /// Under the volume lock the table is the one the lock-only kernels had: + /// half the full scale at the top, every entry within one LSB of half + /// the lease's, and 20 dB below it at the locked level. + #[test] + fn the_volume_lock_keeps_the_legacy_ceiling() { + assert_eq!(SPKR_LEGACY_CEILING_Q31, 1 << 30); + assert_eq!( + spkr_gain_q31(false, SPKR_VOLUME_MAX as usize) as i64, + SPKR_LEGACY_CEILING_Q31 + ); + assert_eq!( + spkr_gain_q31(true, SPKR_VOLUME_MAX as usize) as i64, + SPKR_CEILING_Q31 + ); + assert_eq!(f32_bits_of_q31(SPKR_LEGACY_CEILING_Q31 as i32), 0x3f00_0000); + for i in 0..=SPKR_VOLUME_MAX as usize { + let half = i64::from(SPKR_VOLUME_Q31[i]) / 2; + assert!((i64::from(SPKR_LEGACY_VOLUME_Q31[i]) - half).abs() <= 1); + } + // 0.05 as a float32 is 0x3d4c_cccd; the conversion truncates + let locked = spkr_f32_bits(i32::MIN, spkr_gain_q31(false, LOCKED_VOLUME)); + assert!(locked & 0x7fff_ffff <= 0x3d4c_cccd && locked & 0x7fff_ffff > 0x3d4c_cccd - 64); + } + #[test] fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { assert_eq!( diff --git a/sound/soc/apple/macaudio-iface-test.h b/sound/soc/apple/macaudio-iface-test.h new file mode 100644 index 0000000000000..83155d80b04d9 --- /dev/null +++ b/sound/soc/apple/macaudio-iface-test.h @@ -0,0 +1,264 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * "Speaker Protection Interface": the volume lock in charge from the bind, + * the switch to the lease and back through the floor of both, and the way + * back when the lease has no owner. On the card of fake controls that + * macaudio-protection-test.h builds, with the volume lock's controls added. + */ +#include + +struct macaudio_test_iface { + struct macaudio_test_spk t; + struct snd_kcontrol *slk; + struct snd_kcontrol *iface; + /* The legacy daemon's control file */ + struct snd_ctl_file *daemon; +}; + +/* A bound card with both interfaces: the volume lock in charge, at its ceiling */ +static void macaudio_test_iface_card(struct kunit *test, struct macaudio_test_iface *f) +{ + struct snd_kcontrol_new slk = macaudio_test_control("Speaker Volume Unlock"); + struct snd_kcontrol_new iface = macaudio_test_control("Speaker Protection Interface"); + struct macaudio_snd_data *ma; + unsigned int i; + + macaudio_test_spk_card(test, &f->t, &macaudio_j316_cfg, SN012776_0DB); + ma = f->t.ma; + f->daemon = kunit_kzalloc(test, sizeof(*f->daemon), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f->daemon); + KUNIT_ASSERT_NOT_NULL(test, slk.name); + KUNIT_ASSERT_NOT_NULL(test, iface.name); + f->slk = macaudio_test_add_kctl(test, ma->card.snd_card, &slk, &ma->card); + f->iface = macaudio_test_add_kctl(test, ma->card.snd_card, &iface, &ma->card); + ma->speaker_lock_kctl = f->slk; + ma->spk_iface_kctl = f->iface; + ma->has_safety = true; + + /* As macaudio_late_probe() does at every bind */ + for (i = 0; i < MACAUDIO_TEST_NR_SPK; i++) + macaudio_test_vol(f->t.vols[i])->value = MACAUDIO_TEST_START_VOL; + ma->spk_iface = MACAUDIO_SPK_IFACE_UNLOCK; + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, false); + mutex_unlock(&ma->volume_lock_mutex); +} + +static int macaudio_test_iface_write(struct kunit *test, struct macaudio_test_iface *f, + unsigned int item) +{ + struct snd_ctl_elem_value *uval = kunit_kzalloc(test, sizeof(*uval), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, uval); + uval->value.enumerated.item[0] = item; + return macaudio_spk_iface_put(f->iface, uval); +} + +/* The legacy daemon takes the lock and every speaker volume */ +static void macaudio_test_daemon_take(struct kunit *test, struct macaudio_test_iface *f) +{ + unsigned int i; + + for (i = 0; i < MACAUDIO_TEST_NR_SPK; i++) + f->t.vols[i]->vd[0].owner = f->daemon; + KUNIT_ASSERT_EQ(test, macaudio_slk_lock(f->slk, f->daemon), 0); +} + +static int macaudio_test_daemon_ping(struct kunit *test, struct macaudio_test_iface *f) +{ + struct snd_ctl_elem_value *uval = kunit_kzalloc(test, sizeof(*uval), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, uval); + uval->value.integer.value[0] = SPEAKER_MAGIC_VALUE; + return macaudio_slk_put(f->slk, uval); +} + +/* Every speaker volume's value and limit */ +static void macaudio_test_expect_limit(struct kunit *test, struct macaudio_test_iface *f, + long value, int limit) +{ + unsigned int i; + + for (i = 0; i < MACAUDIO_TEST_NR_SPK; i++) { + KUNIT_EXPECT_EQ(test, macaudio_test_vol(f->t.vols[i])->value, value); + KUNIT_EXPECT_EQ(test, macaudio_test_vol(f->t.vols[i])->mc.platform_max, limit); + } +} + +/* At the bind: the volume lock owns the speakers and the lease grants nothing */ +static void macaudio_iface_volume_lock_from_binding(struct kunit *test) +{ + struct macaudio_test_iface f; + + macaudio_test_iface_card(test, &f); + KUNIT_EXPECT_FALSE(test, macaudio_spk_leased(f.t.ma)); + /* The lock's ceiling, 1 on the SN012776, clips what was there */ + macaudio_test_expect_limit(test, &f, 1, 1); + + /* A lease owner may hold its control, but takes nothing */ + macaudio_test_take(test, &f.t); + macaudio_test_expect_limit(test, &f, 1, 1); + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 50), -EBUSY); + KUNIT_EXPECT_EQ(test, macaudio_test_read(test, &f.t), 0L); + macaudio_spk_unlock(f.t.spk); + macaudio_test_expect_limit(test, &f, 1, 1); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&f.t.ma->spk_unowned_work)); + + /* Speaker volumes are the daemon's to write */ + KUNIT_EXPECT_EQ(test, f.t.vols[0]->write_check(f.t.vols[0], NULL), 0); + + /* The daemon unlocks them as on a lock-only kernel */ + macaudio_test_daemon_take(test, &f); + KUNIT_EXPECT_EQ(test, macaudio_test_daemon_ping(test, &f), 0); + KUNIT_EXPECT_TRUE(test, f.t.ma->speaker_volume_unlocked); + macaudio_test_expect_limit(test, &f, 1, SN012776_0DB); +} + +/* Into the lease: the lock's ceiling, then the lease's floor, then the lift */ +static void macaudio_iface_switch_goes_through_the_floor(struct kunit *test) +{ + struct macaudio_test_iface f; + unsigned int i; + + macaudio_test_iface_card(test, &f); + macaudio_test_daemon_take(test, &f); + KUNIT_EXPECT_EQ(test, macaudio_test_daemon_ping(test, &f), 0); + for (i = 0; i < MACAUDIO_TEST_NR_SPK; i++) + macaudio_test_vol(f.t.vols[i])->value = 180; + + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_LEASE), 1); + KUNIT_EXPECT_TRUE(test, macaudio_spk_leased(f.t.ma)); + KUNIT_EXPECT_FALSE(test, f.t.ma->speaker_volume_unlocked); + /* At the lease's floor, below a ceiling that no longer applies */ + macaudio_test_expect_limit(test, &f, 0, SN012776_0DB); + KUNIT_EXPECT_FALSE(test, macaudio_spk_admitted(f.t.ma)); + + /* The volume lock is refused, and so are the daemon's volume writes */ + KUNIT_EXPECT_EQ(test, macaudio_test_daemon_ping(test, &f), -EBUSY); + KUNIT_EXPECT_EQ(test, f.t.vols[0]->write_check(f.t.vols[0], NULL), -EPERM); + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_LEASE), 0); + + /* The engine is granted its operating gain */ + macaudio_test_take(test, &f.t); + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 50), 0); + KUNIT_EXPECT_TRUE(test, macaudio_spk_admitted(f.t.ma)); + macaudio_test_expect_limit(test, &f, SN012776_0DB, SN012776_0DB); + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 0), 0); + macaudio_test_settle(f.t.ma); +} + +/* + * Back to the volume lock: the speakers come back at the floor under the + * lock's ceiling, and go up only once the daemon pings, as after a cold + * bind; its first ping learns that it lost the lock. + */ +static void macaudio_iface_revert_starts_at_the_floor(struct kunit *test) +{ + struct macaudio_test_iface f; + + macaudio_test_iface_card(test, &f); + macaudio_test_daemon_take(test, &f); + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_LEASE), 1); + macaudio_test_take(test, &f.t); + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 50), 0); + macaudio_test_expect_limit(test, &f, SN012776_0DB, SN012776_0DB); + + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_UNLOCK), 1); + macaudio_test_settle(f.t.ma); + KUNIT_EXPECT_FALSE(test, macaudio_spk_leased(f.t.ma)); + KUNIT_EXPECT_FALSE(test, macaudio_spk_admitted(f.t.ma)); + KUNIT_EXPECT_FALSE(test, f.t.ma->speaker_volume_unlocked); + macaudio_test_expect_limit(test, &f, 0, 1); + /* The lease grants nothing any more */ + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 50), -EBUSY); + macaudio_test_expect_limit(test, &f, 0, 1); + + /* The daemon's lock was lost: told once, then it unlocks */ + KUNIT_EXPECT_EQ(test, macaudio_test_daemon_ping(test, &f), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, f.t.ma->speaker_volume_unlocked); + macaudio_test_expect_limit(test, &f, 0, 1); + KUNIT_EXPECT_EQ(test, macaudio_test_daemon_ping(test, &f), 0); + KUNIT_EXPECT_TRUE(test, f.t.ma->speaker_volume_unlocked); + /* Unlocking lifts the ceiling only: the daemon sets the volumes itself */ + macaudio_test_expect_limit(test, &f, 0, SN012776_0DB); +} + +/* The engine dies: the lease's floor at once, the volume lock after the grace */ +static void macaudio_iface_unowned_lease_reverts(struct kunit *test) +{ + struct macaudio_test_iface f; + + macaudio_test_iface_card(test, &f); + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_LEASE), 1); + /* Entered without an owner: the grace runs already */ + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&f.t.ma->spk_unowned_work)); + macaudio_test_take(test, &f.t); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&f.t.ma->spk_unowned_work)); + KUNIT_EXPECT_EQ(test, macaudio_test_write(test, &f.t, 50), 0); + + macaudio_spk_unlock(f.t.spk); + macaudio_test_settle(f.t.ma); + KUNIT_EXPECT_TRUE(test, macaudio_spk_leased(f.t.ma)); + KUNIT_EXPECT_FALSE(test, macaudio_spk_admitted(f.t.ma)); + macaudio_test_expect_limit(test, &f, 0, SN012776_0DB); + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&f.t.ma->spk_unowned_work)); + + /* A restarted engine takes it back within the grace: no revert */ + macaudio_test_take(test, &f.t); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&f.t.ma->spk_unowned_work)); + macaudio_spk_unlock(f.t.spk); + + /* The grace runs out */ + mod_delayed_work(system_wq, &f.t.ma->spk_unowned_work, 0); + flush_delayed_work(&f.t.ma->spk_unowned_work); + macaudio_test_settle(f.t.ma); + KUNIT_EXPECT_FALSE(test, macaudio_spk_leased(f.t.ma)); + KUNIT_EXPECT_FALSE(test, f.t.ma->speaker_volume_unlocked); + macaudio_test_expect_limit(test, &f, 0, 1); +} + +/* Only a CAP_SYS_ADMIN process hands the speakers over */ +static void macaudio_iface_needs_cap_sys_admin(struct kunit *test) +{ + struct macaudio_test_iface f; + const struct cred *old; + struct cred *cred; + + macaudio_test_iface_card(test, &f); + cred = prepare_creds(); + KUNIT_ASSERT_NOT_NULL(test, cred); + cap_lower(cred->cap_effective, CAP_SYS_ADMIN); + old = override_creds(cred); + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, MACAUDIO_SPK_IFACE_LEASE), + -EPERM); + revert_creds(old); + put_cred(cred); + KUNIT_EXPECT_FALSE(test, macaudio_spk_leased(f.t.ma)); + macaudio_test_expect_limit(test, &f, 1, 1); + KUNIT_EXPECT_EQ(test, macaudio_test_iface_write(test, &f, 2), -EINVAL); +} + +/* The switch is no mixer setting: alsactl neither saves nor restores it */ +static void macaudio_iface_control_is_inactive(struct kunit *test) +{ + struct snd_kcontrol_new iface = macaudio_test_control("Speaker Protection Interface"); + + KUNIT_EXPECT_TRUE(test, iface.access & SNDRV_CTL_ELEM_ACCESS_INACTIVE); + KUNIT_EXPECT_TRUE(test, iface.access & SNDRV_CTL_ELEM_ACCESS_VOLATILE); +} + +static struct kunit_case macaudio_iface_cases[] = { + KUNIT_CASE(macaudio_iface_volume_lock_from_binding), + KUNIT_CASE(macaudio_iface_switch_goes_through_the_floor), + KUNIT_CASE(macaudio_iface_revert_starts_at_the_floor), + KUNIT_CASE(macaudio_iface_unowned_lease_reverts), + KUNIT_CASE(macaudio_iface_needs_cap_sys_admin), + KUNIT_CASE(macaudio_iface_control_is_inactive), + {} +}; + +static struct kunit_suite macaudio_iface_suite = { + .name = "apple-speaker-interface", + .test_cases = macaudio_iface_cases, +}; +kunit_test_suite(macaudio_iface_suite); diff --git a/sound/soc/apple/macaudio-pm-volumes-test.h b/sound/soc/apple/macaudio-pm-volumes-test.h new file mode 100644 index 0000000000000..dab9d69bb0faa --- /dev/null +++ b/sound/soc/apple/macaudio-pm-volumes-test.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * The speaker volumes the system-sleep lock clips are set again at the + * daemon's first ping after it, through the lock control and the PM notifier. + */ +#include + +struct macaudio_pmt_vol { + struct soc_mixer_control mc; + long value; +}; + +static struct macaudio_pmt_vol *macaudio_pmt_vol(struct snd_kcontrol *kctl) +{ + return container_of((struct soc_mixer_control *)kctl->private_value, + struct macaudio_pmt_vol, mc); +} + +static int macaudio_pmt_vol_info(struct snd_kcontrol *kctl, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.max = SN012776_0DB; + return 0; +} + +static int macaudio_pmt_vol_get(struct snd_kcontrol *kctl, struct snd_ctl_elem_value *uval) +{ + uval->value.integer.value[0] = macaudio_pmt_vol(kctl)->value; + return 0; +} + +static int macaudio_pmt_vol_put(struct snd_kcontrol *kctl, struct snd_ctl_elem_value *uval) +{ + struct macaudio_pmt_vol *vol = macaudio_pmt_vol(kctl); + bool changed = vol->value != uval->value.integer.value[0]; + + vol->value = uval->value.integer.value[0]; + return changed; +} + +static void macaudio_pmt_free_kctl(void *kctl) +{ + snd_ctl_free_one(kctl); +} + +/* Last in, so first out: before the controls go */ +static void macaudio_pmt_pm_cleanup(void *data) +{ + struct macaudio_snd_data *ma = data; + + cancel_delayed_work_sync(&ma->lock_timeout_work); + macaudio_pm_volumes_drop(ma); + macaudio_spk_fini(ma); +} + +static const char *const macaudio_pmt_vol_names[] = { + "Left Woofer Speaker Volume", + "Right Woofer Speaker Volume", + "Left Tweeter Speaker Volume", + "Right Tweeter Speaker Volume", + /* Not a speaker volume: never locked, kept or set */ + "Headphone Playback Volume", +}; + +/* As the daemon set them before the sleep */ +static const long macaudio_pmt_vols[] = { 201, 190, 201, 170, 120 }; + +#define MACAUDIO_PMT_NR_VOLS ARRAY_SIZE(macaudio_pmt_vol_names) + +struct macaudio_pmt_pm { + struct macaudio_snd_data *ma; + struct snd_kcontrol *slk; + struct snd_kcontrol *vols[MACAUDIO_PMT_NR_VOLS]; +}; + +static struct snd_kcontrol *macaudio_pmt_add_kctl(struct kunit *test, struct snd_card *card, + const struct snd_kcontrol_new *tmpl, + void *private_data) +{ + struct snd_kcontrol *kctl = snd_ctl_new1(tmpl, private_data); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, kctl); + ret = kunit_add_action_or_reset(test, macaudio_pmt_free_kctl, kctl); + KUNIT_ASSERT_EQ(test, ret, 0); + list_add_tail(&kctl->list, &card->controls); + return kctl; +} + +/* A card with speaker safety, its lock control and the volumes, none owned yet */ +static void macaudio_pmt_pm_card(struct kunit *test, struct macaudio_pmt_pm *t) +{ + struct snd_kcontrol_new slk = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Speaker Volume Unlock", + .info = snd_ctl_boolean_mono_info, + .get = macaudio_slk_get, + .put = macaudio_slk_put, + }; + struct macaudio_snd_data *ma; + struct snd_card *card; + unsigned int i; + int ret; + + ma = kunit_kzalloc(test, sizeof(*ma), GFP_KERNEL); + card = kunit_kzalloc(test, sizeof(*card), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ma); + KUNIT_ASSERT_NOT_NULL(test, card); + + INIT_LIST_HEAD(&card->controls); + INIT_LIST_HEAD(&card->ctl_files); + init_rwsem(&card->controls_rwsem); + rwlock_init(&card->controls_rwlock); + ma->card.snd_card = card; + snd_soc_card_set_drvdata(&ma->card, ma); + ma->cfg = &macaudio_j493_cfg; + ma->has_speakers = true; + ma->has_safety = true; + INIT_LIST_HEAD(&ma->card.rtd_list); + mutex_init(&ma->volume_lock_mutex); + INIT_DELAYED_WORK(&ma->lock_timeout_work, macaudio_vlimit_timeout_work); + /* The volume lock is the interface, as at every bind */ + mutex_init(&ma->spk_mutex); + KUNIT_ASSERT_EQ(test, macaudio_spk_init(ma), 0); + + t->ma = ma; + t->slk = macaudio_pmt_add_kctl(test, card, &slk, &ma->card); + ma->speaker_lock_kctl = t->slk; + + for (i = 0; i < MACAUDIO_PMT_NR_VOLS; i++) { + struct macaudio_pmt_vol *vol = kunit_kzalloc(test, sizeof(*vol), GFP_KERNEL); + struct snd_kcontrol_new tmpl = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = macaudio_pmt_vol_names[i], + .info = macaudio_pmt_vol_info, + .get = macaudio_pmt_vol_get, + .put = macaudio_pmt_vol_put, + }; + + KUNIT_ASSERT_NOT_NULL(test, vol); + vol->mc.max = SN012776_0DB; + vol->value = macaudio_pmt_vols[i]; + tmpl.private_value = (unsigned long)&vol->mc; + t->vols[i] = macaudio_pmt_add_kctl(test, card, &tmpl, NULL); + } + + ret = kunit_add_action_or_reset(test, macaudio_pmt_pm_cleanup, ma); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +/* A daemon takes the lock and every speaker volume, as speakersafetyd does */ +static void macaudio_pmt_take(struct kunit *test, struct macaudio_pmt_pm *t) +{ + struct snd_ctl_file *owner = kunit_kzalloc(test, sizeof(*owner), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, owner); + for (i = 0; i < MACAUDIO_PMT_NR_VOLS - 1; i++) + t->vols[i]->vd[0].owner = owner; + KUNIT_ASSERT_EQ(test, macaudio_slk_lock(t->slk, owner), 0); +} + +static int macaudio_pmt_ping(struct kunit *test, struct macaudio_pmt_pm *t) +{ + struct snd_ctl_elem_value *uval = kunit_kzalloc(test, sizeof(*uval), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, uval); + uval->value.integer.value[0] = SPEAKER_MAGIC_VALUE; + return macaudio_slk_put(t->slk, uval); +} + +static void macaudio_pmt_notify(struct macaudio_pmt_pm *t, unsigned long action) +{ + macaudio_pm_notify(&t->ma->pm_nb, action, NULL); +} + +/* The speaker volumes as the daemon set them, or clipped to the lock's ceiling */ +static void macaudio_pmt_expect_vols(struct kunit *test, struct macaudio_pmt_pm *t, + bool clipped) +{ + unsigned int i; + + for (i = 0; i < MACAUDIO_PMT_NR_VOLS; i++) { + bool speaker = i < MACAUDIO_PMT_NR_VOLS - 1; + + KUNIT_EXPECT_EQ(test, macaudio_pmt_vol(t->vols[i])->value, + speaker && clipped ? 1L : macaudio_pmt_vols[i]); + } +} + +static void macaudio_pm_volumes_restored_after_sleep(struct kunit *test) +{ + struct macaudio_pmt_pm t; + + macaudio_pmt_pm_card(test, &t); + macaudio_pmt_take(test, &t); + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + KUNIT_EXPECT_TRUE(test, t.ma->speaker_volume_unlocked); + macaudio_pmt_expect_vols(test, &t, false); + + macaudio_pmt_notify(&t, PM_SUSPEND_PREPARE); + KUNIT_EXPECT_FALSE(test, t.ma->speaker_volume_unlocked); + macaudio_pmt_expect_vols(test, &t, true); + /* A ping before the freeze leaves them locked */ + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + KUNIT_EXPECT_FALSE(test, t.ma->speaker_volume_unlocked); + + /* Locked after the resume until the daemon pings */ + macaudio_pmt_notify(&t, PM_POST_SUSPEND); + KUNIT_EXPECT_FALSE(test, t.ma->speaker_volume_unlocked); + macaudio_pmt_expect_vols(test, &t, true); + + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + KUNIT_EXPECT_TRUE(test, t.ma->speaker_volume_unlocked); + macaudio_pmt_expect_vols(test, &t, false); + KUNIT_EXPECT_NULL(test, t.ma->pm_volumes); +} + +/* The daemon pings only once a stream runs: a second sleep can come first. */ +static void macaudio_pm_volumes_second_sleep_before_ping(struct kunit *test) +{ + struct macaudio_pmt_pm t; + + macaudio_pmt_pm_card(test, &t); + macaudio_pmt_take(test, &t); + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + + macaudio_pmt_notify(&t, PM_SUSPEND_PREPARE); + macaudio_pmt_notify(&t, PM_POST_SUSPEND); + macaudio_pmt_notify(&t, PM_SUSPEND_PREPARE); + macaudio_pmt_notify(&t, PM_POST_SUSPEND); + macaudio_pmt_expect_vols(test, &t, true); + + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + macaudio_pmt_expect_vols(test, &t, false); +} + +/* A new owner, such as a restarted daemon, sets the volumes itself. */ +static void macaudio_pm_volumes_new_owner(struct kunit *test) +{ + struct macaudio_pmt_pm t; + + macaudio_pmt_pm_card(test, &t); + macaudio_pmt_take(test, &t); + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + macaudio_pmt_notify(&t, PM_SUSPEND_PREPARE); + macaudio_pmt_notify(&t, PM_POST_SUSPEND); + + macaudio_pmt_take(test, &t); + KUNIT_EXPECT_NULL(test, t.ma->pm_volumes); + KUNIT_EXPECT_EQ(test, macaudio_pmt_ping(test, &t), 0); + KUNIT_EXPECT_TRUE(test, t.ma->speaker_volume_unlocked); + macaudio_pmt_expect_vols(test, &t, true); +} + +static struct kunit_case macaudio_pm_volumes_cases[] = { + KUNIT_CASE(macaudio_pm_volumes_restored_after_sleep), + KUNIT_CASE(macaudio_pm_volumes_second_sleep_before_ping), + KUNIT_CASE(macaudio_pm_volumes_new_owner), + {} +}; + +static struct kunit_suite macaudio_pm_volumes_suite = { + .name = "apple-speaker-pm-volumes", + .test_cases = macaudio_pm_volumes_cases, +}; +kunit_test_suite(macaudio_pm_volumes_suite); diff --git a/sound/soc/apple/macaudio-protection-test.h b/sound/soc/apple/macaudio-protection-test.h index ca50d5055a322..7c3c6b03d7cf5 100644 --- a/sound/soc/apple/macaudio-protection-test.h +++ b/sound/soc/apple/macaudio-protection-test.h @@ -33,7 +33,9 @@ static int macaudio_test_vol_info(struct snd_kcontrol *kctl, struct snd_ctl_elem { uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; uinfo->count = 1; - uinfo->value.integer.max = macaudio_test_vol(kctl)->mc.max; + /* As snd_soc_info_volsw(): a limit is the top of the range */ + uinfo->value.integer.max = macaudio_test_vol(kctl)->mc.platform_max ?: + macaudio_test_vol(kctl)->mc.max; return 0; } @@ -98,8 +100,21 @@ struct macaudio_test_spk { /* "Speaker Protection Crossover" */ struct snd_kcontrol *xover; struct snd_ctl_file *owner; + /* The codec the speaker volumes belong to */ + struct snd_soc_component component; }; +/* The card's control named @name */ +static struct snd_kcontrol_new macaudio_test_control(const char *name) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(macaudio_controls); i++) + if (macaudio_controls[i].name && !strcmp(macaudio_controls[i].name, name)) + return macaudio_controls[i]; + return (struct snd_kcontrol_new){}; +} + static struct snd_kcontrol *macaudio_test_add_kctl(struct kunit *test, struct snd_card *card, const struct snd_kcontrol_new *tmpl, void *private_data) @@ -118,7 +133,7 @@ static struct snd_kcontrol *macaudio_test_add_kctl(struct kunit *test, struct sn static void macaudio_test_spk_card(struct kunit *test, struct macaudio_test_spk *t, const struct macaudio_platform_cfg *cfg, long max) { - struct snd_kcontrol_new spk = macaudio_controls[ARRAY_SIZE(macaudio_controls) - 1]; + struct snd_kcontrol_new spk = macaudio_test_control("Speaker Protection"); struct macaudio_snd_data *ma; struct snd_card *card; struct device *dev; @@ -148,7 +163,14 @@ static void macaudio_test_spk_card(struct kunit *test, struct macaudio_test_spk ma->cfg = cfg; ma->has_speakers = true; mutex_init(&ma->spk_mutex); + mutex_init(&ma->volume_lock_mutex); + INIT_DELAYED_WORK(&ma->lock_timeout_work, macaudio_vlimit_timeout_work); + INIT_WORK(&ma->lock_fail_work, macaudio_lock_fail_work); + INIT_WORK(&ma->sw_gain_stop_work, macaudio_sw_gain_stop_work); + /* The lease is the interface, as after "Speaker Protection Interface" */ + ma->spk_iface = MACAUDIO_SPK_IFACE_LEASE; KUNIT_ASSERT_EQ(test, macaudio_spk_init(ma), 0); + t->component.card = &ma->card; t->ma = ma; t->spk = macaudio_test_add_kctl(test, card, &spk, &ma->card); @@ -170,7 +192,7 @@ static void macaudio_test_spk_card(struct kunit *test, struct macaudio_test_spk if (cfg->amp == AMP_MAX98360A && i < MACAUDIO_TEST_NR_SPK) tmpl.name = macaudio_test_sw_vol_names[i]; tmpl.private_value = (unsigned long)&vol->mc; - t->vols[i] = macaudio_test_add_kctl(test, card, &tmpl, NULL); + t->vols[i] = macaudio_test_add_kctl(test, card, &tmpl, &t->component); } t->xover = macaudio_test_add_kctl(test, card, &macaudio_spk_xover_control, &ma->card); @@ -266,9 +288,8 @@ static void macaudio_spk_floor_from_binding(struct kunit *test) /* Userspace can read the speaker volumes but never write them */ for (i = 0; i < MACAUDIO_TEST_NR_SPK; i++) - KUNIT_EXPECT_FALSE(test, t.vols[i]->vd[0].access & SNDRV_CTL_ELEM_ACCESS_WRITE); - KUNIT_EXPECT_TRUE(test, t.vols[MACAUDIO_TEST_NR_SPK]->vd[0].access & - SNDRV_CTL_ELEM_ACCESS_WRITE); + KUNIT_EXPECT_EQ(test, t.vols[i]->write_check(t.vols[i], NULL), -EPERM); + KUNIT_EXPECT_NULL(test, t.vols[MACAUDIO_TEST_NR_SPK]->write_check); } static void macaudio_spk_grant_needs_owner_and_valid_lease(struct kunit *test) @@ -703,7 +724,10 @@ static void macaudio_spk_regrant_waits_for_the_stop(struct kunit *test) macaudio_test_expect_vols(test, &t, SN012776_0DB); } -/* "Amp Gain Volume": held at the configured gain, readable, not writable */ +/* + * "Amp Gain Volume": limited at the configured gain, fixed at it while + * leased and not writable then, the amp's own under the volume lock. + */ static void macaudio_spk_amp_gain_is_fixed(struct kunit *test) { static const char *const names[] = { @@ -715,6 +739,7 @@ static void macaudio_spk_amp_gain_is_fixed(struct kunit *test) struct macaudio_test_spk t; const int level = 15; unsigned int i; + int ret; macaudio_test_spk_card(test, &t, &macaudio_j316_cfg, SN012776_0DB); for (i = 0; i < ARRAY_SIZE(names); i++) { @@ -731,25 +756,35 @@ static void macaudio_spk_amp_gain_is_fixed(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, vols[i]); vols[i]->mc.max = 20; tmpl.private_value = (unsigned long)&vols[i]->mc; - kctls[i] = macaudio_test_add_kctl(test, t.ma->card.snd_card, &tmpl, NULL); + kctls[i] = macaudio_test_add_kctl(test, t.ma->card.snd_card, &tmpl, + &t.component); } /* Above the amp level (any level the ADT gives), and below it */ vols[0]->value = 20; vols[1]->value = 5; - KUNIT_EXPECT_EQ(test, macaudio_set_fixed_volume(&t.ma->card, "* Amp Gain Volume", - level), 2); + KUNIT_EXPECT_EQ(test, snd_soc_limit_volume(&t.ma->card, "* Amp Gain Volume", level), 2); + KUNIT_EXPECT_EQ(test, vols[0]->value, (long)level); + KUNIT_EXPECT_EQ(test, vols[1]->value, 5L); + KUNIT_ASSERT_EQ(test, macaudio_spk_take_amp_gains(t.ma), 0); + KUNIT_EXPECT_EQ(test, t.ma->spk_nr_amp_gains, (unsigned int)ARRAY_SIZE(names)); + + mutex_lock(&t.ma->spk_mutex); + ret = macaudio_spk_fix_amp_gains(t.ma); + mutex_unlock(&t.ma->spk_mutex); + KUNIT_EXPECT_EQ(test, ret, 0); for (i = 0; i < ARRAY_SIZE(names); i++) { KUNIT_EXPECT_EQ(test, vols[i]->value, (long)level); KUNIT_EXPECT_EQ(test, vols[i]->mc.platform_max, level); - KUNIT_EXPECT_TRUE(test, kctls[i]->vd[0].access & SNDRV_CTL_ELEM_ACCESS_READ); - KUNIT_EXPECT_FALSE(test, kctls[i]->vd[0].access & SNDRV_CTL_ELEM_ACCESS_WRITE); + KUNIT_EXPECT_EQ(test, kctls[i]->write_check(kctls[i], NULL), -EPERM); } /* The speaker volumes and the headphone volume are not amp gains */ - KUNIT_EXPECT_TRUE(test, t.vols[MACAUDIO_TEST_NR_SPK]->vd[0].access & - SNDRV_CTL_ELEM_ACCESS_WRITE); - KUNIT_EXPECT_EQ(test, macaudio_set_fixed_volume(&t.ma->card, "* Missing Volume", 15), - -EINVAL); + KUNIT_EXPECT_NULL(test, t.vols[MACAUDIO_TEST_NR_SPK]->write_check); + + /* Under the volume lock they are the amp's again */ + WRITE_ONCE(t.ma->spk_iface, MACAUDIO_SPK_IFACE_UNLOCK); + for (i = 0; i < ARRAY_SIZE(names); i++) + KUNIT_EXPECT_EQ(test, kctls[i]->write_check(kctls[i], NULL), 0); } struct macaudio_test_sleep { diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 220b99b853879..4e1b642e4b0a5 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -66,8 +66,17 @@ */ #define MACAUDIO_MAX_BCLK_FREQ 24576000 +#define SPEAKER_MAGIC_VALUE (s32)0xdec1be15 /* Primary, Secondary and Speaker Sense */ #define MACAUDIO_NUM_FE_LINKS 3 +/* The "Speaker Volume Unlock" lease, in milliseconds */ +#define SPEAKER_LOCK_TIMEOUT 250 +/* + * While "Speaker Protection" is the interface, this long without an owner of + * it (the engine died, or never came) puts "Speaker Volume Unlock" back in + * charge, at its ceiling. + */ +#define SPEAKER_LEASE_UNOWNED_MS 30000 /* The speaker protection lease an owner may ask for, in milliseconds */ #define SPEAKER_LEASE_MIN_MS 10 #define SPEAKER_LEASE_MAX_MS 100 @@ -121,6 +130,11 @@ struct macaudio_platform_cfg { * amplifiers) unless set. */ const char *spk_out_widget; + /* + * Software-gain amplifier: the CPU DAI whose front-end converts the + * speaker samples through the volume control at copy time. + */ + const char *sw_gain_dai; /* * A CPU component other than the MCA: its front-end DAI names in * place of "mca-pcm-N", the DAI format its serial links run (the @@ -192,6 +206,29 @@ struct ma_codec_idle { u32 idle_mask; }; +/* A speaker volume control and its value, as the system-sleep lock found it */ +struct macaudio_pm_volume { + struct snd_kcontrol *kctl; + long value[2]; +}; + +/* + * The interface that owns the speakers (see "Speaker Protection Interface"): + * the volume lock of speakersafetyd and speakerguardd 0.1, or the lease of a + * protection engine in the signal path. + */ +enum macaudio_spk_iface { + MACAUDIO_SPK_IFACE_UNLOCK, + MACAUDIO_SPK_IFACE_LEASE, +}; + +/* + * The software-gain CPU component's register that selects the volume + * table's top: 0 the legacy ceiling (half the amplifier's full scale), 1 the + * full scale (sound/soc/apple/aop_t8140.rs, REG_SPKR_FULL_SCALE). + */ +#define MACAUDIO_SW_GAIN_REG_FULL_SCALE 1 + struct macaudio_snd_data { struct snd_soc_card card; struct snd_soc_jack jack; @@ -202,6 +239,7 @@ struct macaudio_snd_data { const struct macaudio_platform_cfg *cfg; bool has_speakers; bool has_sense; + bool has_safety; unsigned int max_channels; struct gpio_descs *speaker_id_gpios; @@ -209,6 +247,8 @@ struct macaudio_snd_data { /* frontend props */ unsigned int bclk_ratio; bool is_sense; + /* the front-end that applies the software speaker gain */ + bool sw_gain; /* backend props */ bool is_speakers; @@ -280,8 +320,80 @@ struct macaudio_snd_data { */ struct snd_kcontrol *spk_xover_kctl; bool spk_xover; + + /* + * The interface in charge (enum macaudio_spk_iface): written under + * spk_mutex and volume_lock_mutex, read with either or without both. + */ + int spk_iface; + struct snd_kcontrol *spk_iface_kctl; + /* "Speaker Volume Unlock" back in charge after a lease without owner */ + struct delayed_work spk_unowned_work; + /* The amps' own gains, fixed at their limit while leased */ + struct snd_kcontrol **spk_amp_gains; + unsigned int spk_nr_amp_gains; + /* The software-gain CPU component, whose volume table top follows the interface */ + struct snd_soc_component *sw_gain_component; + + /* + * "Speaker Volume Unlock": the speakersafetyd / speakerguardd 0.1 + * interface (see macaudio_vlimit_update()). + */ + int speaker_sample_rate; + struct snd_kcontrol *speaker_sample_rate_kctl; + + struct mutex volume_lock_mutex; + bool speaker_volume_unlocked; + bool speaker_volume_was_locked; + struct snd_kcontrol *speaker_lock_kctl; + struct snd_ctl_file *speaker_lock_owner; + u64 bes_active; + bool speaker_lock_timeout_enabled; + ktime_t speaker_lock_timeout; + ktime_t speaker_lock_remain; + struct delayed_work lock_timeout_work; + struct work_struct lock_update_work; + /* + * The speaker volumes as the system-sleep lock found them. Locking clips + * them to its ceiling and unlocking only lifts the ceiling; the daemon + * keeps its lease across the sleep and does not set them again. + */ + struct macaudio_pm_volume *pm_volumes; + unsigned int pm_nr_volumes; + /* Allocated with them, so that setting them again cannot fail */ + struct snd_ctl_elem_value *pm_uval; + + /* + * Software-gain amplifier: a lower gain cannot change samples that + * were converted before it was set, so every lock event bumps the + * epoch and stops the front-end stream; it is refused until it has + * been prepared (and its buffer cleared) under the new epoch. + */ + atomic_t sw_gain_lock_epoch; + atomic_t sw_gain_prepared_epoch; + atomic_t sw_gain_limit_failed; + struct work_struct sw_gain_stop_work; + + /* + * A lock that did not take on every speaker volume (a control write + * failed) is a failed lock: the playback streams are stopped and + * refused until the driver is reloaded, since a volume above the safe + * value may be left behind. + */ + bool speaker_lock_failed; + struct work_struct lock_fail_work; }; +static int please_blow_up_my_speakers; +module_param(please_blow_up_my_speakers, int, 0644); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Deprecated: cannot bypass speaker admission or protection"); + +/* "Speaker Protection" owns the speakers, not "Speaker Volume Unlock" */ +static bool macaudio_spk_leased(const struct macaudio_snd_data *ma) +{ + return READ_ONCE(ma->spk_iface) == MACAUDIO_SPK_IFACE_LEASE; +} + SND_SOC_DAILINK_DEFS(primary, DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU @@ -374,9 +486,485 @@ static struct macaudio_link_props macaudio_fe_link_props[] = { }; /* - * Speaker protection ownership. + * "Speaker Volume Unlock": speaker protection by a userspace daemon + * (speakersafetyd, speakerguardd 0.1), the default interface. * - * The speakers play only while a protection engine owns them. Without an + * The daemon holds SNDRV_CTL_IOCTL_ELEM_LOCK on "Speaker Volume Unlock" and + * on every speaker volume control, and pings the unlock control with + * SPEAKER_MAGIC_VALUE at least every SPEAKER_LOCK_TIMEOUT ms while the + * speakers play. While it does, the volumes may go up to the amps' 0 dB; + * otherwise they are held at the lock's ceiling. The daemon governs them + * from its thermal model. + */ + +static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) +{ + int ret, max, hits = 0; + const char *name = volume_control_names[ma->cfg->amp]; + struct snd_kcontrol *kctl; + bool failed = false; + + if (!name) { + WARN_ON_ONCE(1); + return; + } + + switch (ma->cfg->amp) { + case AMP_NONE: + WARN_ON_ONCE(1); + return; + case AMP_TAS5770: + if (unlock) + max = TAS5770_0DB; + else + max = 1; //TAS5770_DB(ma->cfg->safe_vol); + break; + case AMP_SN012776: + if (unlock) + max = SN012776_0DB; + else + max = 1; //SN012776_DB(ma->cfg->safe_vol); + break; + case AMP_SSM3515: + if (unlock) + max = SSM3515_0DB; + else + max = SSM3515_DB(ma->cfg->safe_vol); + break; + case AMP_MAX98360A: + if (unlock) + max = MAX98360A_0DB; + else + max = MAX98360A_DB(ma->cfg->safe_vol); + break; + } + + /* + * One control at a time: snd_soc_limit_volume() stops at the first + * control it cannot clip, which would leave the rest at their old + * limit and value. + */ + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + hits++; + ret = snd_soc_limit_volume(&ma->card, kctl->id.name, max); + if (ret < 0) { + dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", + unlock ? "un" : "", kctl->id.name, ret); + failed = true; + } + } + if (!hits) + dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", + unlock ? "un" : "", name, -EINVAL); + + /* + * With a software gain the limit is the whole protection: refuse to + * play rather than run the amplifier at full scale unlimited. + */ + if (ma->cfg->amp == AMP_MAX98360A) + atomic_set(&ma->sw_gain_limit_failed, failed || !hits); + + if (failed && !unlock) { + WRITE_ONCE(ma->speaker_lock_failed, true); + schedule_work(&ma->lock_fail_work); + } +} + +/* + * Stop every running playback stream of the card, or with sw_gain_only those + * on software-gain links whose prepare predates the current lock epoch. + */ +static void macaudio_stop_playback(struct macaudio_snd_data *ma, bool sw_gain_only) +{ + struct snd_soc_pcm_runtime *rtd; + struct snd_pcm_substream *substream; + unsigned long flags; + + /* + * No volume or control lock may be held while taking the stream lock. + * Pick the links from the runtime being walked: not every playback + * substream on the card carries a runtime as its private data. + */ + for_each_card_rtds(&ma->card, rtd) { + /* A back-end's substream shares its front-end's runtime: stop front-ends only. */ + if (!rtd->pcm || rtd->dai_link->no_pcm) + continue; + if (sw_gain_only && !ma->link_props[rtd->dai_link->id].sw_gain) + continue; + substream = rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (!substream) + continue; + snd_pcm_stream_lock_irqsave(substream, flags); + if (substream->runtime && + (!sw_gain_only || + atomic_read(&ma->sw_gain_prepared_epoch) != + atomic_read(&ma->sw_gain_lock_epoch))) { + switch (substream->runtime->state) { + case SNDRV_PCM_STATE_PREPARED: + case SNDRV_PCM_STATE_RUNNING: + case SNDRV_PCM_STATE_DRAINING: + case SNDRV_PCM_STATE_PAUSED: + snd_pcm_stop(substream, SNDRV_PCM_STATE_XRUN); + break; + default: + break; + } + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + } +} + +/* Stop every running playback stream of the card after a failed lock. */ +static void macaudio_lock_fail_work(struct work_struct *work) +{ + struct macaudio_snd_data *ma = container_of(work, struct macaudio_snd_data, + lock_fail_work); + + dev_crit(ma->card.dev, "speaker volume lock failed: stopping and refusing playback\n"); + macaudio_stop_playback(ma, false); +} + +static void macaudio_pm_volumes_drop(struct macaudio_snd_data *ma) +{ + kfree(ma->pm_volumes); + kfree(ma->pm_uval); + ma->pm_volumes = NULL; + ma->pm_nr_volumes = 0; + ma->pm_uval = NULL; +} + +/* + * Keep the speaker volumes before the system-sleep lock clips them. + * + * Context: volume_lock_mutex and the card's controls_rwsem held, so the daemon + * cannot set a volume between this and the clip. + */ +static void macaudio_pm_volumes_save(struct macaudio_snd_data *ma) +{ + const char *name = volume_control_names[ma->cfg->amp]; + struct snd_ctl_elem_value *uval = NULL; + struct macaudio_pm_volume *vols = NULL; + struct snd_kcontrol *kctl; + unsigned int nr = 0, i = 0; + int ret; + + lockdep_assert_held(&ma->volume_lock_mutex); + lockdep_assert_held(&ma->card.snd_card->controls_rwsem); + + if (!ma->has_safety || !name || ma->card.snd_card->shutdown) + return; + /* + * Locked already, as after a sleep the daemon has not pinged since: + * the volumes are at the ceiling, and those from before it are kept. + */ + if (!ma->speaker_volume_unlocked) + return; + macaudio_pm_volumes_drop(ma); + + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) + if (snd_soc_control_matches(kctl, name)) + nr++; + if (!nr) + return; + + vols = kzalloc_objs(*vols, nr); + uval = kzalloc_obj(*uval); + if (!vols || !uval) + goto out; + + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + memset(uval, 0, sizeof(*uval)); + uval->id = kctl->id; + ret = kctl->get(kctl, uval); + if (ret < 0) { + dev_warn(ma->card.dev, "Failed to keep volume %s for the system sleep: %d\n", + kctl->id.name, ret); + goto out; + } + vols[i].kctl = kctl; + vols[i].value[0] = uval->value.integer.value[0]; + vols[i].value[1] = uval->value.integer.value[1]; + i++; + } + + ma->pm_volumes = vols; + ma->pm_nr_volumes = i; + ma->pm_uval = uval; + return; +out: + kfree(uval); + kfree(vols); +} + +/* + * At the unlock after a system sleep: the volumes the daemon had set. + * + * Context: volume_lock_mutex and the card's controls_rwsem held, so no volume + * write races these. + */ +static void macaudio_pm_volumes_restore(struct macaudio_snd_data *ma) +{ + struct snd_ctl_elem_value *uval = ma->pm_uval; + unsigned int i; + int ret; + + if (!ma->pm_volumes) + return; + + for (i = 0; i < ma->pm_nr_volumes; i++) { + struct snd_kcontrol *kctl = ma->pm_volumes[i].kctl; + + memset(uval, 0, sizeof(*uval)); + uval->id = kctl->id; + uval->value.integer.value[0] = ma->pm_volumes[i].value[0]; + uval->value.integer.value[1] = ma->pm_volumes[i].value[1]; + ret = kctl->put(kctl, uval); + if (ret < 0) + dev_err(ma->card.dev, "Failed to restore volume %s: %d\n", + kctl->id.name, ret); + else if (ret > 0) + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &kctl->id); + } + macaudio_pm_volumes_drop(ma); +} + +static void macaudio_vlimit_update(struct macaudio_snd_data *ma) +{ + int i; + bool unlock = true; + struct snd_kcontrol *kctl; + const char *reason; + + /* Do nothing if there is no safety configured */ + if (!ma->has_safety) + return; + + /* + * "Speaker Protection" owns the speakers: the lock is held at its + * ceiling until the interface switches back (macaudio_spk_leave_lease()). + */ + if (macaudio_spk_leased(ma)) + return; + + /* Check that someone is holding the main lock */ + if (!ma->speaker_lock_owner) { + reason = "Main control not locked"; + unlock = false; + } + + /* Check that the control has been pinged within the timeout */ + if (ma->speaker_lock_remain <= 0) { + reason = "Lock timeout"; + unlock = false; + } + + /* + * System sleep: userspace is frozen long before the card suspends (over + * half a second on J616s), and a playback stream left running then plays + * out its stale buffer and stops on an xrun into the still unmuted amps. + * Lock the volumes before the freeze instead of when the frozen daemon's + * lease runs out. + */ + if (ma->pm_quiesced) { + reason = "System suspend"; + unlock = false; + } + + /* Check that *every* limited control is locked by the same owner */ + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if(!snd_soc_control_matches(kctl, volume_control_names[ma->cfg->amp])) + continue; + + for (i = 0; i < kctl->count; i++) { + if (kctl->vd[i].owner != ma->speaker_lock_owner) { + reason = "Not all child controls locked by the same process"; + unlock = false; + } + } + } + + + if (unlock != ma->speaker_volume_unlocked) { + if (unlock) { + dev_info(ma->card.dev, "Speaker volumes unlocked\n"); + } else { + dev_info(ma->card.dev, "Speaker volumes locked: %s\n", reason); + /* The system-sleep lock is the kernel's own, not a lease the daemon lost */ + if (!ma->pm_quiesced) + ma->speaker_volume_was_locked = true; + } + + macaudio_vlimit_unlock(ma, unlock); + if (unlock) + macaudio_pm_volumes_restore(ma); + if (!unlock && ma->cfg->amp == AMP_MAX98360A) { + /* Samples already converted at the old gain must not play */ + atomic_inc(&ma->sw_gain_lock_epoch); + schedule_work(&ma->sw_gain_stop_work); + } + ma->speaker_volume_unlocked = unlock; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_lock_kctl->id); + } +} + +/* + * The volume lock's part of a system sleep (see macaudio_pm_notify()), after + * pm_quiesced has been set or cleared. + * + * Volume writes hold the controls for writing: none can land between + * keeping the volumes and clipping them. Control writes take + * volume_lock_mutex inside it as well. + */ +static void macaudio_vlimit_pm(struct macaudio_snd_data *ma, bool quiesce) +{ + down_read(&ma->card.snd_card->controls_rwsem); + mutex_lock(&ma->volume_lock_mutex); + /* + * The lease ends here, so after the resume the volumes stay locked until + * the daemon's next ping. Its clock (CLOCK_MONOTONIC) stops across s2idle: + * a lease left running would unlock them at resume and expire before the + * thawed daemon pings, which reports a lost lock and restarts it cold. + */ + if (quiesce) { + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_remain = 0; + macaudio_pm_volumes_save(ma); + } + macaudio_vlimit_update(ma); + mutex_unlock(&ma->volume_lock_mutex); + up_read(&ma->card.snd_card->controls_rwsem); +} + +static void macaudio_vlimit_enable_timeout(struct macaudio_snd_data *ma) +{ + mutex_lock(&ma->volume_lock_mutex); + + if (ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + if (ma->speaker_lock_remain > 0) { + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + dev_dbg(ma->card.dev, "Enabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + } + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = true; + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_disable_timeout(struct macaudio_snd_data *ma) +{ + ktime_t now; + + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + now = ktime_get(); + + cancel_delayed_work(&ma->lock_timeout_work); + + if (ktime_after(now, ma->speaker_lock_timeout)) + ma->speaker_lock_remain = 0; + else if (ma->speaker_lock_remain > 0) + ma->speaker_lock_remain = ktime_sub(ma->speaker_lock_timeout, now); + + dev_dbg(ma->card.dev, "Disabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = false; + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_timeout_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(to_delayed_work(wrk), + struct macaudio_snd_data, lock_timeout_work); + ktime_t now, left; + + mutex_lock(&ma->volume_lock_mutex); + + /* + * A ping or a disabled timeout can have cancelled this work after it + * had already started running; neither must expire the lease. + */ + if (!ma->speaker_lock_timeout_enabled || !ma->speaker_lock_owner || + ma->speaker_lock_remain <= 0) + goto out; + + /* A ping renewed the lease while this work was already running */ + now = ktime_get(); + if (ktime_before(now, ma->speaker_lock_timeout)) { + left = ktime_sub(ma->speaker_lock_timeout, now); + schedule_delayed_work(&ma->lock_timeout_work, + usecs_to_jiffies(ktime_to_us(left))); + goto out; + } + + ma->speaker_lock_remain = 0; + macaudio_vlimit_update(ma); + +out: + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_update_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(wrk, + struct macaudio_snd_data, lock_update_work); + + /* An unlock here sets the kept volumes again: no volume write may race it */ + down_read(&ma->card.snd_card->controls_rwsem); + if (ma->bes_active) + macaudio_vlimit_enable_timeout(ma); + else + macaudio_vlimit_disable_timeout(ma); + up_read(&ma->card.snd_card->controls_rwsem); +} + +static bool macaudio_sw_gain_pcm(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + return props->sw_gain && substream->stream == SNDRV_PCM_STREAM_PLAYBACK; +} + +static void macaudio_sw_gain_stop_work(struct work_struct *work) +{ + struct macaudio_snd_data *ma = container_of(work, struct macaudio_snd_data, + sw_gain_stop_work); + + macaudio_stop_playback(ma, true); +} + +/* + * Speaker protection ownership: "Speaker Protection", a protection engine + * in the signal path. It owns the speakers only while it is the interface + * ("Speaker Protection Interface", see macaudio_spk_iface_put()); under + * "Speaker Volume Unlock" an owner may hold the control, but nothing is + * granted and its locks and releases touch no amp. + * + * While leased, the speakers play only while a protection engine owns them. Without an * owner, or once its lease has run out, every speaker amp sits at the safe * level and no speaker playback can start: that is the hardware floor, and it * holds from the card's registration on. @@ -892,6 +1480,7 @@ static void macaudio_spk_fini(void *data) { struct macaudio_snd_data *ma = data; + cancel_delayed_work_sync(&ma->spk_unowned_work); hrtimer_cancel(&ma->spk_lease_timer); /* The expiry may queue a stop: its worker goes first */ if (!IS_ERR_OR_NULL(ma->spk_worker)) @@ -905,12 +1494,15 @@ static void macaudio_spk_fini(void *data) static int macaudio_pm_notify(struct notifier_block *nb, unsigned long action, void *data); +static void macaudio_spk_unowned_work(struct work_struct *work); + /* The lease timer, the workers and the sleep notifier, until macaudio_spk_fini() */ static int macaudio_spk_init(struct macaudio_snd_data *ma) { hrtimer_setup(&ma->spk_lease_timer, macaudio_spk_lease_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); kthread_init_work(&ma->spk_expire_work, macaudio_spk_expire_work); + INIT_DELAYED_WORK(&ma->spk_unowned_work, macaudio_spk_unowned_work); kthread_init_work(&ma->spk_stop_work, macaudio_spk_stop_work); /* First in the chain: no other driver's sleep preparation goes before the floor */ ma->pm_nb.notifier_call = macaudio_pm_notify; @@ -950,42 +1542,326 @@ static int macaudio_pm_notify(struct notifier_block *nb, unsigned long action, ktime_t t0; s64 waited; - /* A card without speakers (the J616s jack) has nothing to floor */ - if (!ma->has_speakers) - return NOTIFY_DONE; + /* A card without speakers (the J616s jack) has nothing to floor */ + if (!ma->has_speakers) + return NOTIFY_DONE; + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + /* + * Before anything that can wait, spk_mutex included: a writer + * that holds it may be stuck behind the sleep itself. No grant + * and no start from here on, and the running streams stop. + */ + WRITE_ONCE(ma->pm_quiesced, true); + macaudio_spk_close_gate(ma, "system sleep"); + t0 = ktime_get(); + mutex_lock(&ma->spk_mutex); + waited = ktime_ms_delta(ktime_get(), t0); + if (waited > 50) + dev_warn(ma->card.dev, + "speaker floor for system sleep waited %lld ms for the protection lock\n", + waited); + if (macaudio_spk_leased(ma)) + macaudio_spk_floor(ma, "system sleep"); + mutex_unlock(&ma->spk_mutex); + macaudio_vlimit_pm(ma, true); + return NOTIFY_OK; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + mutex_lock(&ma->spk_mutex); + WRITE_ONCE(ma->pm_quiesced, false); + if (ma->spk_owner) + ma->spk_lapsed = true; + mutex_unlock(&ma->spk_mutex); + macaudio_vlimit_pm(ma, false); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +/* + * The "Speaker Protection Interface" switch. + * + * At every card bind "Speaker Volume Unlock" is in charge, exactly as on a + * kernel without the lease. A CAP_SYS_ADMIN write of "Speaker Protection" + * hands the speakers to the lease, through the floor of both: the volume + * lock's ceiling first, then no speaker start, the running streams stopped, + * the lease's safe level, the amps' own gains fixed at their limit, and only + * then the ceiling lifted, with every volume still at the safe level. A + * write of "Speaker Volume Unlock", or SPEAKER_LEASE_UNOWNED_MS without an + * owner of "Speaker Protection", hands them back through the lease's floor: + * the volumes stay at the safe level under the lock's ceiling, and go up + * only once a daemon holds the lock and pings it, as after a cold bind. Its + * first ping after the switch fails with -ETIMEDOUT: the lock was lost. + * + * While leased the lock's ping fails with -EBUSY, and the speaker volumes + * and amp gains are the driver's: userspace writes fail with -EPERM. + * + * The switch is state for the boot, not a mixer setting: INACTIVE keeps + * alsactl off it. Whatever makes it persist (the protection engine's + * service) writes it again at every boot. + */ + +static const char * const macaudio_spk_iface_texts[] = { + [MACAUDIO_SPK_IFACE_UNLOCK] = "Speaker Volume Unlock", + [MACAUDIO_SPK_IFACE_LEASE] = "Speaker Protection", +}; + +static void macaudio_spk_iface_notify(struct macaudio_snd_data *ma) +{ + if (ma->spk_dead) + return; + if (ma->spk_iface_kctl) + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->spk_iface_kctl->id); + if (ma->speaker_lock_kctl) + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_lock_kctl->id); + if (ma->spk_kctl) + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->spk_kctl->id); +} + +/* The software gain's table top: the full scale only while leased */ +static int macaudio_sw_gain_full_scale(struct macaudio_snd_data *ma, bool full) +{ + if (!ma->sw_gain_component) + return 0; + return snd_soc_component_write(ma->sw_gain_component, + MACAUDIO_SW_GAIN_REG_FULL_SCALE, full); +} + +/* + * Every amp's own gain at its limit, a fixed operating point, as the + * engine's tuning assumes. Returns 0 or a negative error. + * + * Context: spk_mutex held. + */ +static int macaudio_spk_fix_amp_gains(struct macaudio_snd_data *ma) +{ + struct snd_ctl_elem_value *uval = ma->spk_uval; + struct snd_ctl_elem_info *uinfo; + struct snd_kcontrol *kctl; + unsigned int i; + int ret = 0; + + lockdep_assert_held(&ma->spk_mutex); + + if (!ma->spk_nr_amp_gains) + return 0; + if (!uval) + return -EINVAL; + uinfo = kzalloc_obj(*uinfo); + if (!uinfo) + return -ENOMEM; + + for (i = 0; i < ma->spk_nr_amp_gains; i++) { + kctl = ma->spk_amp_gains[i]; + memset(uinfo, 0, sizeof(*uinfo)); + uinfo->id = kctl->id; + ret = kctl->info(kctl, uinfo); + if (ret < 0) + break; + memset(uval, 0, sizeof(*uval)); + uval->id = kctl->id; + uval->value.integer.value[0] = uinfo->value.integer.max; + uval->value.integer.value[1] = uinfo->value.integer.max; + ret = kctl->put(kctl, uval); + if (ret < 0) + break; + if (ret > 0) + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &kctl->id); + ret = 0; + } + if (ret < 0) + dev_err(ma->card.dev, "Failed to fix %s at its limit: %d\n", + kctl->id.name, ret); + kfree(uinfo); + return ret; +} + +/* + * Hand the speakers to "Speaker Protection". + * + * Context: the card's controls_rwsem and spk_mutex held, the card powered. + */ +static int macaudio_spk_enter_lease(struct macaudio_snd_data *ma) +{ + int ret; + + lockdep_assert_held(&ma->spk_mutex); + + if (ma->spk_dead) + return -ENODEV; + if (!ma->has_speakers || !ma->has_safety) + return -ENODEV; + if (READ_ONCE(ma->pm_quiesced)) + return -EBUSY; + if (READ_ONCE(ma->spk_floor_failed) || READ_ONCE(ma->speaker_lock_failed)) + return -EIO; + + /* The volume lock's ceiling first: whatever a daemon had unlocked goes down */ + mutex_lock(&ma->volume_lock_mutex); + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_remain = 0; + macaudio_pm_volumes_drop(ma); + macaudio_vlimit_update(ma); + if (READ_ONCE(ma->speaker_lock_failed)) { + mutex_unlock(&ma->volume_lock_mutex); + return -EIO; + } + /* No speaker start from here on: nothing is granted yet */ + WRITE_ONCE(ma->spk_iface, MACAUDIO_SPK_IFACE_LEASE); + mutex_unlock(&ma->volume_lock_mutex); + + /* What still plays stops before the amps are touched */ + kthread_queue_work(ma->spk_stop_worker, &ma->spk_stop_work); + kthread_flush_work(&ma->spk_stop_work); + + /* The lease's floor; a failed one refuses speaker playback for good */ + macaudio_spk_floor(ma, "interface switched"); + if (READ_ONCE(ma->spk_floor_failed)) + ret = -EIO; + else + ret = macaudio_spk_fix_amp_gains(ma); + if (ret) { + WRITE_ONCE(ma->spk_floor_failed, true); + dev_crit(ma->card.dev, + "speaker floor for the lease not set: speaker playback refused\n"); + macaudio_spk_iface_notify(ma); + return ret; + } + + /* The volumes stay at the safe level: the ceiling no longer applies */ + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, true); + mutex_unlock(&ma->volume_lock_mutex); + ret = macaudio_sw_gain_full_scale(ma, true); + if (ret < 0) + dev_err(ma->card.dev, "software gain full scale not set: %d\n", ret); + + if (!ma->spk_owner) + mod_delayed_work(system_wq, &ma->spk_unowned_work, + msecs_to_jiffies(SPEAKER_LEASE_UNOWNED_MS)); + dev_info(ma->card.dev, + "Speaker protection interface: Speaker Protection (lease); Speaker Volume Unlock refused\n"); + macaudio_spk_iface_notify(ma); + return 0; +} + +/* + * Hand the speakers back to "Speaker Volume Unlock", at its floor. + * + * Context: the card's controls_rwsem and spk_mutex held. + */ +static void macaudio_spk_leave_lease(struct macaudio_snd_data *ma, const char *why) +{ + lockdep_assert_held(&ma->spk_mutex); + + if (!macaudio_spk_leased(ma)) + return; + cancel_delayed_work(&ma->spk_unowned_work); + + /* No start, the stream stopped, every amp at the safe level */ + macaudio_spk_floor_ramped(ma, why); + if (!ma->spk_dead) { + kthread_flush_work(&ma->spk_stop_work); + if (macaudio_sw_gain_full_scale(ma, false) < 0) + WRITE_ONCE(ma->spk_floor_failed, true); + } + + mutex_lock(&ma->volume_lock_mutex); + /* The lock's ceiling over the safe level; it lifts on a fresh ping only */ + if (!ma->spk_dead) + macaudio_vlimit_unlock(ma, false); + ma->speaker_volume_unlocked = false; + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_remain = 0; + ma->speaker_lock_timeout = 0; + macaudio_pm_volumes_drop(ma); + /* A daemon still holding the lock learns that it lost it */ + ma->speaker_volume_was_locked = true; + if (READ_ONCE(ma->spk_floor_failed) && !READ_ONCE(ma->speaker_lock_failed)) { + WRITE_ONCE(ma->speaker_lock_failed, true); + schedule_work(&ma->lock_fail_work); + } + if (ma->cfg->amp == AMP_MAX98360A) { + /* Samples converted at the lease's gain must not play */ + atomic_inc(&ma->sw_gain_lock_epoch); + schedule_work(&ma->sw_gain_stop_work); + } + WRITE_ONCE(ma->spk_iface, MACAUDIO_SPK_IFACE_UNLOCK); + macaudio_vlimit_update(ma); + mutex_unlock(&ma->volume_lock_mutex); + + dev_info(ma->card.dev, "Speaker protection interface: Speaker Volume Unlock (%s)\n", why); + macaudio_spk_iface_notify(ma); +} + +static void macaudio_spk_unowned_work(struct work_struct *work) +{ + struct macaudio_snd_data *ma = container_of(to_delayed_work(work), + struct macaudio_snd_data, + spk_unowned_work); + + /* The same locks as a control write, in the same order */ + down_write(&ma->card.snd_card->controls_rwsem); + mutex_lock(&ma->spk_mutex); + if (!ma->spk_owner && !ma->spk_dead) + macaudio_spk_leave_lease(ma, "no owner of Speaker Protection"); + mutex_unlock(&ma->spk_mutex); + up_write(&ma->card.snd_card->controls_rwsem); +} + +static int macaudio_spk_iface_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(macaudio_spk_iface_texts), + macaudio_spk_iface_texts); +} + +static int macaudio_spk_iface_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uvalue->value.enumerated.item[0] = READ_ONCE(ma->spk_iface); + return 0; +} + +static int macaudio_spk_iface_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + unsigned int item = uvalue->value.enumerated.item[0]; + int ret; + + if (item >= ARRAY_SIZE(macaudio_spk_iface_texts)) + return -EINVAL; + /* The speakers are not a seat user's to hand over */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; - switch (action) { - case PM_SUSPEND_PREPARE: - case PM_HIBERNATION_PREPARE: - /* - * Before anything that can wait, spk_mutex included: a writer - * that holds it may be stuck behind the sleep itself. No grant - * and no start from here on, and the running streams stop. - */ - WRITE_ONCE(ma->pm_quiesced, true); - macaudio_spk_close_gate(ma, "system sleep"); - t0 = ktime_get(); - mutex_lock(&ma->spk_mutex); - waited = ktime_ms_delta(ktime_get(), t0); - if (waited > 50) - dev_warn(ma->card.dev, - "speaker floor for system sleep waited %lld ms for the protection lock\n", - waited); - macaudio_spk_floor(ma, "system sleep"); - mutex_unlock(&ma->spk_mutex); - return NOTIFY_OK; - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - mutex_lock(&ma->spk_mutex); - WRITE_ONCE(ma->pm_quiesced, false); - if (ma->spk_owner) - ma->spk_lapsed = true; - mutex_unlock(&ma->spk_mutex); - return NOTIFY_OK; - default: - return NOTIFY_DONE; + mutex_lock(&ma->spk_mutex); + if (item == READ_ONCE(ma->spk_iface)) { + ret = 0; + } else if (item == MACAUDIO_SPK_IFACE_LEASE) { + ret = macaudio_spk_enter_lease(ma); + if (!ret) + ret = 1; + } else { + macaudio_spk_leave_lease(ma, "switched by the administrator"); + ret = 1; } + mutex_unlock(&ma->spk_mutex); + return ret; } static int macaudio_spk_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) @@ -1027,6 +1903,12 @@ static int macaudio_spk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_v goto out; } + /* "Speaker Volume Unlock" owns the speakers: nothing to grant or release */ + if (!macaudio_spk_leased(ma)) { + ret = -EBUSY; + goto out; + } + if (!ms) { ma->spk_lapsed = false; macaudio_spk_floor_ramped(ma, "released by the owner"); @@ -1103,7 +1985,10 @@ static int macaudio_spk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file ma->spk_owner = owner; ma->spk_lapsed = false; macaudio_spk_xover_clear(ma); - macaudio_spk_floor_ramped(ma, "new owner"); + if (macaudio_spk_leased(ma)) { + cancel_delayed_work(&ma->spk_unowned_work); + macaudio_spk_floor_ramped(ma, "new owner"); + } mutex_unlock(&ma->spk_mutex); return 0; } @@ -1117,7 +2002,13 @@ static void macaudio_spk_unlock(struct snd_kcontrol *kcontrol) ma->spk_owner = NULL; ma->spk_lapsed = false; macaudio_spk_xover_clear(ma); - macaudio_spk_floor_ramped(ma, "owner gone"); + if (macaudio_spk_leased(ma)) { + macaudio_spk_floor_ramped(ma, "owner gone"); + /* A restarting engine takes it back in time; a dead one does not */ + if (!ma->spk_dead) + mod_delayed_work(system_wq, &ma->spk_unowned_work, + msecs_to_jiffies(SPEAKER_LEASE_UNOWNED_MS)); + } mutex_unlock(&ma->spk_mutex); } @@ -1205,6 +2096,8 @@ static const struct snd_kcontrol_new macaudio_spk_xover_control = { /* ALSA serializes this with protection lock/unlock, including file close. */ static int macaudio_sense_owner_check(struct macaudio_snd_data *ma, struct snd_ctl_file *file) { + if (!macaudio_spk_leased(ma)) + return 0; return ma->spk_owner && ma->spk_owner != file ? -EPERM : 0; } @@ -1255,8 +2148,21 @@ static void macaudio_protect_sense(struct snd_soc_card *card) } /* - * The speaker volumes are the driver's: keep them, and take the write access - * away from userspace before the card is exposed. + * Speaker volumes and amp gains: the volume lock's daemon sets them under + * "Speaker Volume Unlock"; while leased they are the driver's alone. + */ +static int macaudio_spk_owned_write_check(struct snd_kcontrol *kctl, + struct snd_ctl_file *file) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kctl); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(component->card); + + return macaudio_spk_leased(ma) ? -EPERM : 0; +} + +/* + * Keep the speaker volumes, which the lease sets, and refuse userspace + * writes to them while leased (macaudio_spk_owned_write_check()). * * Context: before the card registers, or with the card's controls held. */ @@ -1265,7 +2171,7 @@ static int macaudio_spk_take_volumes(struct macaudio_snd_data *ma) const char *name = volume_control_names[ma->cfg->amp]; struct device *dev = ma->card.dev; struct snd_kcontrol *kctl; - unsigned int i, nr = 0; + unsigned int nr = 0; if (!name) return -EINVAL; @@ -1284,13 +2190,51 @@ static int macaudio_spk_take_volumes(struct macaudio_snd_data *ma) list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { if (!snd_soc_control_matches(kctl, name)) continue; - for (i = 0; i < kctl->count; i++) - kctl->vd[i].access &= ~SNDRV_CTL_ELEM_ACCESS_WRITE; + kctl->write_check = macaudio_spk_owned_write_check; ma->spk_vols[ma->spk_nr_vols++] = kctl; } return 0; } +static bool macaudio_is_amp_gain(const struct macaudio_snd_data *ma, + struct snd_kcontrol *kctl) +{ + if (ma->cfg->amp != AMP_TAS5770 && ma->cfg->amp != AMP_SN012776) + return false; + return snd_soc_control_matches(kctl, "* Amp Gain Volume") || + !strcmp(kctl->id.name, "Amp Gain Volume"); +} + +/* + * The amps' own gains ("Amp Gain Volume"): limited at fixup, theirs to set + * under "Speaker Volume Unlock", fixed at the limit while leased. + * + * Context: before the card registers. + */ +static int macaudio_spk_take_amp_gains(struct macaudio_snd_data *ma) +{ + struct device *dev = ma->card.dev; + struct snd_kcontrol *kctl; + unsigned int nr = 0; + + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) + if (macaudio_is_amp_gain(ma, kctl)) + nr++; + if (!nr) + return 0; + + ma->spk_amp_gains = devm_kcalloc(dev, nr, sizeof(*ma->spk_amp_gains), GFP_KERNEL); + if (!ma->spk_amp_gains) + return -ENOMEM; + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if (!macaudio_is_amp_gain(ma, kctl)) + continue; + kctl->write_check = macaudio_spk_owned_write_check; + ma->spk_amp_gains[ma->spk_nr_amp_gains++] = kctl; + } + return 0; +} + /* The item of enum control @kctl named @text, or a negative error */ static int macaudio_enum_item(struct snd_kcontrol *kctl, const char *text) { @@ -1463,11 +2407,11 @@ static bool macaudio_sn012776_levels_present(struct device *dev, return n > 0; } -static int macaudio_set_fixed_volume(struct snd_soc_card *card, const char *name, - int value); - -/* Every SN012776's "Amp Gain Volume" is fixed at its own level */ -static int macaudio_fix_sn012776_levels(struct snd_soc_card *card) +/* + * Every SN012776's "Amp Gain Volume" ends at its own level; the lease fixes + * it there (macaudio_spk_fix_amp_gains()). + */ +static int macaudio_limit_sn012776_levels(struct snd_soc_card *card) { struct snd_soc_component *component; char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; @@ -1484,10 +2428,10 @@ static int macaudio_fix_sn012776_levels(struct snd_soc_card *card) snprintf(name, sizeof(name), "%s%sAmp Gain Volume", component->name_prefix ?: "", component->name_prefix ? " " : ""); - ret = macaudio_set_fixed_volume(card, name, level); + ret = snd_soc_limit_volume(card, name, level); if (ret != 1) return dev_err_probe(card->dev, ret < 0 ? ret : -EINVAL, - "fixed level on '%s': %d\n", name, ret); + "limit on '%s': %d\n", name, ret); } return 0; } @@ -1660,6 +2604,9 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) link_props->bclk_ratio = ma->cfg->primary_bclk_ratio; if (i == 1 && ma->cfg->secondary_bclk_ratio) link_props->bclk_ratio = ma->cfg->secondary_bclk_ratio; + if (ma->cfg->sw_gain_dai) + link_props->sw_gain = !strcmp(link->cpus[0].dai_name, + ma->cfg->sw_gain_dai); link++; link_props++; } @@ -1907,6 +2854,14 @@ static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, rate->min = rate->max = cpu_dai->symmetric_rate; return 0; + } else { + /* + * Set the sense PCM rate control to inform userspace of the + * new sample rate. + */ + ma->speaker_sample_rate = params_rate(params); + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); } } @@ -2014,6 +2969,11 @@ static int macaudio_be_hw_free(struct snd_pcm_substream *substream) for_each_rtd_dais(rtd, i, dai) { dai->symmetric_rate = 0; } + + /* Notify userspace that the speakers are closed */ + ma->speaker_sample_rate = 0; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); } return 0; @@ -2042,7 +3002,7 @@ static bool macaudio_fe_refuses(struct macaudio_snd_data *ma, struct snd_soc_pcm struct snd_pcm_substream *substream, int cmd) { if (!ma->has_speakers || substream->stream != SNDRV_PCM_STREAM_PLAYBACK || - !macaudio_trigger_starts(cmd)) + !macaudio_trigger_starts(cmd) || !macaudio_spk_leased(ma)) return false; return macaudio_fe_drives_speakers(ma, rtd, substream->stream) && !macaudio_spk_admitted(ma); @@ -2063,12 +3023,57 @@ static int macaudio_spk_refuse(struct macaudio_snd_data *ma) * macaudio_quiet_refusals()), so ASoC fails these triggers without logging * them; each refusal is counted instead, and the next grant reports the run. */ +static int macaudio_fe_prepare(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && + READ_ONCE(ma->speaker_lock_failed)) + return -EIO; + + if (!macaudio_sw_gain_pcm(substream)) + return 0; + + if (atomic_read(&ma->sw_gain_limit_failed)) + return -EIO; + /* The CPU DAI's prepare clears the old samples out of the buffer next */ + atomic_set(&ma->sw_gain_prepared_epoch, + atomic_read(&ma->sw_gain_lock_epoch)); + return 0; +} + +/* + * Runs before the CPU DAI and the back-ends are triggered (see the + * front-end links). While leased, a speaker start without a grant is + * refused. Under the volume lock, a lock event since prepare means the + * buffer holds samples converted at the old gain, so the start is refused + * and the stream has to be prepared again. + */ static int macaudio_fe_trigger(struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); - return macaudio_fe_refuses(ma, rtd, substream, cmd) ? macaudio_spk_refuse(ma) : 0; + if (macaudio_fe_refuses(ma, rtd, substream, cmd)) + return macaudio_spk_refuse(ma); + + /* + * A prepare can pass just before a failed lock is recorded and the + * stop work then find the stream not yet prepared: refuse its start. + */ + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && + READ_ONCE(ma->speaker_lock_failed) && macaudio_trigger_starts(cmd)) + return -EIO; + + if (!macaudio_sw_gain_pcm(substream) || !macaudio_trigger_starts(cmd)) + return 0; + + if (atomic_read(&ma->sw_gain_limit_failed) || + atomic_read(&ma->sw_gain_prepared_epoch) != + atomic_read(&ma->sw_gain_lock_epoch)) + return -EPIPE; + return 0; } /* The gate's -EPERM on every link of a card with speakers is a quiet refusal */ @@ -2099,20 +3104,29 @@ static int macaudio_be_trigger(struct snd_pcm_substream *substream, int cmd) case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - return macaudio_spk_admitted(ma) ? 0 : macaudio_spk_refuse(ma); + if (macaudio_spk_leased(ma) && !macaudio_spk_admitted(ma)) + return macaudio_spk_refuse(ma); + ma->bes_active |= BIT(rtd->dai_link->id); + break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: case SNDRV_PCM_TRIGGER_STOP: - return 0; + ma->bes_active &= ~BIT(rtd->dai_link->id); + break; default: return -EINVAL; } + + /* The volume lock's lease runs only while the speakers play */ + schedule_work(&ma->lock_update_work); + return 0; } static const struct snd_soc_ops macaudio_fe_ops = { .startup = macaudio_fe_startup, .shutdown = macaudio_dpcm_shutdown, .hw_params = macaudio_fe_hw_params, + .prepare = macaudio_fe_prepare, .trigger = macaudio_fe_trigger, }; @@ -2417,10 +3431,20 @@ static int macaudio_late_probe(struct snd_soc_card *card) } } - /* The hardware floor holds from here on, before userspace sees the card */ + /* + * The volume lock's floor holds from here on, before userspace sees the + * card: "Speaker Volume Unlock" is in charge at every bind. + */ if (ma->has_speakers) { + ma->speaker_sample_rate_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Sample Rate"); + ma->speaker_lock_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Volume Unlock"); ma->spk_kctl = snd_soc_card_get_kcontrol(card, "Speaker Protection"); - if (!ma->spk_kctl) + ma->spk_iface_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Protection Interface"); + if (!ma->speaker_sample_rate_kctl || !ma->speaker_lock_kctl || + !ma->spk_kctl || !ma->spk_iface_kctl) return -EINVAL; ret = macaudio_spk_take_volumes(ma); if (ret) @@ -2430,12 +3454,25 @@ static int macaudio_late_probe(struct snd_soc_card *card) if (ret) return dev_err_probe(card->dev, ret, "tweeter high-pass controls missing\n"); - mutex_lock(&ma->spk_mutex); - macaudio_spk_floor(ma, "card bound"); - ret = ma->spk_floor_failed ? -EIO : 0; - mutex_unlock(&ma->spk_mutex); + ret = macaudio_spk_take_amp_gains(ma); + if (ret) + return ret; + for_each_card_rtds(card, rtd) { + if (!rtd->dai_link->no_pcm && + ma->link_props[rtd->dai_link->id].sw_gain) + ma->sw_gain_component = snd_soc_rtd_to_cpu(rtd, 0)->component; + } + /* The volume table's top under the lock: the legacy ceiling */ + ret = macaudio_sw_gain_full_scale(ma, false); + if (ret < 0) + return dev_err_probe(card->dev, ret, "software gain ceiling not set\n"); + + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, false); + ret = READ_ONCE(ma->speaker_lock_failed) ? -EIO : 0; + mutex_unlock(&ma->volume_lock_mutex); if (ret) - return dev_err_probe(card->dev, ret, "speaker safe level not set\n"); + return dev_err_probe(card->dev, ret, "speaker volume lock not set\n"); #ifdef CONFIG_DEBUG_FS debugfs_create_atomic_t("speaker_starts_refused", 0444, card->debugfs_card_root, &ma->spk_refused); @@ -2484,6 +3521,16 @@ static int macaudio_card_remove(struct snd_soc_card *card) hrtimer_cancel(&ma->spk_lease_timer); kthread_cancel_work_sync(&ma->spk_expire_work); kthread_cancel_work_sync(&ma->spk_stop_work); + cancel_delayed_work_sync(&ma->spk_unowned_work); + /* + * No back-end can trigger any more, but a lock update it queued takes + * the card's controls_rwsem: finish it while the snd_card still exists. + */ + cancel_work_sync(&ma->lock_update_work); + cancel_delayed_work_sync(&ma->lock_timeout_work); + mutex_lock(&ma->volume_lock_mutex); + macaudio_pm_volumes_drop(ma); + mutex_unlock(&ma->volume_lock_mutex); return 0; } @@ -2523,48 +3570,6 @@ static int macaudio_set_fixed_enum(struct snd_soc_card *card, const char *name, return ret; } -/* - * Hold the volumes matching @name at @value, their maximum, and take them - * away from userspace: a fixed operating point, as the protection engine's - * tuning assumes. Returns the number of controls, as snd_soc_limit_volume(). - * - * Called during control fixup, before the card is exposed to userspace. - */ -static int macaudio_set_fixed_volume(struct snd_soc_card *card, const char *name, - int value) -{ - struct snd_ctl_elem_value *uval; - struct snd_kcontrol *kctl; - int ret, hits; - - hits = snd_soc_limit_volume(card, name, value); - if (hits < 0) - return hits; - - uval = kzalloc_obj(*uval); - if (!uval) - return -ENOMEM; - list_for_each_entry(kctl, &card->snd_card->controls, list) { - if (!snd_soc_control_matches(kctl, name)) - continue; - memset(uval, 0, sizeof(*uval)); - uval->id = kctl->id; - uval->value.integer.value[0] = value; - uval->value.integer.value[1] = value; - ret = kctl->put(kctl, uval); - if (ret < 0) { - hits = ret; - break; - } - } - kfree(uval); - if (hits < 0) - return hits; - - macaudio_deny_write(card, name); - return hits; -} - #define CHECK(call, pattern, value, min) \ { \ int ret = call(card, pattern, value); \ @@ -2598,15 +3603,15 @@ static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, b if (ma->cfg->stereo) CHECK_CONCAT(macaudio_set_fixed_enum, "ASI1 Sel", "Left"); - CHECK_CONCAT(macaudio_set_fixed_volume, "Amp Gain Volume", ma->cfg->amp_gain); + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); break; case AMP_SN012776: if (ma->cfg->stereo) CHECK_CONCAT(macaudio_set_fixed_enum, "ASI1 Sel", "Left"); /* - * Each amp's gain is fixed at its level from the Mac's ADT - * (macaudio_fix_sn012776_levels()). The tweeters' playback + * Each amp's gain ends at its level from the Mac's ADT + * (macaudio_limit_sn012776_levels()). The tweeters' playback * high-pass starts at the protection floor; the woofers keep * the corner the amps were given from the ADT. */ @@ -2662,7 +3667,7 @@ static int macaudio_fixup_controls(struct snd_soc_card *card) macaudio_protect_sense(card); if (ma->cfg->amp == AMP_SN012776) { - int ret = macaudio_fix_sn012776_levels(card); + int ret = macaudio_limit_sn012776_levels(card); if (ret) return ret; @@ -2737,13 +3742,184 @@ static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { SND_SOC_DAPM_AIF_IN("Speaker Sense Capture", NULL, 0, SND_SOC_NOPM, 0, 0), }; +static int macaudio_sss_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = 192000; + + return 0; +} + +static int macaudio_sss_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + /* + * TODO: Check if any locking is in order here. I would + * assume there is some ALSA-level lock, but DAPM implementations + * of kcontrol ops do explicit locking, so look into it. + */ + uvalue->value.integer.value[0] = ma->speaker_sample_rate; + + return 0; +} + +static int macaudio_slk_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = INT_MIN; + uinfo->value.integer.max = INT_MAX; + + return 0; +} + +static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + long val = uvalue->value.integer.value[0]; + + /* + * Reads return the lock state (0 or 1), so a saved mixer state holds + * one of those. Writing it back changes nothing: only the lock owner's + * magic value renews the lease. Accept it as a no-op so that restoring + * a saved state before the daemon has taken the lock is not an error. + */ + if (val == 0 || val == 1) + return 0; + + if (val != SPEAKER_MAGIC_VALUE) + return -EINVAL; + + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_owner) { + mutex_unlock(&ma->volume_lock_mutex); + return -EPERM; + } + + /* "Speaker Protection" owns the speakers: no unlock to renew */ + if (macaudio_spk_leased(ma)) { + mutex_unlock(&ma->volume_lock_mutex); + return -EBUSY; + } + + /* Serves as a notification that the lock was lost at some point */ + if (ma->speaker_volume_was_locked) { + ma->speaker_volume_was_locked = false; + mutex_unlock(&ma->volume_lock_mutex); + return -ETIMEDOUT; + } + + /* Between suspend-prepare and the freeze: the lease starts again after the resume */ + if (ma->pm_quiesced) { + mutex_unlock(&ma->volume_lock_mutex); + return 0; + } + + cancel_delayed_work(&ma->lock_timeout_work); + + ma->speaker_lock_remain = ms_to_ktime(SPEAKER_LOCK_TIMEOUT); + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + macaudio_vlimit_update(ma); + + if (ma->speaker_lock_timeout_enabled) { + dev_dbg(ma->card.dev, "Volume limit timeout ping: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + } + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static int macaudio_slk_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uvalue->value.integer.value[0] = ma->speaker_volume_unlocked ? 1 : 0; + + return 0; +} + +static int macaudio_slk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file *owner) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + mutex_lock(&ma->volume_lock_mutex); + + /* + * A lease belongs to the owner that pinged for it. Whatever the + * previous owner had left must not unlock the volume for a new one + * before it has pinged itself. + */ + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_remain = 0; + ma->speaker_lock_timeout = 0; + /* A new owner sets the volumes itself */ + macaudio_pm_volumes_drop(ma); + + ma->speaker_lock_owner = owner; + macaudio_vlimit_update(ma); + + /* + * Reset the unintended lock flag when the control is first locked. + * At this point the state is locked and cannot be unlocked until + * userspace writes to this control, so this cannot spuriously become + * true again until that point. + */ + ma->speaker_volume_was_locked = false; + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + mutex_lock(&ma->volume_lock_mutex); + cancel_delayed_work(&ma->lock_timeout_work); + ma->speaker_lock_owner = NULL; + ma->speaker_lock_remain = 0; + ma->speaker_lock_timeout = 0; + macaudio_vlimit_update(ma); + mutex_unlock(&ma->volume_lock_mutex); +} + /* The speaker controls go last: a card without speakers drops them. */ -#define MACAUDIO_NUM_SPEAKER_CONTROLS 1 +#define MACAUDIO_NUM_SPEAKER_CONTROLS 4 static const struct snd_kcontrol_new macaudio_controls[] = { SOC_DAPM_PIN_SWITCH("Speaker"), SOC_DAPM_PIN_SWITCH("Headphone"), SOC_DAPM_PIN_SWITCH("Headset Mic"), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Sample Rate", + .info = macaudio_sss_info, .get = macaudio_sss_get, + }, + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_WRITE | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Volume Unlock", + .info = macaudio_slk_info, + .put = macaudio_slk_put, .get = macaudio_slk_get, + .lock = macaudio_slk_lock, .unlock = macaudio_slk_unlock, + }, { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .access = MACAUDIO_SPK_OWNERSHIP_ACCESS, @@ -2752,6 +3928,13 @@ static const struct snd_kcontrol_new macaudio_controls[] = { .put = macaudio_spk_put, .get = macaudio_spk_get, .lock = macaudio_spk_lock, .unlock = macaudio_spk_unlock, }, + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = MACAUDIO_SPK_OWNERSHIP_ACCESS, + .name = "Speaker Protection Interface", + .info = macaudio_spk_iface_info, + .put = macaudio_spk_iface_put, .get = macaudio_spk_iface_get, + }, }; static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { @@ -2803,8 +3986,9 @@ struct macaudio_platform_cfg macaudio_j313_cfg = { * launched on the rising SCLK edge: DSP_A with an inverted bit clock. The * speakers are one MAX98360A per channel on the "spkr" back-end, with no * sense lines and no amplifier controls; the volume is the CPU DAI's - * software gain: full scale while a protection engine owns the speakers, - * 20 dB down otherwise. The "Speaker Sense" capture is the SpeakerTap, a + * software gain, which the volume lock holds 20 dB down until a protection + * daemon takes it, and the lease 20 dB down below the full scale until its + * engine is granted. The "Speaker Sense" capture is the SpeakerTap, a * digital tap of the words the amplifier receives; there is no V/I sense. */ struct macaudio_platform_cfg macaudio_j700_cfg = { @@ -2814,6 +3998,7 @@ struct macaudio_platform_cfg macaudio_j700_cfg = { .stereo = true, .safe_vol = -20, .spk_out_widget = "Speaker", + .sw_gain_dai = "j700-pcm-1", .fe_dai_names = { "j700-pcm-0", "j700-pcm-1", "j700-pcm-2" }, .dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBC_CFC | SND_SOC_DAIFMT_IB_NF, .primary_bclk_ratio = 125, @@ -3096,6 +4281,7 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) snd_soc_card_set_drvdata(card, data); dev_set_drvdata(&pdev->dev, data); mutex_init(&data->spk_mutex); + mutex_init(&data->volume_lock_mutex); card->owner = THIS_MODULE; card->driver_name = "macaudio"; @@ -3142,10 +4328,12 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) card->num_dapm_routes--; /* Admitted speakers always have a safe level (macaudio_speakers_allowed()) */ - if (!data->has_speakers) + if (!data->has_speakers) { card->num_controls -= MACAUDIO_NUM_SPEAKER_CONTROLS; - else + } else { + data->has_safety = true; macaudio_quiet_refusals(card); + } for_each_card_prelinks(card, i, link) { if (link->no_pcm) { @@ -3158,6 +4346,15 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) } } + INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); + INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); + INIT_WORK(&data->sw_gain_stop_work, macaudio_sw_gain_stop_work); + INIT_WORK(&data->lock_fail_work, macaudio_lock_fail_work); + atomic_set(&data->sw_gain_lock_epoch, 0); + atomic_set(&data->sw_gain_prepared_epoch, -1); + atomic_set(&data->sw_gain_limit_failed, 0); + data->spk_iface = MACAUDIO_SPK_IFACE_UNLOCK; + ret = macaudio_spk_init(data); if (ret) return dev_err_probe(dev, ret, "speaker protection workers not started\n"); @@ -3174,6 +4371,11 @@ static void macaudio_snd_platform_remove(struct platform_device *pdev) /* The card's removal stops the lease timer and its works; devm frees them */ snd_soc_unregister_card(&ma->card); + cancel_work_sync(&ma->lock_update_work); + cancel_delayed_work_sync(&ma->lock_timeout_work); + cancel_work_sync(&ma->sw_gain_stop_work); + cancel_work_sync(&ma->lock_fail_work); + cancel_delayed_work_sync(&ma->spk_unowned_work); } static struct platform_driver macaudio_snd_driver = { @@ -3195,6 +4397,8 @@ MODULE_LICENSE("GPL"); #include "macaudio-admission-test.h" #include "macaudio-protection-test.h" #include "macaudio-hpf-test.h" +#include "macaudio-pm-volumes-test.h" +#include "macaudio-iface-test.h" #if IS_ENABLED(CONFIG_OF_OVERLAY) #include "macaudio-amp-level-test.h" #endif diff --git a/tools/testing/selftests/alsa/macaudio-fixed-controls.py b/tools/testing/selftests/alsa/macaudio-fixed-controls.py index 12a94f6c201c3..5ca87d990c11f 100644 --- a/tools/testing/selftests/alsa/macaudio-fixed-controls.py +++ b/tools/testing/selftests/alsa/macaudio-fixed-controls.py @@ -74,8 +74,10 @@ def function(source, name): struct snd_ctl_file *spk_owner; struct snd_soc_card card; struct snd_kcontrol **spk_hpfs; unsigned int *spk_hpf_granted; unsigned int spk_nr_hpfs, spk_hpf_safe; struct snd_ctl_elem_value *spk_uval; - bool spk_granted; + bool spk_granted; int spk_iface; }; +/* MACAUDIO_SPK_IFACE_LEASE: the sense switches are the lease owner's */ +static bool macaudio_spk_leased(const struct macaudio_snd_data *ma) { return ma->spk_iface == 1; } #define GFP_KERNEL 0 #define devm_kcalloc(d, n, s, g) calloc(n, s) #define kzalloc_obj(o) calloc(1, sizeof(o)) @@ -112,8 +114,8 @@ def function(source, name): static unsigned int snd_soc_enum_item_to_val(struct soc_enum *e, unsigned int item) { return item; } static int snd_soc_component_update_bits(struct snd_soc_component *c, unsigned int r, unsigned int m, unsigned int v) { unsigned int old = c->reg; c->writes++; c->reg = (old & ~m) | (v & m); return old != c->reg; } -/* Amp gain is a fixed volume (KUnit apple-speaker-protection), not a fixed enum. */ -static int macaudio_set_fixed_volume(struct snd_soc_card *c, const char *n, int v) { return 1; } +/* Amp gain is volume-limited (KUnit apple-speaker-protection), not a fixed enum. */ +static int snd_soc_limit_volume(struct snd_soc_card *c, const char *n, int v) { return 1; } static int enum_info(struct snd_kcontrol *k, struct snd_ctl_elem_info *i) { struct soc_enum *e = (void *)k->private_value; @@ -211,7 +213,10 @@ def function(source, name): v.id.index = j; ma.spk_owner = NULL; check(snd_ctl_elem_write(&card, &other, &v) == 0, "unowned writable", names[i]); - ma.spk_owner = &owner; + /* Under the volume lock a lease owner owns nothing */ + ma.spk_owner = &owner; ma.spk_iface = 0; + check(snd_ctl_elem_write(&card, &other, &v) == 0, "lease owner without the lease", names[i]); + ma.spk_iface = 1; unsigned int before = regs[i].writes; int rc = snd_ctl_elem_write(&card, &other, &v); check(rc == (i < 6 ? -EPERM : 0), "other file during ownership", names[i]); @@ -223,7 +228,7 @@ def function(source, name): controls[i].vd[j].owner = NULL; ma.spk_owner = NULL; check(snd_ctl_elem_write(&card, &other, &v) == 0, "after owner close", names[i]); - attempts += 5; + attempts += 6; } } /* The speaker route is fixed while protection is granted, for every file. */ From 55facfacd40df9bd8979759566a16983f8337b95 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:11:39 +1100 Subject: [PATCH 130/392] pmdomain: apple: no PS reset recovery during system sleep A power-on that stops with bit 11 latched is retried after a PS reset pulse while the domain is gated (commit a10defefab5d). The genpd noirq resume runs that power-on with the domain's DARTs and engine still in their noirq phase, and on the T8140 the recovery there hangs the machine. In a suspend loop on the J700 (12 systemd and 12 direct pm_test=platform suspends per boot, from a fresh boot), kernels with the recovery hung on 3 of 3 boots. The hw watchdog brought each one back, and no "cleared by a PS reset" line ever reached the disk. Kernels without it (fleet36, and the same tree with the recovery reverted) hung on none of 6 boots. Two of those boots latched ANE_SYS, quarantined the ANE and kept going. The freezer and devices test levels, which never power a domain off, hung on none of 6 boots with the recovery in place. This is the J700 "hang on suspend entry" seen in fleet43 suites and on aneres2. Skip the recovery while the domain is in a system sleep transition (from its suspend_late to its resume_early). The power-on then fails as before #1255. genpd ignores the error in that phase and keeps the domain requested on, and the ANE quarantines at PM_POST_SUSPEND as it did before. Runtime power-ons, including apple_pmgr_ps_cycle(), keep the recovery. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/pmdomain/apple/pmgr-pwrstate-test.c | 17 +++++++++++++++-- drivers/pmdomain/apple/pmgr-pwrstate.c | 13 +++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate-test.c b/drivers/pmdomain/apple/pmgr-pwrstate-test.c index b3a8cd1f5715c..67698ccfc74aa 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate-test.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate-test.c @@ -296,7 +296,11 @@ static void pmgr_ps_power_on_latch_cleared_test(struct kunit *test) KUNIT_EXPECT_EQ(test, r->auto_writes, 1U); } -/* The same in the noirq phase of a system sleep, where genpd ignores errors. */ +/* + * Not in the noirq phase of a system sleep (no PS reset there: it hangs the + * T8140). The power-on fails, which genpd ignores in that phase, and the + * domain stays requested on. + */ static void pmgr_ps_power_on_latch_sleep_test(struct kunit *test) { struct pmgr_ps_test_reg *r; @@ -305,9 +309,18 @@ static void pmgr_ps_power_on_latch_sleep_test(struct kunit *test) ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, true, &r); ps->system_sleep = true; r->latched = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, r->reset_writes, 0U); + KUNIT_EXPECT_FALSE(test, r->val & APPLE_PMGR_PS_RESET); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val), APPLE_PMGR_PS_CLKGATE); + KUNIT_EXPECT_EQ(test, r->auto_writes, 0U); + + /* The same domain after the sleep: the recovery runs again. */ + ps->system_sleep = false; KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), 0); + KUNIT_EXPECT_EQ(test, r->reset_writes, 1U); KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val), APPLE_PMGR_PS_ACTIVE); - KUNIT_EXPECT_EQ(test, r->auto_writes, 0U); } /* A latch the reset does not clear still fails, and the domain is requested off. */ diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 0c825a64b42bd..688842390a6a2 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -94,14 +94,23 @@ static int apple_pmgr_ps_wait_parent(struct apple_pmgr_ps *ps, u32 *cur) * their state there anyway), then request @reg, the on state, again. * Domains whose bit 11 means a parent that is not up yet (wait_parent) and * those with their own state contract are left alone. + * + * Never during a system sleep transition: there the power-on comes from the + * genpd noirq resume, with the DARTs and the engine of the domain still in + * their noirq phase. On the T8140 a recovery at that point hangs the machine: + * in a suspend loop on the J700, kernels with it hung on 3 of 3 boots and came + * back only through the watchdog, while kernels without it hung on none of 6, + * two of which latched ANE_SYS and carried on with the ANE quarantined. The + * power-on then fails as it did before, and genpd, which ignores errors in + * that phase, keeps the domain requested on. */ static int apple_pmgr_ps_clear_latch(struct apple_pmgr_ps *ps, u32 reg, u32 *cur) { u32 off = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_PS_PWRGATE; int ret; - if (ps->wait_parent || ps->strict_state || ps->externally_clocked || - !(*cur & APPLE_PMGR_BUSY) || + if (ps->system_sleep || ps->wait_parent || ps->strict_state || + ps->externally_clocked || !(*cur & APPLE_PMGR_BUSY) || FIELD_GET(APPLE_PMGR_PS_ACTUAL, *cur) == APPLE_PMGR_PS_ACTIVE) return -ETIMEDOUT; From 12a3560110366df3d3ebdf81bc676e27d2d53d78 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:55:55 +1100 Subject: [PATCH 131/392] dmaengine: apple-sio: configure DisplayPort audio channels as macOS does macOS 26.6 writes a six-word record for a DisplayPort audio channel: [2, 0, 0x800, 0x600, 0x800, 0] on T6041 and [2, 0, 0x800, 0x800, 0x800, 0] on T8140. The T6041 backend sent 5 as the limit: that is the host's mailbox call timeout, not part of the record. Send 0x800, as on the earlier SoCs. Add the T8140 (MacBook Neo) SIO: its firmware reports protocol 10 and takes the T6041 boot tables and driver-owned arrays (its ADT describes the same map-range, DMA shim and device-type tables key for key); its DisplayPort audio channels refill at 0x800, as its ADT dp-audioN records say. KUnit apple-sio-t6041: the T6041 and T8140 records, and the T8140 match. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/dma/apple-sio-t6041-test.c | 22 +++++++++++++++++++++- drivers/dma/apple-sio.c | 24 +++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/drivers/dma/apple-sio-t6041-test.c b/drivers/dma/apple-sio-t6041-test.c index 1c9b1a56f8ae7..78b1f7419fc52 100644 --- a/drivers/dma/apple-sio-t6041-test.c +++ b/drivers/dma/apple-sio-t6041-test.c @@ -217,7 +217,9 @@ static void sio_t6041_config_test(struct kunit *test) .datashape = 2, .timeout = 0, .fifo = 0x800, .threshold = sio_hw_t6041.fifo_threshold, .limit = sio_hw_t6041.cfg_word4, }; - static const u32 t6041[] = { 2, 0, 0x800, 0x600, 5, 0 }; + /* macOS 26.6 DisplayPort audio records */ + static const u32 t6041[] = { 2, 0, 0x800, 0x600, 0x800, 0 }; + static const u32 t8140[] = { 2, 0, 0x800, 0x800, 0x800, 0 }; u32 words[8]; memset(words, 0xa5, sizeof(words)); @@ -225,6 +227,15 @@ static void sio_t6041_config_test(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, words, t6041, sizeof(t6041)); KUNIT_EXPECT_EQ(test, words[6], 0xa5a5a5a5U); + memset(words, 0xa5, sizeof(words)); + cfg.threshold = sio_hw_t8140.fifo_threshold; + cfg.limit = sio_hw_t8140.cfg_word4; + sio_write_config(&sio_hw_t8140, words, &cfg); + KUNIT_EXPECT_MEMEQ(test, words, t8140, sizeof(t8140)); + KUNIT_EXPECT_EQ(test, words[6], 0xa5a5a5a5U); + cfg.threshold = sio_hw_t6041.fifo_threshold; + cfg.limit = sio_hw_t6041.cfg_word4; + /* The other layouts keep their five words and leave the sixth alone. */ memset(words, 0xa5, sizeof(words)); cfg.threshold = sio_hw_t6030.fifo_threshold; @@ -244,6 +255,15 @@ static void sio_t6041_match_test(struct kunit *test) break; KUNIT_ASSERT_TRUE(test, id->compatible[0] != '\0'); KUNIT_EXPECT_PTR_EQ(test, id->data, (const void *)&sio_hw_t6041); + /* T8140 takes the T6041 tables, with its own channel record */ + for (id = sio_of_match; id->compatible[0]; id++) + if (!strcmp(id->compatible, "apple,t8140-sio")) + break; + KUNIT_ASSERT_TRUE(test, id->compatible[0] != '\0'); + KUNIT_EXPECT_PTR_EQ(test, id->data, (const void *)&sio_hw_t8140); + KUNIT_EXPECT_EQ(test, sio_hw_t8140.protocol, 10U); + KUNIT_EXPECT_PTR_EQ(test, sio_hw_t8140.arrays, &sio_t6041_arrays); + KUNIT_EXPECT_TRUE(test, sio_hw_t8140.check_boot_tables == sio_t6041_check_boot_tables); KUNIT_EXPECT_EQ(test, sio_hw_t6041.protocol, 10U); KUNIT_EXPECT_TRUE(test, sio_hw_t6041.clear_period_desc); KUNIT_EXPECT_PTR_EQ(test, sio_hw_t6041.arrays, &sio_t6041_arrays); diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 94c48dffb983b..e1a8ec545b8c6 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -621,10 +621,31 @@ static const struct sio_hw_data sio_hw_t6030 = { .check_boot_tables = sio_t6030_check_boot_tables, }; +/* + * macOS 26.6 configures a T6041 DisplayPort audio channel with + * [2, 0, 0x800, 0x600, 0x800, 0]: the limit is 0x800, as on earlier SoCs; + * the 5 of the host's mailbox call timeout is not part of the record. + */ static const struct sio_hw_data sio_hw_t6041 = { .protocol = 10, .fifo_threshold = 0x600, - .cfg_word4 = 5, /* the host's default timeout */ + .cfg_word4 = 0x800, + .cfg_six_words = true, + .clear_period_desc = true, + .arrays = &sio_t6041_arrays, + .check_boot_tables = sio_t6041_check_boot_tables, + .send_dt_params = sio_t6041_send_dt_params, +}; + +/* + * T8140 (MacBook Neo) firmware takes the T6041 boot tables (its ADT + * describes them alike); its DisplayPort audio channels refill at 0x800 + * (ADT dp-audioN dma-channels): [2, 0, 0x800, 0x800, 0x800, 0]. + */ +static const struct sio_hw_data sio_hw_t8140 = { + .protocol = 10, + .fifo_threshold = 0x800, + .cfg_word4 = 0x800, .cfg_six_words = true, .clear_period_desc = true, .arrays = &sio_t6041_arrays, @@ -634,6 +655,7 @@ static const struct sio_hw_data sio_hw_t6041 = { static const struct of_device_id sio_of_match[] = { { .compatible = "apple,t6041-sio", .data = &sio_hw_t6041 }, + { .compatible = "apple,t8140-sio", .data = &sio_hw_t8140 }, { .compatible = "apple,t6030-sio", .data = &sio_hw_t6030 }, { .compatible = "apple,sio", .data = &sio_hw_generic }, { } From d14ce94cdc25bcc1837e8da74138501c02a8e2f4 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:16:06 +1100 Subject: [PATCH 132/392] accel/ane: h16: close the channel-IRQ arm/disarm race Review of the IRQ wake found three holes: - ane_h16_irq_wait_set(false) cleared irq_armed under irq_lock but called disable_irq() after dropping it. A pause that had already seen irq_wait set could re-arm in that gap: an unbalanced enable_irq() warning, then a line left disabled while irq_armed was true, so every later wait silently fell back to its timeout. Disarm with disable_irq_nosync() under the lock, synchronize_irq() after it, and let a pause arm only when it sees irq_wait set under the same lock. - Concurrent waiters (pipelined EXECUTEs from several processes) shared one irq_fired flag, and a new waiter cleared it before an earlier one had seen it. The handler now bumps a sequence count and each waiter waits for it to move past its own snapshot. - The warm-boot resume path never turned the IRQ wait back on after power down. H16 does not take that path today; turn it on there like the cold boot does. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/accel/ane/ane_h16_backend.c | 45 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 010dca71d8c07..9e0b061204a2b 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -70,15 +70,16 @@ struct ane_h16_device { int irq; /* * The channel IRQ (ANE_FW_IRQ_CHANNELS) as a wakeup for the polled - * waits once the firmware is up: relax() arms it before sleeping and + * waits once the firmware is up: pause() arms it before sleeping and * the handler disables it again. The transport still reads, - * acknowledges and drains every ring itself, as before. + * acknowledges and drains every ring itself, as before. irq_seq + * counts IRQs, so concurrent waiters each see their own wakeup. */ spinlock_t irq_lock; wait_queue_head_t irq_wq; bool irq_wait; bool irq_armed; - bool irq_fired; + unsigned long irq_seq; u64 irq_wakeups; u64 irq_timeouts; struct ane_fw_rtk rtk; @@ -210,19 +211,22 @@ static void ane_h16_sleep_trace(struct ane_h16_device *h, const char *when); */ static void ane_h16_irq_wait_set(struct ane_h16_device *h, bool on) { - bool disarm = false; - if (h->irq < 0 || h->boot.soc->irq_role != ANE_FW_IRQ_CHANNELS) return; + /* + * Arming and disarming both happen under irq_lock, so a pause racing + * with power down either armed before this (and is disarmed here) or + * sees irq_wait clear and polls. + */ scoped_guard(spinlock_irqsave, &h->irq_lock) { WRITE_ONCE(h->irq_wait, on); if (!on && h->irq_armed) { h->irq_armed = false; - disarm = true; + disable_irq_nosync(h->irq); } } - if (disarm) - disable_irq(h->irq); + if (!on) + synchronize_irq(h->irq); } static int ane_h16_suspend_firmware(struct ane_h16_device *h) @@ -392,8 +396,10 @@ static int ane_h16_warm_boot(struct ane_h16_device *h) return ret; ret = ane_fw_transport_check(&b->transport); h->resume_ready_ns = ktime_get_ns(); - if (!ret) + if (!ret) { + ane_h16_irq_wait_set(h, true); dev_info(h->dev, "H16 ANE: firmware resumed (warm boot); programs kept\n"); + } return ret; } @@ -1026,17 +1032,26 @@ static u64 ane_h16_now(void *ctx) static void ane_h16_irq_pause(struct ane_h16_device *h, u64 waited_ns) { unsigned int min_us, max_us; + unsigned long seq = 0; + bool armed = false; long ret; min_us = ane_fw_poll_sleep_us(waited_ns, &max_us); scoped_guard(spinlock_irqsave, &h->irq_lock) { - h->irq_fired = false; - if (!h->irq_armed) { - h->irq_armed = true; - enable_irq(h->irq); + if (h->irq_wait) { + seq = h->irq_seq; + if (!h->irq_armed) { + h->irq_armed = true; + enable_irq(h->irq); + } + armed = true; } } - ret = wait_event_hrtimeout(h->irq_wq, READ_ONCE(h->irq_fired), + if (!armed) { + ane_fw_poll_pause(waited_ns); + return; + } + ret = wait_event_hrtimeout(h->irq_wq, READ_ONCE(h->irq_seq) != seq, us_to_ktime(min_us)); scoped_guard(spinlock_irqsave, &h->irq_lock) { if (ret) @@ -1278,7 +1293,7 @@ static irqreturn_t ane_h16_irq(int irq, void *data) disable_irq_nosync(irq); h->irq_armed = false; } - WRITE_ONCE(h->irq_fired, true); + WRITE_ONCE(h->irq_seq, h->irq_seq + 1); } wake_up(&h->irq_wq); return IRQ_HANDLED; From 39204544c26bd05500d2eb444f7141c9fe3654f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:12:27 +1100 Subject: [PATCH 133/392] accel/ane: survive an event backlog and recover with a shared PMP SRAM On the M1 Pro (T6000), an ANE MobileNet loop plus four busy CPUs wedges the engine within a minute, and only a reboot brings it back. Two bugs make that happen. The trigger: every task of a request can post a completion event, and the poll treated more than 64 pending events on a line as a bad read and failed the request with -EIO. The linked MobileNet has 99 tasks. When the poller runs late because every CPU is busy, the backlog passes 64 (84 events were seen on line 1). Drain any count a queue can produce, and keep the bad-read check for counts no queue reaches. The wedge: recovery waits until every power domain of the ANE reads off before it powers the engine back on. Since the ANE level control (659e2cce), the M1 Pro/Max ANE also holds ps_pms_sram, the SRAM of its performance controller. That domain stays on whenever the PMP report provider is up, so the wait always timed out ('engine power-off actual=0x0 rc=-110') and every recovery failed. Domains that the ANE shares with its performance controller are not part of the engine's power state, so recovery no longer waits for them. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/accel/ane/ane_tm.c | 48 +++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index eb157b684e0e4..f259edcf1ed6c 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "ane_tm.h" #include "ane_pm.h" @@ -224,13 +225,21 @@ static void ane_tm_push_tq(struct ane_device *ane, struct ane_request *req) */ u32 ane_last_tmst; +/* + * Every task of a request can post an event, so a program with more tasks + * than one poll drains leaves a backlog: a delayed poller (all CPUs busy) + * saw 84 events on line 1 for a 99-task MobileNet. Only a count no queue + * can produce means the read itself is bad. + */ +#define ANE_TM_EVENTS_MAX 0x1000 + static int ane_tm_collect_events(struct ane_device *ane, struct ane_request *req, u32 *finished) { for (unsigned int line = 0; line < 2; line++) { u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); - if (count > 64) + if (count > ANE_TM_EVENTS_MAX) return -EIO; for (u32 n = 0; n < count; n++) { u32 info = tm_read32(ane, TM_IRQ_INFO(line)); @@ -419,6 +428,32 @@ static int ane_ps_verify_on(struct ane_device *ane) return err; } +/* + * A power domain the ANE shares with its performance controller (the T600x + * PMP report SRAM, ps_pms_sram) stays on for as long as the controller is + * up, whatever the ANE does, so it is no part of the engine's power state. + */ +static bool ane_pd_is_perf_supply(struct ane_device *ane, int index) +{ + struct device_node *report = ane->perf.report; + struct of_phandle_args pd, rp; + bool shared = false; + + if (!report || of_parse_phandle_with_args(ane->dev->of_node, "power-domains", + "#power-domain-cells", index, &pd)) + return false; + for (int i = 0; !shared; i++) { + if (of_parse_phandle_with_args(report, "power-domains", + "#power-domain-cells", i, &rp)) + break; + shared = rp.np == pd.np && rp.args_count == pd.args_count && + !memcmp(rp.args, pd.args, pd.args_count * sizeof(pd.args[0])); + of_node_put(rp.np); + } + of_node_put(pd.np); + return shared; +} + static bool ane_partitions_off(struct ane_device *ane) { if (ane_ps_act(ane)) @@ -432,7 +467,8 @@ static bool ane_partitions_off(struct ane_device *ane) if (ane->pd_count == 1) return !dev_pm_genpd_is_on(ane->dev); for (int i = 0; i < ane->pd_count; i++) - if (dev_pm_genpd_is_on(ane->pd_dev[i])) + if (dev_pm_genpd_is_on(ane->pd_dev[i]) && + !ane_pd_is_perf_supply(ane, i)) return false; return true; } @@ -480,8 +516,8 @@ static int ane_pd_cycle(struct ane_device *ane) * Two things must go, or the retained TM never reads idle again: * - pending events on both IRQ lines. Each event pops by reading * INFO/UNK1/TMST/UNK2, exactly as ane_tm_collect_events pops them; - * a stale backlog (or one over the 64-entry collect cap) would make - * the next submit's completion poll miss or fail. + * a stale backlog would make the next submit's completion poll + * miss its events. * - the latched queue entry of the dead task. TQ_STATUS/NID1 get the * same clear the normal completion path gives a finished queue. */ @@ -501,8 +537,8 @@ static void ane_tm_drain_retained(struct ane_device *ane) u32 count = tm_read32(ane, TM_IRQ_EVTC(line)); u32 n; - if (count > 64) - count = 64; + if (count > ANE_TM_EVENTS_MAX) + count = ANE_TM_EVENTS_MAX; for (n = 0; n < count; n++) { tm_read32(ane, TM_IRQ_INFO(line)); tm_read32(ane, TM_IRQ_UNK1(line)); From 6546b58683ada4332bac9de9fc5a24354bb5e35b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:46:51 +1100 Subject: [PATCH 134/392] arm64: dts: apple: t8112: match the ISP and DPA1 power states to the ADT The M2 ADT gates the ISP on ISPSENS0-3 in addition to the ISP_SYS and ISP set domains, but the ISP node did not list them, so genpd switched them off as unused at boot and never turned them on for a capture. List them after the existing domains. The ISP driver powers every secondary domain with the firmware and drops them when it stops, so they follow the camera like the other ISP domains do. The T6030 and T6040 ISP nodes already carry ps_ispsens0. The ADT marks DPA1 critical (iBoot leaves it on and macOS never gates it). Every other critical T8112 power state is apple,always-on in the DT, except PMP, which is deliberately left to its consumers. Mark ps_dpa1 always-on to match. Found by the DTCOVER ADT/DT coverage audit of the M2 (J413). Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 1 + arch/arm64/boot/dts/apple/t8112.dtsi | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 9ff4ed911d482..e2564eac95e50 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -455,6 +455,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dpa1"; + apple,always-on; power-domains = <&ps_audio_p>; }; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index f9ead58538938..3b733a4e61b1b 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -731,7 +731,9 @@ <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, <&ps_isp_set10>, <&ps_isp_set11>, - <&ps_isp_set12>; + <&ps_isp_set12>, <&ps_ispsens0>, + <&ps_ispsens1>, <&ps_ispsens2>, + <&ps_ispsens3>; apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; From 048819f3e28c2c3be9f853bab74e710d9cce82e5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:47:23 +1100 Subject: [PATCH 135/392] arm64: dts: apple: t6040-j616s: give the external display chains their PMGR parents Check the J616s PMGR power states against the ADT's device list: - DISPEXT0_SYS has two parents, FAB2_SOC and the DISPEXT rail. The DT had only the rail (the PMP report pmp-dispext0). It worked because ATC0..2_COMMON keep FAB2_SOC on. List both, as DISPEXT1_SYS already does. - FAB7_SOC's parent is AFI, as t6040-pmgr.dtsi describes it. The extdisp-usbc override moved it under FAB6_SOC because it misread the pmgr record (id2 134 is DISPEXT1_SYS; FAB7_SOC is 133). Drop the override and fix the comment. Both fabrics are always-on, so nothing changes at runtime. The power tree now matches the ADT. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .../boot/dts/apple/t6040-j616s-extdisp-hdmi.dtsi | 3 ++- .../boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi | 11 +++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-hdmi.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-hdmi.dtsi index 0e4e59ca5bb3a..3213c6e8f615a 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-hdmi.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-hdmi.dtsi @@ -32,8 +32,9 @@ }; }; +/* ADT DISPEXT0_SYS parents: FAB2_SOC and the DISPEXT rail (PMP request). */ &ps_dispext0_sys { - power-domains = <&pmp_dispext0>; + power-domains = <&ps_fab2_soc>, <&pmp_dispext0>; }; &ps_dispext0_cpu { diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi index 47995f1bc8e7b..1bffe53b5ec83 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi @@ -23,15 +23,10 @@ }; /* - * DISPEXT1_SYS sits on FAB7_SOC (ADT pmgr device id2 134, pmgr_aux 0x488, parent - * FAB6_SOC, off at boot), which t604x-pmgr.dtsi does not describe. m4.62 enabled - * DISPEXT1_SYS without it and the first DCPEXT1 register access (its DART) reset - * the SoC. DISPEXT0_SYS's parent FAB2_SOC is on anyway through ATC0..2_COMMON. + * DISPEXT1_SYS sits on FAB7_SOC (ADT pmgr id2 133, parent AFI as described + * in t6040-pmgr.dtsi, off at boot). m4.62 enabled DISPEXT1_SYS without it + * and the first DCPEXT1 register access (its DART) reset the SoC. */ -&ps_fab7_soc { - power-domains = <&ps_fab6_soc>; -}; - &ps_dispext1_sys { power-domains = <&ps_fab7_soc>, <&pmp_dispext1>; }; From 7764b26a7002e37f61b810d99c3b7154f391846f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:47:24 +1100 Subject: [PATCH 136/392] arm64: dts: apple: t6040-j616s: keep the SIO's SPI, UART and DPA parents on The SIO firmware drives the SPI, UART and DP audio shims. On T8140 it crashes at boot if SPI_P and UART_P are off, and T6000 and T8112 make those states parents of SIO_CPU for that reason. On the J616s, SIO_CPU had only SIO as its parent, as in the ADT. SPI_P stayed on only because the QSPI controller never runtime-suspends, and UART_P was not described at all, so it stayed however iBoot left it. Make SPI_P, UART_P and DPA_P parents of SIO_CPU and describe UART_P. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi index 597feaa71ac51..e406a733c350d 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -86,7 +86,19 @@ }; }; +/* + * The SIO firmware drives the SPI, UART and DP audio shims, so keep their + * parents on while it runs, as on T6000, T8112 and T8140. The ADT lists + * only SIO as SIO_CPU's parent. Without this, SPI_P stays on only because + * the QSPI controller never runtime-suspends, and UART_P is left as iBoot + * set it; a T8140 SIO with SPI_P/UART_P off crashes at boot. + */ &ps_sio_cpu { + power-domains = <&ps_sio>, <&ps_spi_p>, <&ps_uart_p>, <&ps_dpa_p>; + status = "okay"; +}; + +&ps_uart_p { status = "okay"; }; From b24975d20e36fc24f909b074e2ca27f9eb8b55c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:47:24 +1100 Subject: [PATCH 137/392] arm64: dts: apple: t6040-j616s: describe the internal PCIe power states The ADT's apcie0 needs APCIE_GP, APCIE_SYS_GP and APCIE_PHY_SW on top of ANS and APCIE_ST0/1 and APCIE_SYS_ST0/1, which are always-on. The DT had the first three disabled and gave pcie0 no power domain, so Linux relied on iBoot leaving them on. pcie-apple keeps the internal links up through suspend-to-idle and does not retrain them after a power cycle, so describe the three states as always-on and give pcie0 APCIE_SYS_GP, whose chain covers APCIE_GP, FAB3_SOC and AFC. Gating these states would need multi-domain support in pcie-apple, with a cold restart of the port on resume. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 0d7c60454d480..c263d1da558cc 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -280,3 +280,30 @@ &usb4_2_nhi { status = "okay"; }; + +/* + * The internal PCIe controller (Wi-Fi/Bluetooth, SD reader) needs APCIE_GP, + * APCIE_SYS_GP and APCIE_PHY_SW on top of the always-on ANS/APCIE_ST* + * states (ADT apcie0 clock-gates). iBoot leaves them on. The driver keeps + * internal links up through suspend-to-idle and cannot retrain them after + * a power cycle, so describe them as always-on: Linux now holds what it + * needs instead of relying on iBoot's state. + */ +&ps_apcie_gp { + apple,always-on; + status = "okay"; +}; + +&ps_apcie_sys_gp { + apple,always-on; + status = "okay"; +}; + +&ps_apcie_phy_sw { + apple,always-on; + status = "okay"; +}; + +&pcie0 { + power-domains = <&ps_apcie_sys_gp>; +}; From 0064b755c8de0abbe83cd2d2d97b35ba9260a95d Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:52:08 +1100 Subject: [PATCH 138/392] arm64: dts: apple: j316s, j413: enable the AOP accelerometer and gyroscope The bootloader now hands the accelerometer and gyroscope calibration of the M1 Pro (J316s) and M2 (J413) to the OS, as on the M4 Pro (aura#124). The ADT carries it on both machines in the same properties and format. Without it the aop_motion driver leaves the sensors off. Drop the disabled status from both service children. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000-j316s.dts | 2 -- arch/arm64/boot/dts/apple/t8112-j413.dts | 2 -- 2 files changed, 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 5fca6c4d4a8c9..453c383052f75 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -75,12 +75,10 @@ &aop { accel { compatible = "apple,t6000-aop-accel", "apple,aop-accel"; - status = "disabled"; }; gyro { compatible = "apple,t6000-aop-gyro", "apple,aop-gyro"; - status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6bea58173d2ab..a0f597d47f609 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -187,12 +187,10 @@ accel { compatible = "apple,t8112-aop-accel", "apple,aop-accel"; - status = "disabled"; }; gyro { compatible = "apple,t8112-aop-gyro", "apple,aop-gyro"; - status = "disabled"; }; }; From f709f16b15347aaf3edeff3298b6207bc5790eee Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:57:20 +1100 Subject: [PATCH 139/392] arm64: dts: apple: t600x: power the ISP sensor domains with the ISP The T600x ISP reaches its camera sensor interfaces through the four ISPSENS power states, and the Apple device tree gates them together with the ISP. The ISP node did not list them, so the ISPSENS domains had no consumer and stayed off while the camera ran. Append them to the ISP power-domains, as the T6041 ISP already does for its sensor domain. The driver keeps the first domain as its runtime PM link and switches the others on in order when the firmware boots, so the sensor domains come up last and go down first. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 63da88e035da3..cd50cd66cff39 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -1013,7 +1013,9 @@ #endif <&ps_isp_set1>, <&ps_isp_fe>, <&ps_isp_set3>, <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, - <&ps_isp_set7>, <&ps_isp_set8>; + <&ps_isp_set7>, <&ps_isp_set8>, + <&ps_ispsens0>, <&ps_ispsens1>, + <&ps_ispsens2>, <&ps_ispsens3>; apple,dart-vm-size = <0x0 0xa0000000>; status = "disabled"; From fb559cd5348d76e904b4683bc1e1f4a2fb2ae647 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:57:20 +1100 Subject: [PATCH 140/392] arm64: dts: apple: t600x-j314-j316: disable the PCIe-C tunnel on ATC3 On the 14-inch and 16-inch MacBook Pro the fourth Type-C PHY (ATC3) only drives the HDMI port. It has no USB4 controller (the board already drops usb4_3_acio), so there is no PCIe-C root complex or DART behind it, and the Apple device tree describes none. The USB DARTs on ATC3 are disabled for the same reason; mark apciec3 and its DART disabled too, so the DT describes only hardware that exists. No runtime change: nothing could reach these nodes without a port 3 USB4 controller. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 8055596720149..9dbbcd52d4ec4 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -744,6 +744,15 @@ status = "disabled"; }; +/* ATC3 has no USB4/Thunderbolt, so there is no PCIe-C tunnel behind it either */ +&apciec3_dart { + status = "disabled"; +}; + +&apciec3 { + status = "disabled"; +}; + &dwc3_3 { status = "disabled"; }; From 6fc482823a8348ac36486e14bf54787805df47aa Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:57:21 +1100 Subject: [PATCH 141/392] arm64: dts: apple: t6000: drop power states the M1 Pro does not have T6000 is a cut-down T6001. The Apple device tree for T6000 lists the AFNC3 fabric controllers and the USB-A AON domain as always on with no power state register, so the PMGR does not switch them. Disable the power-controller nodes inherited from the T6001 dtsi, so Linux does not manage them as switchable domains on this die. Nothing on T6000 uses these domains. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000.dtsi | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 6c5e086626bda..d4bd9a56fbc6e 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -91,6 +91,26 @@ <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_pms_sram>; }; +/* + * T6000 has no power state registers for the AFNC3 fabric controllers or + * the USB-A AON domain: they are always on without a PMGR switch. + */ +&ps_afnc3_ioa { + status = "disabled"; +}; + +&ps_afnc3_ls { + status = "disabled"; +}; + +&ps_afnc3_lw0 { + status = "disabled"; +}; + +&ps_gp_usb_aon { + status = "disabled"; +}; + #include "t6000-peripherals.dtsi" #include "t6000-aux-spmi.dtsi" From 895c1acc38981a663f2c9c8f8ca5618c0f3a5022 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:00 +1100 Subject: [PATCH 142/392] soc: apple: vote GPU activity to the PMP so memory leaves idle DVFS on t600x GPU reads on the M1 Max (t6001) top out at 241 GB/s on Linux; macOS reads 361 GB/s. Bit 9 of the PMP report PS-REQ word is the AGX power request: with it clear the PMP ignores the GPU bandwidth demand and keeps memory at its idle floor. With it set the same read microbenchmark reaches 368 GB/s. The GPU driver reports operations in flight with apple_pmp_agx_activity_begin()/end(). pmp-report sets bit 9 when the first operation starts and clears it 250 ms (pmp_report.agx_vote_idle_ms) after the last one ends, at GPU unbind, across system sleep and when the report goes away. A set that is not acknowledged is cleared and retried after a second; a PMP that does not run gets no write. The decisions are plain functions in pmp-agx-vote.h, covered by a host test suite with mutant controls (tools/asahi/pmp-agx-vote-tests.*). apple_pmp_report_set_request() is a locked, acknowledged update of one PS-REQ bit for clients outside this driver; it refuses the AGX bit on t600x so the vote stays its only writer. Off by default (pmp_report.agx_vote), apple,t6001 only. This is omacom/linux-aurora#334 (head b36d0192f408) squashed onto aurora-silicon integration/aurora; conflicts in the GPU unbind signature, the pmp-report.h stubs and bindings_helper.h resolved keeping both sides. The aurora-ci host-tests.json registration is left out (no such file here). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/gpu/drm/asahi/driver.rs | 6 + drivers/gpu/drm/asahi/gpu.rs | 5 + drivers/pmdomain/apple/pmp-agx-vote.h | 174 ++++++++++ drivers/pmdomain/apple/pmp-report.c | 292 ++++++++++++++++ include/linux/soc/apple/pmp-report.h | 37 ++ rust/bindings/bindings_helper.h | 1 + tools/asahi/pmp-agx-vote-tests.c | 474 ++++++++++++++++++++++++++ tools/asahi/pmp-agx-vote-tests.py | 130 +++++++ tools/asahi/t6030-pmp-graph-tests.c | 4 + 9 files changed, 1123 insertions(+) create mode 100644 drivers/pmdomain/apple/pmp-agx-vote.h create mode 100644 tools/asahi/pmp-agx-vote-tests.c create mode 100755 tools/asahi/pmp-agx-vote-tests.py diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 93c52f1de9968..f3707ec05066a 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -238,6 +238,9 @@ impl platform::Driver for AsahiPlatformDriver { type Data<'bound> = AsahiPlatformDriverData<'bound>; fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + // SAFETY: Takes no arguments, process context. Clears the T6001 AGX request in the PMP report if the pmp_report.agx_vote + // parameter set it; a no-op everywhere else. + unsafe { bindings::apple_pmp_agx_activity_release() }; if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); @@ -253,6 +256,9 @@ impl platform::Driver for AsahiPlatformDriver { debug::update_debug_flags(); crate::memacct::LEDGER.configure(); + // SAFETY: Takes no arguments, process context. Lets the T6001 AGX vote run again after an earlier unbind released it; a no-op everywhere else. + unsafe { bindings::apple_pmp_agx_activity_resume() }; + dev_info!(pdev.as_ref(), "Probing...\n"); let cfg = match info.ok_or(ENODEV)? { diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 42fd1cb680adc..0a7a3e6ce065b 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -1276,6 +1276,9 @@ impl GpuManager::ver { .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + // SAFETY: Takes no arguments and does nothing unless pmp_report.agx_vote is set on a T6001. The matching end call is made + // by OpGuard's Drop, and the guard is created right below, so every path out of here (including a failed kick) pairs it. + unsafe { kernel::bindings::apple_pmp_agx_activity_begin() }; let guard = OpGuard(self.clone()); self.kick_firmware()?; Ok(guard) @@ -1801,5 +1804,7 @@ impl GpuManagerPriv for GpuManager::ver { .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); + // SAFETY: Pairs with the begin call in start_op(); callable from any context. + unsafe { kernel::bindings::apple_pmp_agx_activity_end() }; } } diff --git a/drivers/pmdomain/apple/pmp-agx-vote.h b/drivers/pmdomain/apple/pmp-agx-vote.h new file mode 100644 index 0000000000000..9a1f9ecebec7f --- /dev/null +++ b/drivers/pmdomain/apple/pmp-agx-vote.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * When the T6001 AGX request (PS-REQ bit 9 of the PMP report) is set and cleared. + * + * The GPU driver counts operations in flight (begin/end). The request is set when the first operation starts and cleared + * idle_ms after the last one ends, so a decode stream (a burst every ~10 ms) holds one vote and an idle GPU drops it. All state + * changes are in these functions; the caller serializes them with one spinlock and does the register write between + * agx_vote_decide() and agx_vote_applied(). Times are milliseconds from any wrapping clock. + */ +#ifndef PMP_AGX_VOTE_H +#define PMP_AGX_VOTE_H + +#ifdef __KERNEL__ +#include +/* jiffies_to_msecs() returns u32: the clock wraps every 49.7 days, and every compare below is done in 32 bits. */ +typedef u32 agx_ms_t; +typedef s32 agx_sms_t; +#else +#include +#include +typedef uint32_t agx_ms_t; +typedef int32_t agx_sms_t; +#endif + +/* How long the request is held after the last operation ends. w7X measured idle gaps of up to 111.5 ms inside one 9B generation, so 100 ms + * was not enough; 250 ms is about 2.2 times the largest gap seen. The parameter pmp_report.agx_vote_idle_ms overrides it at run time. */ +#define AGX_VOTE_IDLE_MS_DEFAULT 250 + +/* After a set that was not acknowledged (or a PMP that is not running), wait this long before trying again. */ +#define AGX_VOTE_RETRY_MS 1000 + +struct agx_vote { + bool enabled; /* the module parameter */ + bool voted; /* bit 9 is set and was acknowledged */ + bool stopped; /* released (GPU unbind, report removal, sleep): no new vote until agx_vote_resume() */ + unsigned int users; /* GPU operations in flight */ + agx_ms_t idle_ms; /* keep the vote this long after the last operation ends */ + agx_ms_t idle_at; /* when users last fell to 0 */ + unsigned int failures; /* consecutive failed sets; the backoff applies only while this is nonzero */ + agx_ms_t retry_at; /* with failures: no new set before this */ +}; + +enum agx_vote_action { + AGX_VOTE_NONE, + AGX_VOTE_SET, + AGX_VOTE_CLEAR, +}; + +static inline bool agx_vote_reached(agx_ms_t now, agx_ms_t t) +{ + return (agx_sms_t)(now - t) >= 0; +} + +/* A retry deadline more than one backoff window in the future cannot be real: an idle gap beyond half the + * 32-bit millisecond range makes the wrapped arithmetic read an expired deadline as future. Treat it as expired. */ +static inline bool agx_vote_retry_reached(const struct agx_vote *v, agx_ms_t now) +{ + return agx_vote_reached(now, v->retry_at) + || (agx_ms_t)(v->retry_at - now) > AGX_VOTE_RETRY_MS; +} + +/* An operation starts. now is the operation's start time in ms (any wrapping clock; the caller reads it + * under its lock). Returns the delay in ms until the reconcile step must run, in the same terms as + * agx_vote_end(): 1 runs it now, 0 means nothing is pending. A begin during a failed-set backoff returns 0 + * because next() already arms the retry from the failure state, and the caller's queueing can only pull a + * deadline earlier, never push it out; it returns 1 once the deadline has passed. */ +static inline agx_ms_t agx_vote_begin(struct agx_vote *v, agx_ms_t now) +{ + v->users++; + if (!v->enabled || v->stopped) + return 0; + if (v->voted) + return 0; + return (!v->failures || agx_vote_retry_reached(v, now)) ? 1 : v->retry_at - now; +} + +/* An operation ends. Returns the delay in ms after which the reconcile step must run, 0 when none is needed. */ +static inline agx_ms_t agx_vote_end(struct agx_vote *v, agx_ms_t now) +{ + if (!v->users) /* unbalanced end: ignored */ + return 0; + if (--v->users) + return 0; + v->idle_at = now; + if (!v->enabled || !v->voted) + return 0; + return v->idle_ms ? v->idle_ms : 1; +} + +/* What the reconcile step must do now. */ +static inline enum agx_vote_action agx_vote_decide(const struct agx_vote *v, agx_ms_t now) +{ + if (!v->enabled || v->stopped) + return v->voted ? AGX_VOTE_CLEAR : AGX_VOTE_NONE; + if (v->users) + return !v->voted && (!v->failures || agx_vote_retry_reached(v, now)) ? AGX_VOTE_SET : AGX_VOTE_NONE; + if (v->voted && agx_vote_reached(now, v->idle_at + v->idle_ms)) + return AGX_VOTE_CLEAR; + return AGX_VOTE_NONE; +} + +/* + * Delay in ms until the reconcile step must run again, 0 when no timer is needed. 1 when work is due now: an + * operation began while a clear was being written (begin saw voted still true and did not kick), or an end + * landed inside the SET write so the vote is set with an already-reached idle deadline. The caller must run the + * reconcile those cases would otherwise skip; it cannot read 1 as "nothing pending". + */ +static inline agx_ms_t agx_vote_next(const struct agx_vote *v, agx_ms_t now) +{ + agx_ms_t at; + + if (!v->enabled || v->stopped) + return 0; + if (v->users && !v->voted && !v->failures) + return 1; + if (v->users && !v->voted && v->failures) + at = agx_vote_retry_reached(v, now) ? now : v->retry_at; + else if (!v->users && v->voted) + at = v->idle_at + v->idle_ms; + else if (!v->users && !v->voted && v->failures) + /* The write outlasted the operation: arm the retry from the failure state + * itself. The caller clears failures when this reconcile finds nobody waiting + * (decide returns NONE), so it fires once. */ + at = agx_vote_retry_reached(v, now) ? now : v->retry_at; + else + return 0; + /* An already-reached deadline returns 1: there is work, due now. The SET-write race leaves exactly + * this state (voted, no user, deadline passed), and 0 here would strand the vote set forever. */ + return agx_vote_reached(now, at) ? 1 : at - now; +} + +/* Record the outcome of the register write for `action` (ret 0: acknowledged). */ +static inline void agx_vote_applied(struct agx_vote *v, enum agx_vote_action action, int ret, agx_ms_t now) +{ + if (action == AGX_VOTE_SET) { + if (ret) { + v->failures++; + v->retry_at = now + AGX_VOTE_RETRY_MS; + } else { + v->voted = true; + v->failures = 0; + } + } else if (action == AGX_VOTE_CLEAR) { + v->voted = false; /* the write was made; the caller logs an unacknowledged clear */ + } +} + +/* + * The GPU driver unbinds, the report goes away, or the system sleeps: drop the vote now and refuse new ones. True when bit 9 is set and the caller must + * write the clear. An operation that begins afterwards (an ioctl already past the unplug check) counts as a user but does not vote. + */ +static inline bool agx_vote_release(struct agx_vote *v) +{ + bool was_voted = v->voted; + + v->voted = false; + v->stopped = true; + return was_voted; +} + +/* Refuse new votes without touching the one that is set (the first step of a release, before the work item is waited out). */ +static inline void agx_vote_refuse(struct agx_vote *v) +{ + v->stopped = true; +} + +/* The GPU probes (or the report comes back, or the system resumes): votes are allowed again. True when work is already in flight and the caller must run the reconcile step. */ +static inline bool agx_vote_resume(struct agx_vote *v) +{ + v->stopped = false; + return v->enabled && v->users && !v->voted; +} + +#endif /* PMP_AGX_VOTE_H */ diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index b3dceb84afa5a..c61ab5c41b7a5 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -28,10 +28,12 @@ #include #include #include +#include #include #include #include +#include "pmp-agx-vote.h" #include "pmp-report-validation.h" #include "pmp-dram.h" #include "pmp-ane.h" @@ -152,6 +154,19 @@ static const struct attribute_group apple_pmp_dram_group = { }; static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030; +static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x; + +/* T6001 AGX request (see the engine below set_state): AGX is soc-device record 9 on the T6001. */ +#define APPLE_PMP_T6001_AGX_ID 9 +static bool agx_vote; +static struct agx_vote apple_pmp_agx_vote_state = {.idle_ms = AGX_VOTE_IDLE_MS_DEFAULT}; +static DEFINE_SPINLOCK(apple_pmp_agx_vote_lock); +/* Serializes the vote work, the unbind release and the report's removal. */ +static DEFINE_MUTEX(apple_pmp_agx_vote_mutex); +static void apple_pmp_agx_vote_fn(struct work_struct *work); +static DECLARE_DELAYED_WORK(apple_pmp_agx_vote_work, apple_pmp_agx_vote_fn); +static void apple_pmp_agx_vote_stop(void); +static void apple_pmp_agx_vote_stop_new(void); /* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); @@ -376,6 +391,12 @@ static void apple_pmp_report_release(void *data) cancel_delayed_work_sync(&rep->ready_work); /* Withdraw child providers while their power callbacks can still use MMIO. */ of_platform_depopulate(rep->dev); + if (rep->offsets == &apple_pmp_offsets_t600x) { + apple_pmp_agx_vote_stop_new(); + cancel_delayed_work_sync(&apple_pmp_agx_vote_work); + guard(mutex)(&apple_pmp_agx_vote_mutex); + apple_pmp_agx_vote_stop(); + } mutex_lock(&apple_pmp_reports_lock); if (!list_empty(&rep->list)) list_del_rcu(&rep->list); @@ -1370,6 +1391,10 @@ static int apple_pmp_report_probe(struct platform_device *pdev) list_add_tail_rcu(&rep->list, &apple_pmp_reports); mutex_unlock(&apple_pmp_reports_lock); + /* A report that comes back (rebind) takes votes again; its removal refused them. */ + if (rep->offsets == &apple_pmp_offsets_t600x) + apple_pmp_agx_activity_resume(); + if (rep->offsets->fast_die || rep->start) rep->debugfs = debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, rep, @@ -1586,6 +1611,224 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* + * T6001 AGX request. T2 showed that setting PS-REQ bit 9 (AGX, soc-device record 9) lifts the GPU read roof from 241 to 368 GB/s: the PMP + * leaves memory at its idle floor for a GPU that is busy but has not asked. The GPU driver reports operations in flight + * (apple_pmp_agx_activity_begin/end); the request is set while any operation runs and cleared agx_vote_idle_ms after the last one ends, at + * GPU unbind and when this report goes away. The decisions are in pmp-agx-vote.h (unit tested); this is the glue. + * Off by default (pmp_report.agx_vote): with it off nothing is written. + */ +static unsigned int agx_vote_idle_ms = AGX_VOTE_IDLE_MS_DEFAULT; + +static agx_ms_t apple_pmp_agx_now_ms(void) +{ + return jiffies_to_msecs(jiffies); +} + +/* + * Queue the reconcile step. agx_vote=1 on the kernel command line runs the parameter setter from parse_args(), before workqueue_init_early() has + * created system_dfl_wq; queueing on the NULL workqueue would oops in start_kernel(). Nothing is owed then: begin() and the work function read the + * parameter themselves, so the first GPU operation kicks. + */ +static void apple_pmp_agx_vote_kick(unsigned long delay_ms) +{ + struct workqueue_struct *wq = READ_ONCE(system_dfl_wq); + + if (!wq) + return; + mod_delayed_work(wq, &apple_pmp_agx_vote_work, msecs_to_jiffies(delay_ms)); +} + +static int apple_pmp_agx_vote_param_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (!ret) + apple_pmp_agx_vote_kick(0); /* switching off clears at once, switching on votes if the GPU is busy */ + return ret; +} + +static const struct kernel_param_ops apple_pmp_agx_vote_ops = { + .set = apple_pmp_agx_vote_param_set, + .get = param_get_bool, +}; +module_param_cb(agx_vote, &apple_pmp_agx_vote_ops, &agx_vote, 0644); +MODULE_PARM_DESC(agx_vote, "T6001: set the AGX power request in the PMP report while the GPU has work in flight (default: off)"); +module_param(agx_vote_idle_ms, uint, 0644); +MODULE_PARM_DESC(agx_vote_idle_ms, "Keep the AGX request this many ms after the last GPU operation ends (default: 250)"); + +/* Write the AGX request of the bound T6001 report: -ENODEV without one, -EAGAIN (nothing written) when the PMP does not run. */ +static int apple_pmp_agx_write(bool enable, bool no_ack) +{ + struct apple_pmp_report *rep; + int ret = -ENODEV; + + if (!of_machine_is_compatible("apple,t6001")) + return ret; + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t600x) + continue; + if (enable && !(readq(rep->base + rep->offsets->status) & PMP_REPORT_READY)) + ret = -EAGAIN; + else + ret = apple_pmp_report_set_state(rep, APPLE_PMP_T6001_AGX_ID, enable, no_ack); + break; + } + rcu_read_unlock(); + return ret; +} + +/* Refuse new votes (the stopped flag), without writing anything. */ +static void apple_pmp_agx_vote_stop_new(void) +{ + unsigned long flags; + + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + agx_vote_refuse(&apple_pmp_agx_vote_state); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); +} + +/* Clear a request that is still set. Caller holds apple_pmp_agx_vote_mutex. */ +static void apple_pmp_agx_vote_stop(void) +{ + unsigned long flags; + bool voted; + + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + voted = agx_vote_release(&apple_pmp_agx_vote_state); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + if (voted) + apple_pmp_agx_write(false, true); +} + +static void apple_pmp_agx_vote_fn(struct work_struct *work) +{ + struct agx_vote *v = &apple_pmp_agx_vote_state; + enum agx_vote_action act; + unsigned long flags; + agx_ms_t now, next = 0; + int ret = 0; + + guard(mutex)(&apple_pmp_agx_vote_mutex); + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + v->enabled = READ_ONCE(agx_vote); + v->idle_ms = READ_ONCE(agx_vote_idle_ms); + now = apple_pmp_agx_now_ms(); + act = agx_vote_decide(v, now); + if (act == AGX_VOTE_NONE && v->failures && !v->users && !v->voted && + agx_vote_retry_reached(v, now)) + v->failures = 0; /* the backoff fired with nobody waiting: it is spent */ + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + + if (act != AGX_VOTE_NONE) { + const bool set = act == AGX_VOTE_SET; + + ret = apple_pmp_agx_write(set, false); + if (ret == -ETIMEDOUT) { + pr_err("pmp_report: AGX request %s not acknowledged, clearing\n", set ? "set" : "clear"); + apple_pmp_agx_write(false, true); + } + if (!set) + ret = 0; /* a clear is final: the state is "not voted" whatever happened */ + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + agx_vote_applied(v, act, ret, now); + next = agx_vote_next(v, apple_pmp_agx_now_ms()); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + } else { + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + next = agx_vote_next(v, now); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + } + if (next) + apple_pmp_agx_vote_kick(next); +} + +void apple_pmp_agx_activity_begin(void) +{ + struct agx_vote *v = &apple_pmp_agx_vote_state; + unsigned long flags; + agx_ms_t delay; + + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + v->enabled = READ_ONCE(agx_vote); + delay = agx_vote_begin(v, apple_pmp_agx_now_ms()); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + if (delay) + apple_pmp_agx_vote_kick(delay); +} +EXPORT_SYMBOL_GPL(apple_pmp_agx_activity_begin); + +void apple_pmp_agx_activity_end(void) +{ + struct agx_vote *v = &apple_pmp_agx_vote_state; + unsigned long flags; + agx_ms_t delay; + + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + v->enabled = READ_ONCE(agx_vote); + v->idle_ms = READ_ONCE(agx_vote_idle_ms); + delay = agx_vote_end(v, apple_pmp_agx_now_ms()); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + if (delay) + apple_pmp_agx_vote_kick(delay); +} +EXPORT_SYMBOL_GPL(apple_pmp_agx_activity_end); + +/* + * The GPU driver unbinds: clear the request now instead of waiting out the idle time, so it is never left set past the GPU's + * power-domain release (the T6030 fix clears before it releases its vote). Process context. + */ +void apple_pmp_agx_activity_release(void) +{ + /* Stop first so that a begin() from now on cannot queue a set, then wait out a work item that already decided, then clear what it set. */ + apple_pmp_agx_vote_stop_new(); + cancel_delayed_work_sync(&apple_pmp_agx_vote_work); + guard(mutex)(&apple_pmp_agx_vote_mutex); + apple_pmp_agx_vote_stop(); +} +EXPORT_SYMBOL_GPL(apple_pmp_agx_activity_release); + +/* The GPU probes again, the report comes back or the system resumes: allow votes, and vote now if operations are already in flight. Process context. */ +void apple_pmp_agx_activity_resume(void) +{ + struct agx_vote *v = &apple_pmp_agx_vote_state; + unsigned long flags; + bool kick; + + spin_lock_irqsave(&apple_pmp_agx_vote_lock, flags); + v->enabled = READ_ONCE(agx_vote); + kick = agx_vote_resume(v); + spin_unlock_irqrestore(&apple_pmp_agx_vote_lock, flags); + if (kick) + apple_pmp_agx_vote_kick(0); +} +EXPORT_SYMBOL_GPL(apple_pmp_agx_activity_resume); + +/* + * System sleep: withdraw the request before the devices suspend and decide again after resume. Whether bit 9 held through s2idle costs power is not + * measured (a held display request cost about 0.25 W on J514S), and the PMP may drop the word across sleep, which would leave voted true with nothing + * on the wire. Clearing and re-voting covers both. + */ +static int apple_pmp_agx_pm_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + apple_pmp_agx_activity_release(); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + apple_pmp_agx_activity_resume(); + break; + } + return NOTIFY_DONE; +} + +static struct notifier_block apple_pmp_agx_pm_nb = { + .notifier_call = apple_pmp_agx_pm_notify, +}; + /* Find and check the report before taking the GPU vote; optionally update it. */ static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) { @@ -1654,6 +1897,53 @@ static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, writeq(val, rep->base + rep->offsets->update + 8 * entry); } +/** + * apple_pmp_report_set_request() - set or clear one bit of the PMP power request word of a bound report + * @report_np: device tree node of the report region (compatible apple,t6000-pmp-v2-report and its + * siblings) + * @id: bit number of the request, the index of the soc-device record, 0 to 63 (ANE0 is 10 and AGX + * is 9 on the T6001) + * @enable: set or clear the bit + * @no_ack: do not wait for the acknowledge + * + * One read-modify-write of the word, made under the report's lock: the same lock every report + * entry and the AGX vote take, so requests for different bits cannot lose each other. Unless + * @no_ack, waits (atomic poll, at most 50 ms) until the acknowledge word shows the bit as + * requested. A PMP that does not run has nothing to acknowledge: the request is written and 0 is + * returned, as for a report entry. + * + * The caller owns the bit it requests; this function does not arbitrate between callers of the + * same bit. On the T6001 bit 9 belongs to the AGX vote (pmp_report.agx_vote) and is refused here + * with -EBUSY. After -ETIMEDOUT the request was written but not acknowledged and stays in the + * word: the caller must clear it (@enable false, @no_ack true) before giving up. + * + * Context: process context (the acknowledge wait is a busy poll of up to 50 ms). + * Return: 0, -ENODEV if no report is bound to @report_np, -EINVAL if @id is out of range, + * -EBUSY for the T6001 AGX bit, -ETIMEDOUT if the acknowledge did not follow. + */ +int apple_pmp_report_set_request(const struct device_node *report_np, unsigned int id, + bool enable, bool no_ack) +{ + struct apple_pmp_report *rep; + int ret = -ENODEV; + + if (id > 63) + return -EINVAL; + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == report_np) { + if (id == APPLE_PMP_T6001_AGX_ID && rep->offsets == &apple_pmp_offsets_t600x) + ret = -EBUSY; /* the AGX vote owns this bit */ + else + ret = apple_pmp_report_set_state(rep, id, enable, no_ack); + break; + } + } + rcu_read_unlock(); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_request); + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); @@ -1939,6 +2229,7 @@ MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); static int __init apple_pmp_report_init(void) { + register_pm_notifier(&apple_pmp_agx_pm_nb); apple_pmp_report_debugfs = debugfs_create_dir("apple-pmp-report", NULL); platform_driver_register(&apple_pmp_report_entry_driver); platform_driver_register(&apple_pmp_report_driver); @@ -1947,6 +2238,7 @@ static int __init apple_pmp_report_init(void) static void __exit apple_pmp_report_exit(void) { + unregister_pm_notifier(&apple_pmp_agx_pm_nb); debugfs_remove_recursive(apple_pmp_report_debugfs); platform_driver_unregister(&apple_pmp_report_entry_driver); platform_driver_unregister(&apple_pmp_report_driver); diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index dd3373517bf90..66e97565edc1e 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -24,6 +24,21 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); +/* + * Set or clear one bit of the PMP power request word of the report bound to @report_np, under the + * report's lock, with the acknowledge wait (see pmp-report.c). + */ +int apple_pmp_report_set_request(const struct device_node *report_np, unsigned int id, + bool enable, bool no_ack); +/* + * T6001: the GPU driver reports each operation in flight. With the pmp_report.agx_vote parameter set, the AGX request (PS-REQ bit 9) is + * set while any operation runs and cleared agx_vote_idle_ms after the last one ends. Parameter off: no effect. Both may be called from + * any context. release() clears the request at once (GPU driver unbind), process context. + */ +void apple_pmp_agx_activity_begin(void); +void apple_pmp_agx_activity_end(void); +void apple_pmp_agx_activity_release(void); +void apple_pmp_agx_activity_resume(void); #else static inline int apple_pmp_report_ane_available(const struct device_node *np) { @@ -37,6 +52,22 @@ static inline int apple_pmp_report_ane_perf(const struct device_node *np, return -EOPNOTSUPP; } +static inline void apple_pmp_agx_activity_begin(void) +{ +} + +static inline void apple_pmp_agx_activity_end(void) +{ +} + +static inline void apple_pmp_agx_activity_resume(void) +{ +} + +static inline void apple_pmp_agx_activity_release(void) +{ +} + static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms) { @@ -64,6 +95,12 @@ static inline int apple_pmp_report_fast_die_effort(const struct device_node *np, { return -ENODEV; } + +static inline int apple_pmp_report_set_request(const struct device_node *report_np, + unsigned int id, bool enable, bool no_ack) +{ + return -ENODEV; +} #endif #endif /* _LINUX_SOC_APPLE_PMP_REPORT_H_ */ diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 24d4840aabadb..405831d77da76 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -113,6 +113,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/asahi/pmp-agx-vote-tests.c b/tools/asahi/pmp-agx-vote-tests.c new file mode 100644 index 0000000000000..03a3374a5304f --- /dev/null +++ b/tools/asahi/pmp-agx-vote-tests.c @@ -0,0 +1,474 @@ +// Unit test and simulation of pmp-agx-vote.h: the policy that decides when the T6001 AGX request (PS-REQ bit 9) is set and cleared. +// A fake clock, a fake PMP bit and a fake timer drive the same transition functions the kernel glue calls. No kernel, no hardware. +#include +#include +#include +#include + +#include "pmp-agx-vote.h" + +struct sim { + struct agx_vote v; + uint32_t now; + bool bit; // the PMP's PS-REQ bit 9 + int sets, clears; // register writes that tried to set / clear it + int fail_sets; // the next N set attempts are not acknowledged + bool armed; + uint32_t timer_at; + int begin_in_write; // begin this many operations while the register write of the next reconcile is in progress + int end_in_write; // end this many operations while the register write of the next reconcile is in progress + int kicks; // reconciliations the operations asked for +}; + +static void arm(struct sim *s, uint32_t delay_ms) +{ + uint32_t at = s->now + delay_ms; + if (!s->armed || (int32_t)(at - s->timer_at) < 0) { + s->armed = true; + s->timer_at = at; + } +} + +static void begin(struct sim *s); +static void end(struct sim *s); + +// What the kernel work function does. +static void reconcile(struct sim *s) +{ + enum agx_vote_action act = agx_vote_decide(&s->v, s->now); + int ret = 0; + + s->armed = false; + if (act == AGX_VOTE_SET) { + s->sets++; + if (s->fail_sets > 0) { + s->fail_sets--; + ret = -110; + s->bit = false; // glue force-clears a request that was not acknowledged + } else { + s->bit = true; + } + } else if (act == AGX_VOTE_CLEAR) { + s->clears++; + s->bit = false; + } else if (s->v.failures && !s->v.users && !s->v.voted && + agx_vote_retry_reached(&s->v, s->now)) { + s->v.failures = 0; // the glue clears a spent backoff when the retry fires idle + } + while (s->end_in_write > 0) { + s->end_in_write--; + end(s); + } + while (s->begin_in_write > 0) { + s->begin_in_write--; + begin(s); + } + if (act != AGX_VOTE_NONE) + agx_vote_applied(&s->v, act, ret, s->now); + uint32_t next = agx_vote_next(&s->v, s->now); + if (next) + arm(s, next); +} + +static void begin(struct sim *s) +{ + agx_ms_t d = agx_vote_begin(&s->v, s->now); + if (d) { + s->kicks++; + /* delay 1 is "due now": the glue runs it within the same tick */ + arm(s, d == 1 ? 0 : d); + } +} + +static void end(struct sim *s) +{ + uint32_t d = agx_vote_end(&s->v, s->now); + if (d) + arm(s, d); +} + +static void advance_to(struct sim *s, uint32_t t) +{ + while (s->armed && (int32_t)(s->timer_at - t) <= 0) { + if ((int32_t)(s->timer_at - s->now) > 0) + s->now = s->timer_at; + reconcile(s); + } + s->now = t; +} + +static struct sim fresh(bool enabled, unsigned long idle_ms, uint32_t start) +{ + struct sim s = {.now = start}; + s.v.enabled = enabled; + s.v.idle_ms = idle_ms; + return s; +} + +#define CHECK(cond) do { if (!(cond)) { fprintf(stderr, "FAIL line %d: %s\n", __LINE__, #cond); exit(1); } } while (0) + +int main(void) +{ + // 1. parameter off: nothing is ever written, however much the GPU runs + { + struct sim s = fresh(false, 100, 1000); + for (int i = 0; i < 50; i++) { begin(&s); advance_to(&s, s.now + 4); end(&s); advance_to(&s, s.now + 7); } + advance_to(&s, s.now + 1000); + CHECK(s.sets == 0 && s.clears == 0 && !s.bit); + } + // 2. one operation: set at once, kept for idle_ms, then cleared + { + struct sim s = fresh(true, 100, 1000); + begin(&s); advance_to(&s, 1000); + CHECK(s.bit && s.sets == 1); + advance_to(&s, 1004); end(&s); + advance_to(&s, 1103); CHECK(s.bit); // 99 ms after the end: still voted + advance_to(&s, 1104); CHECK(!s.bit && s.clears == 1); // 100 ms after the end: dropped + } + // 3. decode-like bursts every 10.58 ms for one second: one set, no clear until the idle time after the last burst + { + struct sim s = fresh(true, 100, 5000); + uint32_t last_end = 0; + for (int i = 0; i < 95; i++) { + uint32_t t0 = 5000 + (uint32_t)(i * 10.58); + advance_to(&s, t0); begin(&s); + advance_to(&s, t0 + 4); end(&s); last_end = s.now; + } + CHECK(s.sets == 1 && s.clears == 0 && s.bit); + advance_to(&s, last_end + 99); CHECK(s.bit); + advance_to(&s, last_end + 100); CHECK(!s.bit && s.clears == 1 && s.sets == 1); + } + // 4. work arrives while the drop is pending: the drop is cancelled, then rescheduled from the new end + { + struct sim s = fresh(true, 100, 0); + begin(&s); advance_to(&s, 0); end(&s); + advance_to(&s, 50); begin(&s); + advance_to(&s, 100); CHECK(s.bit && s.clears == 0); + advance_to(&s, 120); end(&s); + advance_to(&s, 219); CHECK(s.bit); + advance_to(&s, 220); CHECK(!s.bit && s.clears == 1 && s.sets == 1); + } + // 5. a set that is not acknowledged is cleared again, not retried in a hot loop, and retried after the backoff + { + struct sim s = fresh(true, 100, 0); + s.fail_sets = 1; + begin(&s); advance_to(&s, 0); + CHECK(s.sets == 1 && !s.bit && !s.v.voted && s.v.failures == 1); + advance_to(&s, AGX_VOTE_RETRY_MS - 1); CHECK(s.sets == 1); + advance_to(&s, AGX_VOTE_RETRY_MS); CHECK(s.sets == 2 && s.bit && s.v.voted && s.v.failures == 0); + } + // 6. an end without a begin is ignored: no underflow, the next begin still works + { + struct sim s = fresh(true, 100, 0); + end(&s); end(&s); + CHECK(s.v.users == 0 && !s.armed); + begin(&s); advance_to(&s, 0); + CHECK(s.v.users == 1 && s.bit); + } + // 7. the parameter is switched off while voted and busy: cleared at the next reconcile, even with work in flight + { + struct sim s = fresh(true, 100, 0); + begin(&s); advance_to(&s, 0); CHECK(s.bit); + s.v.enabled = false; arm(&s, 0); advance_to(&s, 1); + CHECK(!s.bit && s.clears == 1 && !s.v.voted); + s.v.enabled = true; arm(&s, 0); advance_to(&s, 2); // switched on again with work still in flight + CHECK(s.bit && s.sets == 2); + } + // 8. zero idle time still waits one tick (no clear inside the end call's own tick) + { + struct sim s = fresh(true, 0, 0); + begin(&s); advance_to(&s, 0); end(&s); + CHECK(s.armed && s.bit); + advance_to(&s, 1); CHECK(!s.bit); + } + // 9. the millisecond clock wraps: idle_at + idle_ms overflows, the decision stays right + { + struct sim s = fresh(true, 100, UINT32_MAX - 50); + begin(&s); advance_to(&s, s.now); end(&s); + advance_to(&s, s.now + 99); CHECK(s.bit); + advance_to(&s, s.now + 1); CHECK(!s.bit && s.clears == 1); + } + // 10. release (GPU driver unbind): a held vote is cleared once, at once, and the idle timer is not needed afterwards + { + struct sim s = fresh(true, 100, 0); + begin(&s); advance_to(&s, 0); end(&s); + CHECK(s.bit && s.armed); + if (agx_vote_release(&s.v)) { s.clears++; s.bit = false; } + CHECK(!s.bit && s.clears == 1 && !s.v.voted); + advance_to(&s, 500); // the pending timer fires and finds nothing to do + CHECK(s.clears == 1 && !s.armed); + CHECK(!agx_vote_release(&s.v)); // a second release writes nothing + } + // 11. w7X's reproduction: jiffies_to_msecs is 32 bits wide, so a vote whose last operation ended at 0xFFFFFFF0 must still clear + // 100 ms later, after the clock has wrapped to 0x54 (a 64-bit compare of idle_at + idle_ms never decided it) + { + struct sim s = fresh(true, 100, 0xFFFFFFF0u); + begin(&s); advance_to(&s, s.now); end(&s); + CHECK(s.v.idle_at == 0xFFFFFFF0u); + advance_to(&s, 0xFFFFFFF0u + 99u); CHECK(s.bit && s.now == 0x53u); + advance_to(&s, 0x54u); CHECK(!s.bit && s.clears == 1); + CHECK(agx_vote_next(&s.v, s.now) == 0); + } + // 12. a retry deadline across the same wrap + { + struct sim s = fresh(true, 100, 0xFFFFFC00u); + s.fail_sets = 1; + begin(&s); advance_to(&s, s.now); + CHECK(s.sets == 1 && !s.bit); + advance_to(&s, 0xFFFFFC00u + AGX_VOTE_RETRY_MS - 1u); CHECK(s.sets == 1); + advance_to(&s, 0xFFFFFC00u + AGX_VOTE_RETRY_MS); CHECK(s.sets == 2 && s.bit); + } + // 13. the shipped default hold: w7X measured gaps of up to 111.5 ms inside one 9B generation (ctx 8192, once per process), over the old + // 100 ms hold. The default must ride through that gap with one vote, and still drop the vote after a long idle. + { + struct sim s = fresh(true, AGX_VOTE_IDLE_MS_DEFAULT, 1000); + CHECK(AGX_VOTE_IDLE_MS_DEFAULT >= 250); + begin(&s); advance_to(&s, s.now); end(&s); + advance_to(&s, s.now + 112); // the 111.5 ms interval rounded up + begin(&s); advance_to(&s, s.now); end(&s); + CHECK(s.sets == 1 && s.clears == 0 && s.bit); + advance_to(&s, s.now + AGX_VOTE_IDLE_MS_DEFAULT - 1); CHECK(s.bit); + advance_to(&s, s.now + 1); CHECK(!s.bit && s.clears == 1); + } + // 14. review finding 2: an operation begins while the clear is being written (voted is still true, so begin does not kick). After the clear is + // applied the vote must be set again without waiting for another operation. + { + struct sim s = fresh(true, 100, 0); + begin(&s); advance_to(&s, 0); end(&s); + CHECK(s.bit && s.armed); + s.begin_in_write = 1; + advance_to(&s, 100); // the idle hold expires: reconcile decides CLEAR, begin lands inside the write + CHECK(s.v.users == 1); + advance_to(&s, 101); // the owed set runs + CHECK(s.bit && s.v.voted && s.sets == 2 && s.clears == 1); + // the header alone, as the reviewer reproduced it: decide (CLEAR), begin, applied, next + struct agx_vote v = {.enabled = true, .voted = true, .idle_ms = 100, .idle_at = 0}; + CHECK(agx_vote_decide(&v, 100) == AGX_VOTE_CLEAR); + CHECK(!agx_vote_begin(&v, 100)); + agx_vote_applied(&v, AGX_VOTE_CLEAR, 0, 100); + CHECK(v.users == 1 && !v.voted); + CHECK(agx_vote_next(&v, 100) == 1); // not 0: a set is owed + CHECK(agx_vote_decide(&v, 101) == AGX_VOTE_SET); + } + // 15. review finding 3: after release (GPU unbind or report removal) no begin may vote again, until the GPU probes again + { + struct sim s = fresh(true, 100, 0); + begin(&s); advance_to(&s, 0); + CHECK(s.bit && s.v.users == 1); + if (agx_vote_release(&s.v)) { s.clears++; s.bit = false; } + CHECK(!s.bit && s.v.stopped && s.clears == 1); + end(&s); begin(&s); advance_to(&s, 50); // an ioctl already past the unplug check starts an operation + CHECK(!s.bit && s.sets == 1 && !s.armed); + CHECK(agx_vote_decide(&s.v, 50) == AGX_VOTE_NONE); + CHECK(agx_vote_next(&s.v, 50) == 0); + // a work item that was queued before the release finds nothing to set + arm(&s, 0); advance_to(&s, 51); + CHECK(!s.bit && s.sets == 1); + // the GPU probes again: the vote is allowed, and the operation still in flight gets its set + CHECK(agx_vote_resume(&s.v)); + arm(&s, 0); advance_to(&s, 52); + CHECK(s.bit && s.sets == 2 && !s.v.stopped); + } + // 16. review finding 4: system sleep. The vote is withdrawn before suspend and re-decided on resume: with work still in flight it is set + // again (the PMP may have dropped the word), with an idle GPU nothing is written. + { + struct sim s = fresh(true, 250, 0); + begin(&s); advance_to(&s, 0); + CHECK(s.bit); + if (agx_vote_release(&s.v)) { s.clears++; s.bit = false; } // suspend prepare + CHECK(!s.bit && s.v.stopped && s.clears == 1); + if (agx_vote_resume(&s.v)) arm(&s, 0); // resume + advance_to(&s, 10); + CHECK(s.bit && s.sets == 2 && s.v.voted); + end(&s); + if (agx_vote_release(&s.v)) { s.clears++; s.bit = false; } // a second sleep while idle inside the hold + CHECK(!s.bit && s.clears == 2); + CHECK(!agx_vote_resume(&s.v)); // idle: nothing is owed + advance_to(&s, 1000); + CHECK(!s.bit && s.sets == 2 && s.clears == 2); + begin(&s); advance_to(&s, 1001); // the next operation after resume votes as usual + CHECK(s.bit && s.sets == 3); + } + // 17. disabled and stopped never set, whatever begins + { + struct sim s = fresh(false, 100, 0); + s.v.stopped = true; + begin(&s); advance_to(&s, 1); + CHECK(!s.bit && s.sets == 0 && !s.armed); + CHECK(!agx_vote_resume(&s.v)); // disabled: resume owes nothing + } + // 18. mutants that survived the first review: begin() must itself refuse while stopped (not only decide()), a failed set must schedule the retry + // deadline and not the 1 ms owed-set kick (a 1 kHz spin for the whole 1000 ms window otherwise), and agx_vote_refuse() must stop votes. + { + struct agx_vote v = {.enabled = true, .idle_ms = 100}; + agx_vote_refuse(&v); + CHECK(v.stopped && !v.voted); + CHECK(!agx_vote_begin(&v, 0)); // a user, but no kick + CHECK(agx_vote_decide(&v, 0) == AGX_VOTE_NONE && agx_vote_next(&v, 0) == 0); + CHECK(agx_vote_resume(&v)); // work in flight: the reconcile step must run + struct agx_vote w = {.enabled = true, .idle_ms = 100}; + CHECK(agx_vote_begin(&w, 0)); + agx_vote_applied(&w, AGX_VOTE_SET, -110, 500); // the set was not acknowledged + CHECK(w.failures == 1 && w.retry_at == 500 + AGX_VOTE_RETRY_MS); + CHECK(agx_vote_next(&w, 500) == AGX_VOTE_RETRY_MS); // the retry deadline, not 1 ms + CHECK(agx_vote_next(&w, 500 + AGX_VOTE_RETRY_MS - 1) == 1); + } + // 19. w7X's wrap mutant: reached() must compare in signed 32-bit arithmetic, so a deadline that has + // wrapped into the low range is not "already reached" on the pre-wrap side + { + struct agx_vote v = {.enabled = true, .voted = true, .idle_ms = 100, .idle_at = 0xFFFFFFF0u}; + CHECK(agx_vote_decide(&v, 0xFFFFFFF8u) == AGX_VOTE_NONE); + CHECK(agx_vote_next(&v, 0xFFFFFFF8u) == 0x5Cu); + struct agx_vote w = {.enabled = true, .idle_ms = 100, .users = 1, .failures = 1, .retry_at = 0x54u}; + CHECK(agx_vote_decide(&w, 0xFFFFFFF8u) == AGX_VOTE_NONE); + } + // 20. chriskearney's race: the last end lands between decide (SET) and applied. end saw voted false and + // armed nothing; applied then sets the vote with no user and an already-reached idle deadline. The + // caller must still be told to run the reconcile that clears it. + { + // exactly the reviewer's interleaving, on the header alone: decide, then end, then the write lands + struct agx_vote v = {.enabled = true, .idle_ms = 0}; + CHECK(agx_vote_begin(&v, 5000) && v.users == 1); + CHECK(agx_vote_decide(&v, 5000) == AGX_VOTE_SET); + CHECK(agx_vote_end(&v, 5001) == 0); // the end during the write sees voted false: no timer + CHECK(v.users == 0); + agx_vote_applied(&v, AGX_VOTE_SET, 0, 5000); + CHECK(v.voted && v.users == 0); + CHECK(agx_vote_next(&v, 5001) == 1); // an already-reached deadline is due now, not "nothing" + CHECK(agx_vote_decide(&v, 5001) == AGX_VOTE_CLEAR); + agx_vote_applied(&v, AGX_VOTE_CLEAR, 0, 5001); + CHECK(!v.voted && agx_vote_next(&v, 5001) == 0); // and it settles + } + // 21. chriskearney's bound: an idle gap beyond half the 32-bit millisecond range makes the wrapped + // arithmetic read an expired retry deadline as future, which would keep the vote off for days. A retry + // deadline more than one backoff window in the future cannot be real, so it counts as reached. + { + struct sim s = fresh(true, 100, 0); + s.fail_sets = 1; + begin(&s); advance_to(&s, 0); + CHECK(s.v.failures == 1); + s.v.retry_at = 0x54u; + advance_to(&s, 0x54u + 0x80000005u); // the timer fires absurdly late + CHECK(s.bit && s.sets == 2 && s.v.failures == 0); // the vote is set again at once + // the header alone + struct agx_vote w = {.enabled = true, .users = 1, .failures = 1, .retry_at = 0x54u}; + CHECK(agx_vote_decide(&w, 0x54u + 0x80000005u) == AGX_VOTE_SET); + CHECK(agx_vote_next(&w, 0x54u + 0x80000005u) == 1); + CHECK(agx_vote_decide(&w, 0x54u + 0x80000000u) == AGX_VOTE_SET); + CHECK(agx_vote_next(&w, 0x54u + 0x80000000u) == 1); + } + // 22. the same review: during backoff begin must not ask for a reconciliation decide would refuse + // (every begin during the window kicked a NONE reconcile and the retry timer did the work anyway) + { + struct sim s = fresh(true, 100, 0); + s.fail_sets = 1; + begin(&s); advance_to(&s, 0); + CHECK(s.v.failures == 1 && s.armed); + int kicks = s.kicks; + uint32_t armed_at = s.timer_at; + for (int i = 0; i < 5; i++) + begin(&s); + CHECK(s.kicks == kicks + 5 && s.sets == 1); // each begin arms the same deadline, none reconciles early + CHECK(s.armed && s.timer_at == armed_at); // mod_delayed_work can only pull the deadline earlier + advance_to(&s, AGX_VOTE_RETRY_MS); + CHECK(s.sets == 2 && s.bit && s.v.failures == 0); + // the header alone + struct agx_vote v = {.enabled = true, .failures = 1, .retry_at = 100}; + CHECK(agx_vote_begin(&v, 0) == 100); // arms the remaining delay + CHECK(agx_vote_begin(&v, 100) == 1); // past the deadline: now + } + // 23. w7X's regression repro for the first cut of the backoff gate: a failed set whose retry timer fires + // while the GPU is idle left failures set forever, and begin never asked for the set again. The kick + // must come back once the deadline has passed. + { + struct sim s = fresh(true, 100, 0); + s.fail_sets = 1; + begin(&s); advance_to(&s, 0); + CHECK(s.v.failures == 1 && s.armed); + end(&s); // the operation ends long before the retry + advance_to(&s, AGX_VOTE_RETRY_MS); // the timer fires idle: nothing to do + CHECK(s.v.failures == 0 && !s.armed && !s.bit); // the spent backoff cleared at the idle fire + int kicks = s.kicks; + begin(&s); // a later operation: the kick must come back + advance_to(&s, s.now + 1); + CHECK(s.kicks == kicks + 1 && s.sets == 2 && s.bit && s.v.failures == 0); + // the header alone: inside the window no kick, past it the kick returns + struct agx_vote v = {.enabled = true, .failures = 1, .retry_at = 100}; + CHECK(agx_vote_begin(&v, 50) == 50); // arms the remaining delay + CHECK(agx_vote_begin(&v, 100) == 1); + } + // 24. chriskearney's second race at efff25c1: the last operation ends during a SET ack that then times + // out. applied records the failure with no user and no timer (next returns 0 on that path), and a begin + // before the retry deadline must arm the remaining delay instead of returning "no kick", or the retry + // never runs. The release/resume cycle must not drop it either. + { + struct agx_vote v = {.enabled = true, .idle_ms = 100}; + CHECK(agx_vote_begin(&v, 5000) == 1); + CHECK(agx_vote_decide(&v, 5000) == AGX_VOTE_SET); + CHECK(agx_vote_end(&v, 5001) == 0); // the end during the ack: no timer + agx_vote_applied(&v, AGX_VOTE_SET, -110, 5000); // the ack times out + CHECK(v.failures == 1 && agx_vote_next(&v, 5001) == 999); // next arms the retry from the failure state + unsigned delay = agx_vote_begin(&v, 5002); + CHECK(delay == 998); // the begin arms the remaining delay + CHECK(v.users == 1); + CHECK(agx_vote_decide(&v, 6000) == AGX_VOTE_SET); // the retry lands and sets + agx_vote_applied(&v, AGX_VOTE_SET, 0, 6000); + CHECK(v.voted && v.failures == 0 && agx_vote_next(&v, 6000) == 0); + // the cancel-on-release/resume variant: the retry survives the cycle + struct agx_vote w = {.enabled = true, .failures = 1, .retry_at = 6000}; + CHECK(!agx_vote_release(&w)); // idle release: no clear write + CHECK(!agx_vote_resume(&w)); // idle resume: still no immediate kick + CHECK(agx_vote_begin(&w, 6100) == 1); // past the deadline the kick is back + } + // 25. the residual w7X flagged on the delay contract: the end lands inside a failed SET write and no new + // operation starts. The write path arms nothing (users is already 0 when applied runs), so the work + // function must arm the retry from the failure state itself, and when the retry fires with nobody + // waiting it must clear the backoff instead of re-arming forever. + { + struct sim s = fresh(true, 100, 0); + s.fail_sets = 1; + s.end_in_write = 1; // the op ends inside the ack + begin(&s); advance_to(&s, 0); + CHECK(s.v.failures == 1 && s.v.users == 0 && s.bit == false); + CHECK(s.armed); // the retry is armed from the failure state + advance_to(&s, AGX_VOTE_RETRY_MS); + CHECK(s.v.failures == 0 && !s.armed && !s.bit); // the idle fire clears the backoff, once + advance_to(&s, AGX_VOTE_RETRY_MS + 500); + CHECK(!s.armed && s.sets == 1); // no re-arm, no hot loop + begin(&s); advance_to(&s, s.now + 1); + CHECK(s.sets == 2 && s.bit); // and a later operation votes as usual + // the header alone: next arms the remaining delay with no user and no pending timer + struct agx_vote v = {.enabled = true, .failures = 1, .retry_at = 6000}; + CHECK(agx_vote_next(&v, 5500) == 500); + CHECK(agx_vote_next(&v, 6000) == 1); + } + // 26. chriskearney's missing-retry-timer repro at c3487a52: the end lands inside a failed SET ack + // (no timer exists: applied ran with users 0) and the next operation begins before the retry + // deadline. The begin arms the remaining delay, the deadline is never pulled later, and the + // vote lands inside the new operation. A begin during the window cannot strand the retry even + // when the glue timer is already armed. + { + struct sim s = fresh(true, 250, 0); + s.fail_sets = 1; + s.end_in_write = 1; // the last end lands inside the ack poll + begin(&s); advance_to(&s, 0); + CHECK(s.v.failures == 1 && s.v.users == 0 && !s.bit); // the reviewer's steps 1 to 3 + advance_to(&s, AGX_VOTE_RETRY_MS); // the retry fires idle: spent backoff clears + CHECK(s.v.failures == 0 && !s.armed && !s.bit); + begin(&s); // step 4: the new sole operation, before any deadline + advance_to(&s, s.now + AGX_VOTE_RETRY_MS + 5000); // far past where the vote must have landed + CHECK(s.bit && s.v.voted && s.v.users == 1 && s.v.failures == 0); + // the header alone: begin inside the window arms the remaining delay (never 0), and a second + // begin cannot push the deadline later (mod_delayed_work only pulls it earlier) + struct agx_vote v = {.enabled = true, .failures = 1, .retry_at = 6000}; + CHECK(agx_vote_begin(&v, 5500) == 500); + CHECK(v.users == 1); + CHECK(agx_vote_begin(&v, 5500) == 500); // same begin time: the deadline is unchanged + CHECK(agx_vote_begin(&v, 6000) == 1); // at the deadline: reconcile now + // 1 ms before the deadline + struct agx_vote w = {.enabled = true, .failures = 1, .retry_at = 6000}; + CHECK(agx_vote_begin(&w, 5999) == 1); + } + printf("AGX-VOTE-SELFCHECK-OK\n"); + return 0; +} diff --git a/tools/asahi/pmp-agx-vote-tests.py b/tools/asahi/pmp-agx-vote-tests.py new file mode 100755 index 0000000000000..fbd57bc7003d2 --- /dev/null +++ b/tools/asahi/pmp-agx-vote-tests.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host test for the T6001 AGX vote policy (drivers/pmdomain/apple/pmp-agx-vote.h). + +The policy is a header of static inline transition functions, so the suite +compiles tools/asahi/pmp-agx-vote-tests.c against a copy of that header: a +fake clock, a fake PMP bit and a fake timer drive the same begin, decide, +next and applied calls the kernel glue makes. No kernel, no hardware. + +--mutation NAME / --mutations apply textual mutants to the header copy and +require the suite to fail; like the other self-checking suites, exit status 0 +means every mutant was caught. --ref reads the header at a Git revision (a +pre-fix header fails the suite itself, which is the point of those cases). +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +HEADER = 'drivers/pmdomain/apple/pmp-agx-vote.h' + +# Each mutant removes one guard. The first two are the mutants that survived +# the first review round, when begin() did not refuse while stopped and the +# retry deadline was not distinguished from the 1 ms owed-set kick. +CONTROLS = [ + ('owed-set-branch', '\tif (v->users && !v->voted && !v->failures)\n\t\treturn 1;\n', ''), + ('owed-failures', 'if (v->users && !v->voted && !v->failures)\n\t\treturn 1;', 'if (v->users && !v->voted)\n\t\treturn 1;'), + ('stopped-in-begin', 'v->users++;\n if (!v->enabled || v->stopped)\n return 0;', 'v->users++;\n if (!v->enabled)\n return 0;'), + ('stopped-in-decide', 'if (!v->enabled || v->stopped)\n\t\treturn v->voted ? AGX_VOTE_CLEAR : AGX_VOTE_NONE;', + 'if (!v->enabled)\n\t\treturn v->voted ? AGX_VOTE_CLEAR : AGX_VOTE_NONE;'), + ('wrap-reached', 'return (agx_sms_t)(now - t) >= 0;', 'return now >= t;'), + ('set-write-ends', 'return agx_vote_reached(now, at) ? 1 : at - now;', 'return agx_vote_reached(now, at) ? 0 : at - now;'), + ('retry-dropped-when-idle', + '\telse if (!v->users && !v->voted && v->failures)\n' + '\t\t/* The write outlasted the operation: arm the retry from the failure state\n' + '\t\t * itself. The caller clears failures when this reconcile finds nobody waiting\n' + '\t\t * (decide returns NONE), so it fires once. */\n' + '\t\tat = agx_vote_retry_reached(v, now) ? now : v->retry_at;\n', ''), + ('begin-backoff-kick', 'return (!v->failures || agx_vote_retry_reached(v, now)) ? 1 : v->retry_at - now;', 'return (void)now, 1;'), + ('begin-no-retry-arm', 'return (!v->failures || agx_vote_retry_reached(v, now)) ? 1 : v->retry_at - now;', 'return (void)now, 0;'), + ('begin-drops-users', ' v->users++;\n if (!v->enabled || v->stopped)', ' if (!v->enabled || v->stopped)'), + ('retry-revival', '|| (agx_ms_t)(v->retry_at - now) > AGX_VOTE_RETRY_MS', ''), +] + + +def header_text(ref): + if ref: + return subprocess.check_output(['git', '-C', str(ROOT), 'show', f'{ref}:{HEADER}'], text=True) + return (ROOT / HEADER).read_text() + + +def compile_and_run(out, text): + out.mkdir(parents=True, exist_ok=True) + (out / 'pmp-agx-vote.h').write_text(text) + compiler = shlex.split(os.environ.get('CC', 'clang')) + command = compiler + ['-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-I', str(out), str(ROOT / 'tools/asahi/pmp-agx-vote-tests.c'), + '-o', str(out / 'vote')] + built = subprocess.run(command, capture_output=True, text=True) + (out / 'build.log').write_text(built.stdout + built.stderr) + if built.returncode: + return {'build_rc': built.returncode, 'run_rc': None, 'note': 'compilation failure is not regression evidence'} + prefix = ['setarch', '-R'] if shutil.which('setarch') else [] # ASan startup can crash under ASLR on a loaded host + ran = subprocess.run(prefix + [str(out / 'vote')], capture_output=True, text=True) + (out / 'run.log').write_text(ran.stdout + ran.stderr) + return {'build_rc': 0, 'run_rc': ran.returncode} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--ref', help='read the header at a Git revision (negative control)') + p.add_argument('--mutation', help='apply one named mutant and require the suite to fail') + p.add_argument('--mutations', action='store_true', help='apply every mutant and require each to fail') + p.add_argument('--output', required=True, type=Path) + args = p.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + text = header_text(args.ref) + + base = compile_and_run(args.output, text) + print(Path(args.output / 'run.log').read_text() if base['run_rc'] is not None else Path(args.output / 'build.log').read_text()) + + mutations = [] + if args.mutations or args.mutation: + # A base failure would fail every mutant at the same case and read as caught. + # The negative control (--ref) has no passing base: the pre-fix header cannot compile this suite. + if not args.ref: + assert base['run_rc'] == 0, base + names = [c[0] for c in CONTROLS] if args.mutations else [args.mutation] + for name in names: + old, new = next((c[1], c[2]) for c in CONTROLS if c[0] == name) + count = text.count(old) + result = {} + result['applied'] = count == 1 + if count == 1: + result.update(compile_and_run(args.output / name, text.replace(old, new))) + else: + # A guard that does not exist in this revision cannot be removed: the + # behaviour the guard carries is absent, so the mutant survives there. + result.update({'build_rc': None, 'run_rc': 0, 'note': f'{HEADER} at this revision has no such line ({count} matches)'}) + result['name'] = name + mutations.append(result) + caught = result['applied'] and result['build_rc'] == 0 and result['run_rc'] not in (0, None) + log = args.output / name / ('run.log' if result.get('run_rc') is not None else 'build.log') + detail = log.read_text().strip().splitlines()[-1][:150] if log.exists() else result.get('note', '') + print(f"{name}: {'CAUGHT' if caught else 'SURVIVED'} | {detail}") + + if args.mutations: + assert all(m['applied'] and m['build_rc'] == 0 and m['run_rc'] not in (0, None) for m in mutations), mutations + print('PASS: every vote-policy case passed and every mutant caught') + return 0 + if args.mutation: + m = mutations[0] + assert m['applied'] and m['build_rc'] == 0 and m['run_rc'] not in (0, None), m + return 0 + + (args.output / 'receipt.json').write_text(json.dumps({ + 'source_ref': args.ref, 'source_sha256': hashlib.sha256(text.encode()).hexdigest(), + 'command': [], 'returncode': base['run_rc'] if base['run_rc'] is not None else base['build_rc'], + 'mutations': mutations}, indent=2) + '\n') + return bool(base['run_rc']) if base['run_rc'] is not None else bool(base['build_rc']) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 5f6d0c4d6214b..f9f758cbf15e6 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -739,6 +739,10 @@ static const struct apple_pmp_start_ops apple_pmp_start_hw_ops = { hw_check, hw_bound }; static const struct apple_pmp_start_timing apple_pmp_start_timing = { 2, 0, 2, 2 }; +/* The extracted probe compares rep->offsets with the T6001 table and tells the AGX vote that a report came back. This harness has neither: the table is empty and + * never matches, and the call does nothing. */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x; +static void apple_pmp_agx_activity_resume(void) {} #include "production.h" #if HOST_HAS_DRAM static int devm_device_add_group(struct device *dev, const struct attribute_group *group) From 131966c24a7185ae852a811ee74321a58275d676 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:58 +1100 Subject: [PATCH 143/392] arm64: dts: apple: t8140: watchdog IRQ and SMC I2C master parents from the ADT The J700 ADT gives the watchdog AIC interrupt 481, not 507. apple_wdt does not request it, so nothing changes at runtime; the description is now the one the hardware publishes. SMC_I2CM1 and SMC_I2CM2 hang off SMC_FABRIC in the ADT power tree, as SMC_I2CM0 already does in our tree. Give them the same parent. All three are always-on and SMC_FABRIC is on, so the live state does not change. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 2 ++ arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index a2be4fc980916..7aa9f2bd3a102 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -655,6 +655,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "smc_i2cm1"; + power-domains = <&ps_smc_fabric>; apple,always-on; /* used by the SMC firmware */ }; @@ -664,6 +665,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "smc_i2cm2"; + power-domains = <&ps_smc_fabric>; apple,always-on; /* used by the SMC firmware */ }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index dbe9bd0d4e59c..b781dd756a2d2 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -830,7 +830,7 @@ reg = <0x3 0x082b0000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; }; nub_spmi0: spmi@308714000 { From a2fb0b2385dc434dbc678f6658c0eec7fd55b4ad Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:58 +1100 Subject: [PATCH 144/392] pmdomain: apple: pmp-report: vote GPU activity on T6000 too, by default T6000 has the AGX soc-device record and request bit of T6001 (soc-device 10, PS-REQ bit 9), and the J314s ships with the PMP running. There the PMP runs the fabric at p-state 3 and DCS at 6 and, without the AGX request, never raises them for the GPU. Holding bit 9 on a J316s booted with the J314s PMP profile (same T6000 SoC, same session, glmark2 800x600 offscreen): PMP off 9938 - 10117 PMP on 7222 - 7327 (-25%) PMP on, AGX request 11909 - 12086 (+19% over PMP off) ANE MobileNet stays at 3.67 ms (4.85 with the PMP off) and idle Total System Power at 5.28 W (6.69 W with the PMP off). Take apple,t6000 into the vote and turn it on by default. It writes only while the PMP runs, so boards that keep the PMP off are unchanged; pmp_report.agx_vote=0 still turns it off. The vote is Joshua Warren's (previous commit); this only widens and enables it. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/pmdomain/apple/pmp-report.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index c61ab5c41b7a5..a31b51683bfbb 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -158,7 +158,7 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x; /* T6001 AGX request (see the engine below set_state): AGX is soc-device record 9 on the T6001. */ #define APPLE_PMP_T6001_AGX_ID 9 -static bool agx_vote; +static bool agx_vote = true; static struct agx_vote apple_pmp_agx_vote_state = {.idle_ms = AGX_VOTE_IDLE_MS_DEFAULT}; static DEFINE_SPINLOCK(apple_pmp_agx_vote_lock); /* Serializes the vote work, the unbind release and the report's removal. */ @@ -1616,7 +1616,8 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, * leaves memory at its idle floor for a GPU that is busy but has not asked. The GPU driver reports operations in flight * (apple_pmp_agx_activity_begin/end); the request is set while any operation runs and cleared agx_vote_idle_ms after the last one ends, at * GPU unbind and when this report goes away. The decisions are in pmp-agx-vote.h (unit tested); this is the glue. - * Off by default (pmp_report.agx_vote): with it off nothing is written. + * On by default (pmp_report.agx_vote): it only writes while the PMP runs, so a board that leaves the PMP off is unchanged; with it off + * nothing is written. T6000 has the same record and bit (soc-device 10, PS-REQ bit 9) as T6001. */ static unsigned int agx_vote_idle_ms = AGX_VOTE_IDLE_MS_DEFAULT; @@ -1653,17 +1654,17 @@ static const struct kernel_param_ops apple_pmp_agx_vote_ops = { .get = param_get_bool, }; module_param_cb(agx_vote, &apple_pmp_agx_vote_ops, &agx_vote, 0644); -MODULE_PARM_DESC(agx_vote, "T6001: set the AGX power request in the PMP report while the GPU has work in flight (default: off)"); +MODULE_PARM_DESC(agx_vote, "T6000/T6001: set the AGX power request in the PMP report while the GPU has work in flight (default: on)"); module_param(agx_vote_idle_ms, uint, 0644); MODULE_PARM_DESC(agx_vote_idle_ms, "Keep the AGX request this many ms after the last GPU operation ends (default: 250)"); -/* Write the AGX request of the bound T6001 report: -ENODEV without one, -EAGAIN (nothing written) when the PMP does not run. */ +/* Write the AGX request of the bound T6000/T6001 report: -ENODEV without one, -EAGAIN (nothing written) when the PMP does not run. */ static int apple_pmp_agx_write(bool enable, bool no_ack) { struct apple_pmp_report *rep; int ret = -ENODEV; - if (!of_machine_is_compatible("apple,t6001")) + if (!of_machine_is_compatible("apple,t6000") && !of_machine_is_compatible("apple,t6001")) return ret; rcu_read_lock(); list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { From e7d56a54f8fe7cadf12a69359cc32bc64ae80213 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:01:18 +1100 Subject: [PATCH 145/392] arm64: dts: apple: t600x: report the JPEG codecs to the PMP With APPLE_USE_PMP, put both JPEG codecs on their PMP report entry (soc-device JPEG0, request bit 19) as AVD, DCP and ISP already are, so the PMP knows when they work. On a J316s with the J314s PMP profile and the AGX vote, NV12 to JPEG encode runs 413-432 fps at 1080p (406-419 without the request) and 107 fps at 4K (105). Boards without APPLE_USE_PMP are unchanged. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000-peripherals.dtsi | 8 ++++++++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 2 ++ 2 files changed, 10 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi index 57a65f7d8db0d..354ec7f161a77 100644 --- a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi @@ -78,7 +78,11 @@ reg = <0x4 0x09000000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; +#ifdef APPLE_USE_PMP + power-domains = <&pmp_report_jpg>; +#else power-domains = <&ps_jpg>; +#endif clocks = <&peripheral_clocks 354>, <&peripheral_clocks 355>; iommus = <&jpeg0_dart 0>; status = "disabled"; @@ -99,7 +103,11 @@ reg = <0x4 0x09008000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; +#ifdef APPLE_USE_PMP + power-domains = <&pmp_report_jpg>; +#else power-domains = <&ps_jpg>; +#endif clocks = <&peripheral_clocks 354>, <&peripheral_clocks 355>; iommus = <&jpeg1_dart 0>; status = "disabled"; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 63da88e035da3..f113ede85dd1a 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -230,7 +230,9 @@ label = "pmp-jpg"; #power-domain-cells = <0>; power-domains = <&ps_jpg>; +#ifndef APPLE_USE_PMP status = "disabled"; +#endif }; pmp_report_scodec: report@14 { From fc1a72521871ac70ab1bd09fb55bfc49ade7f259 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:05:47 +1100 Subject: [PATCH 146/392] arm64: dts: apple: t8140-j700: enable the second JPEG engine The J700 ADT describes two JPEG engines, jpeg0 and jpeg1, each with its own DART (dart-jpeg0-h17p, dart-jpeg1-h17p) and a JPG0/JPG1 vote on the shared JPG power state. Only jpeg0 is enabled; jpeg1 has been described but disabled since the engines were added. apple-jpeg supports apple,t8140-jpeg, and the other Macs we ship already run both engines. Enable jpeg1 and its DART. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-j700.dts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 5841470c3c28d..e4634a965e642 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -476,6 +476,14 @@ status = "okay"; }; +&jpeg1_dart { + status = "okay"; +}; + +&jpeg1 { + status = "okay"; +}; + &apr_dart { status = "okay"; }; From be38b34d2a7a6ae7cf322d6b11b1359acd3bc818 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:17:01 +1100 Subject: [PATCH 147/392] tools: asahi: run the AGX vote host suite with the other host tests Register pmp-agx-vote-tests.py (and its mutant run) in run-host-tests.py, which takes the place of the aurora-ci host-tests.json registration that omacom/linux-aurora#334 carries. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index e472c9d74cc10..b878404b2d9fa 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -148,6 +148,7 @@ def mutations(flag, names, base, expect=KILL): + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], 'pciec-policy-tests.py': [(out(), PASS)], 'pmgr-bus-tests.py': [([], PASS)], + 'pmp-agx-vote-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'pmp-j613-hwmon-tests.py': [([], PASS)], 'pmu-rtc-tests.py': [(out(), PASS)] + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', From 3906fcf2c4b69f82fa3f57c92fde2d0867f4e739 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:22:45 +1100 Subject: [PATCH 148/392] dt-bindings: media: apple,isp: allow 18 power domains The T8112 ISP lists its system domain, the FE domain, twelve set domains and the four ISPSENS sensor domains: 18 in all. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- Documentation/devicetree/bindings/media/apple,isp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/media/apple,isp.yaml b/Documentation/devicetree/bindings/media/apple,isp.yaml index 8b3eb242358d6..6b3ed26aa9640 100644 --- a/Documentation/devicetree/bindings/media/apple,isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,isp.yaml @@ -57,7 +57,7 @@ properties: power-domains: minItems: 7 - maxItems: 16 + maxItems: 18 description: Register-backed power domains required by the ISP. memory-region: From b5a7a3efa1d3cd25c52415e5e5c887cfa6a5329c Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:24:53 +1100 Subject: [PATCH 149/392] pmdomain: apple: pmp-report: T600x device performance requests in request classes A report entry can send a SOC-DEV-DVFS performance request with its power request (apple,dvfs), so far only as a level in the default lane, bits 4:0. T6000 devices also take their level in a request class: a four-bit lane at bits 35:32 (class 0x21) through 51:48 (class 0x25). Add apple,dvfs-request-class and build both the active and the release word from it, with KUnit checks of the encodings and the limits. Give the T600x offsets the update aperture and the SOC-DEV-DVFS range (PTD 0x108, 0x28 entries; the ANE level at 0x10c is one of them). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../power/apple,t8140-pmp-report.yaml | 8 ++++++ drivers/pmdomain/apple/pmp-report-test.c | 22 ++++++++++++++++ .../pmdomain/apple/pmp-report-validation.h | 26 +++++++++++++++++++ drivers/pmdomain/apple/pmp-report.c | 25 +++++++++++++----- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 2a8dd3ddc532b..5b36d27734605 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -143,6 +143,14 @@ patternProperties: level (with the valid bit) to the entry once; before clearing the device request it writes level 0. + apple,dvfs-request-class: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0x21, 0x22, 0x23, 0x24, 0x25] + description: + Request class holding the level. Classes 0x21 through + 0x25 occupy the four-bit fields at bits 35:32 through 51:48. If + absent, apple,dvfs uses the default five-bit field at bits 4:0. + apple,always-on: type: boolean description: Report the device as on from probe. diff --git a/drivers/pmdomain/apple/pmp-report-test.c b/drivers/pmdomain/apple/pmp-report-test.c index d99ece1ac4b6f..204e19fbf28ce 100644 --- a/drivers/pmdomain/apple/pmp-report-test.c +++ b/drivers/pmdomain/apple/pmp-report-test.c @@ -254,6 +254,27 @@ static void pmp_bootargs_reject_integer_width(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_pmp_bootargs_valid(args, 29)); } +static void pmp_dvfs_words(struct kunit *test) +{ + u64 word; + + KUNIT_ASSERT_TRUE(test, apple_pmp_dvfs_word(0, 2, &word)); + KUNIT_EXPECT_EQ(test, word, 0x2000000000000002ULL); + KUNIT_ASSERT_TRUE(test, apple_pmp_dvfs_word(0x21, 0, &word)); + KUNIT_EXPECT_EQ(test, word, 0x2000000000000000ULL); + KUNIT_ASSERT_TRUE(test, apple_pmp_dvfs_word(0x21, 1, &word)); + KUNIT_EXPECT_EQ(test, word, 0x2000000100000000ULL); + KUNIT_ASSERT_TRUE(test, apple_pmp_dvfs_word(0x21, 2, &word)); + KUNIT_EXPECT_EQ(test, word, 0x2000000200000000ULL); + KUNIT_ASSERT_TRUE(test, apple_pmp_dvfs_word(0x25, 2, &word)); + KUNIT_EXPECT_EQ(test, word, 0x2002000000000000ULL); + KUNIT_EXPECT_FALSE(test, apple_pmp_dvfs_word(0x20, 1, &word)); + KUNIT_EXPECT_FALSE(test, apple_pmp_dvfs_word(0x26, 1, &word)); + KUNIT_EXPECT_FALSE(test, apple_pmp_dvfs_word(0x21, 0x10, &word)); + KUNIT_EXPECT_FALSE(test, apple_pmp_dvfs_word(0, 0x20, &word)); + KUNIT_EXPECT_FALSE(test, apple_pmp_dvfs_word(0x21, 1, NULL)); +} + static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_ranges_t6030), KUNIT_CASE(pmp_ranges_t8122), @@ -267,6 +288,7 @@ static struct kunit_case apple_pmp_report_cases[] = { KUNIT_CASE(pmp_bootargs_valid_ids), KUNIT_CASE(pmp_bootargs_reject_truncation), KUNIT_CASE(pmp_bootargs_reject_integer_width), + KUNIT_CASE(pmp_dvfs_words), {} }; diff --git a/drivers/pmdomain/apple/pmp-report-validation.h b/drivers/pmdomain/apple/pmp-report-validation.h index 94a682a9f18f2..c3e5ad495650b 100644 --- a/drivers/pmdomain/apple/pmp-report-validation.h +++ b/drivers/pmdomain/apple/pmp-report-validation.h @@ -7,12 +7,38 @@ #include #include +#define PMP_DVFS_VALID BIT_ULL(61) + #define PMP_SOC_DEVICE_SIZE 0x7c #define PMP_SOC_DEVICE_ACK BIT(1) #define PMP_PTD_RANGE_SIZE 32 #define PMP_PTD_RANGE_REQUEST 10 #define PMP_PTD_RANGE_ACK 11 +/* + * A SOC-DEV-DVFS request word: the valid bit, and the level either in the + * default five-bit lane at bits 4:0 or, for request classes 0x21 through + * 0x25, in the four-bit lane at bits 35:32 through 51:48. + */ +static inline bool apple_pmp_dvfs_word(u32 request_class, u32 level, u64 *word) +{ + u32 shift; + + if (!word) + return false; + if (!request_class) { + if (level > 0x1f) + return false; + *word = PMP_DVFS_VALID | level; + return true; + } + if (request_class < 0x21 || request_class > 0x25 || level > 0xf) + return false; + shift = 32 + 4 * (request_class - 0x21); + *word = PMP_DVFS_VALID | (u64)level << shift; + return true; +} + /* The PTD request read, request update and acknowledgment apertures. */ static inline bool apple_pmp_ranges_valid(const u8 *table, size_t len, u32 request_read, u32 request_write, diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index a31b51683bfbb..789cab1c29397 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1415,6 +1415,9 @@ static int apple_pmp_report_probe(struct platform_device *pdev) static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x = { .tgt_read = 0xf80, .tgt_write = 0x107c0, + .update = 0x10000, + .dvfs_first = 0x108, + .dvfs_count = 0x28, .actual = 0x1000, .status = 0x10, }; @@ -1567,6 +1570,7 @@ struct apple_pmp_report_entry { */ bool dvfs; u32 dvfs_entry; + u64 dvfs_off; u64 dvfs_on; bool removed; }; @@ -1885,9 +1889,6 @@ int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) return apple_pmp_report_t6030_gfx(supplier, &enable); } EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); -/* SOC-DEV-DVFS request words carry a valid bit; level 0 is the floor. */ -#define PMP_DVFS_VALID BIT_ULL(61) - /* * One 64-bit store of the whole request word. Never read a request entry * back after storing it: the read retires the pending PTD update before the @@ -1960,7 +1961,7 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo continue; /* The device request is admitted first and the level dropped first. */ if (ent->dvfs && !enable) - apple_pmp_report_dvfs_write(rep, ent->dvfs_entry, PMP_DVFS_VALID); + apple_pmp_report_dvfs_write(rep, ent->dvfs_entry, ent->dvfs_off); ret = apple_pmp_report_set_state(rep, ent->id, enable, ent->no_ack); if (ent->dvfs && enable && !ret) apple_pmp_report_dvfs_write(rep, ent->dvfs_entry, ent->dvfs_on); @@ -2118,18 +2119,28 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } ent->sleep_release = of_property_read_bool(node, "apple,release-for-sleep"); if (of_property_present(node, "apple,dvfs")) { + u32 request_class = 0; u32 dvfs[2]; + if (of_property_read_u32(node, "apple,dvfs-request-class", + &request_class) && + of_property_present(node, "apple,dvfs-request-class")) { + ret = dev_err_probe(dev, -EINVAL, + "invalid apple,dvfs-request-class\n"); + goto err_free; + } ret = of_property_read_u32_array(node, "apple,dvfs", dvfs, 2); - if (ret || !rep || !rep->offsets->update || dvfs[1] > 0x1f || + if (ret || of_property_count_u32_elems(node, "apple,dvfs") != 2 || + !rep || !rep->offsets->update || dvfs[0] < rep->offsets->dvfs_first || - dvfs[0] >= rep->offsets->dvfs_first + rep->offsets->dvfs_count) { + dvfs[0] >= rep->offsets->dvfs_first + rep->offsets->dvfs_count || + !apple_pmp_dvfs_word(request_class, 0, &ent->dvfs_off) || + !apple_pmp_dvfs_word(request_class, dvfs[1], &ent->dvfs_on)) { ret = dev_err_probe(dev, -EINVAL, "invalid apple,dvfs\n"); goto err_free; } ent->dvfs = true; ent->dvfs_entry = dvfs[0]; - ent->dvfs_on = PMP_DVFS_VALID | dvfs[1]; } ret = of_property_read_string(node, "label", &name); From 2795fdb85d3319bee2f32859b0497fbdeb8d1654 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:24:53 +1100 Subject: [PATCH 150/392] arm64: dts: apple: t6000: AVD and JPEG performance requests to the PMP With APPLE_USE_PMP the PMP runs the fabric and DCS low, and AVD and the JPEG codecs asked it for nothing: on a J316s with the J314s PMP profile, AVD decode ran about 40% slower than with the PMP off and 4K JPEG encode about 12% slower. Let their report entries send level 2 in request class 0x21 of SOC-DEV-DVFS entries 0x112 (AVD) and 0x114 (JPEG, shared by both codecs) while they are powered; the level is dropped before the power request. Only boards with APPLE_USE_PMP are affected. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000.dtsi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 6c5e086626bda..291c854d8cabf 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -85,6 +85,24 @@ reg-names = "report", "ane-dvfs"; }; +#ifdef APPLE_USE_PMP +/* + * AVD and the JPEG codecs (one shared soc-device) also send the PMP a + * performance request while they are powered: level 2 (VMAX) in request + * class 0x21 of their SOC-DEV-DVFS entries, 0x112 and 0x114. The PMP then + * holds the SoC rail and fabric up for them. + */ +&pmp_report_avd_sys { + apple,dvfs = <0x112 2>; + apple,dvfs-request-class = <0x21>; +}; + +&pmp_report_jpg { + apple,dvfs = <0x114 2>; + apple,dvfs-request-class = <0x21>; +}; +#endif + &ane { apple,performance-controller = <&pmp_report>; power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, From b6c76e4a3ce5096b2e9ffe01d9393f803ac2e588 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:38:28 +1100 Subject: [PATCH 151/392] arm64: dts: apple: t8140: give the ISP power states their ADT parents The J700 ADT power tree chains the ISP states: ISP_SYS -> ISP_CPU -> CORE0/1, ISP_SYS -> ISP_FE -> DPRX, SECURE, STS0/1, PEARL, BE, and BE -> CLR, CVD -> CVM. None of the parents were described. The ISP driver attaches all of these domains and holds them on while its firmware is resident, so the parent links only order transitions the way the hardware requires. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 7aa9f2bd3a102..7123e79b3498a 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -288,6 +288,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_cpu"; + power-domains = <&ps_isp_sys>; }; ps_isp_cpu_core0: power-controller@4008 { @@ -296,6 +297,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_cpu_core0"; + power-domains = <&ps_isp_cpu>; }; ps_isp_cpu_core1: power-controller@4010 { @@ -304,6 +306,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_cpu_core1"; + power-domains = <&ps_isp_cpu>; }; ps_isp_fe: power-controller@4018 { @@ -312,6 +315,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_fe"; + power-domains = <&ps_isp_sys>; }; ps_isp_dprx: power-controller@4020 { @@ -320,6 +324,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_dprx"; + power-domains = <&ps_isp_fe>; }; ps_isp_secure: power-controller@4028 { @@ -328,6 +333,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_secure"; + power-domains = <&ps_isp_fe>; }; ps_isp_sts0: power-controller@4030 { @@ -336,6 +342,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_sts0"; + power-domains = <&ps_isp_fe>; }; ps_isp_sts1: power-controller@4038 { @@ -344,6 +351,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_sts1"; + power-domains = <&ps_isp_fe>; }; ps_isp_pearl: power-controller@4040 { @@ -352,6 +360,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_pearl"; + power-domains = <&ps_isp_fe>; }; ps_isp_be: power-controller@4048 { @@ -360,6 +369,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_be"; + power-domains = <&ps_isp_fe>; }; ps_isp_clr: power-controller@4050 { @@ -368,6 +378,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_clr"; + power-domains = <&ps_isp_be>; }; ps_isp_cvd: power-controller@4058 { @@ -376,6 +387,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_cvd"; + power-domains = <&ps_isp_be>; }; ps_isp_cvm: power-controller@4060 { @@ -384,6 +396,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "isp_cvm"; + power-domains = <&ps_isp_cvd>; }; ps_disp_cpu: power-controller@10000 { compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; From 7b73e8a2706b4ea1785e8b19976ce96ec885f168 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:12:30 +1100 Subject: [PATCH 152/392] EDAC: apple_soc: add T6000 platform error and event-log handlers Register the T6000 (M1 Pro) error-handler and event-log interrupt groups. The error-handler has 76 lines in 17 groups. Claim SEP, IOA, NI0-NI8 and NSR; leave AMCC, DCS, NSC and NSI unclaimed as on T6041. PMGR has no line on this SoC. All 28 event-log lines are claimed. T6000 uses the IRQ-only path, as T8140 does: the T6041 NI status offset has not been checked on this SoC, so no status window is mapped. Enable both nodes on T6000. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000.dtsi | 9 ++++++++ drivers/edac/apple_soc_edac.c | 32 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index d4bd9a56fbc6e..725901e1ea630 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -114,6 +114,15 @@ #include "t6000-peripherals.dtsi" #include "t6000-aux-spmi.dtsi" +/* Platform error and event-log interrupt groups (apple_soc_edac). */ +&soc_errors { + status = "okay"; +}; + +&soc_events { + status = "okay"; +}; + &apr0_dart { status = "okay"; }; diff --git a/drivers/edac/apple_soc_edac.c b/drivers/edac/apple_soc_edac.c index 1626a19dcd8f0..24aec4c6abcd0 100644 --- a/drivers/edac/apple_soc_edac.c +++ b/drivers/edac/apple_soc_edac.c @@ -85,6 +85,25 @@ static const struct apple_soc_edac_range t6041_error_ranges[] = { { 68, 2, "ni6" }, }; +/* + * T6000 follows the T6041 policy: the AMCC, DCS, NSC and NSI lines stay + * unclaimed. PMGR has no interrupt line on this SoC. + */ +static const struct apple_soc_edac_range t6000_error_ranges[] = { + { 0, 8, "sep" }, + { 12, 4, "ioa" }, + { 32, 4, "ni0" }, + { 36, 3, "ni1" }, + { 39, 3, "ni2" }, + { 42, 2, "ni3" }, + { 44, 2, "ni4" }, + { 46, 4, "ni5" }, + { 50, 4, "ni6" }, + { 54, 4, "ni7" }, + { 58, 4, "ni8" }, + { 72, 4, "nsr" }, +}; + static const struct apple_soc_edac_range t8140_error_ranges[] = { { 4, 1, "pmgr" }, { 5, 9, "sep" }, @@ -98,6 +117,12 @@ static const struct apple_soc_edac_range t8140_error_ranges[] = { { 25, 1, "pio" }, }; +static const struct apple_soc_edac_hw t6000_error_hw = { + .ranges = t6000_error_ranges, + .nranges = ARRAY_SIZE(t6000_error_ranges), + .num_irqs = 76, +}; + static const struct apple_soc_edac_hw t6041_error_hw = { .ranges = t6041_error_ranges, .nranges = ARRAY_SIZE(t6041_error_ranges), @@ -111,6 +136,11 @@ static const struct apple_soc_edac_hw t8140_error_hw = { .num_irqs = 26, }; +static const struct apple_soc_edac_hw t6000_event_hw = { + .num_irqs = 28, + .event_log = true, +}; + static const struct apple_soc_edac_hw t6041_event_hw = { .num_irqs = 21, .event_log = true, @@ -353,8 +383,10 @@ static void apple_soc_edac_remove(struct platform_device *pdev) } static const struct of_device_id apple_soc_edac_of_match[] = { + { .compatible = "apple,t6000-error-handler", .data = &t6000_error_hw }, { .compatible = "apple,t6041-error-handler", .data = &t6041_error_hw }, { .compatible = "apple,t8140-error-handler", .data = &t8140_error_hw }, + { .compatible = "apple,t6000-event-log", .data = &t6000_event_hw }, { .compatible = "apple,t6041-event-log", .data = &t6041_event_hw }, { .compatible = "apple,t8140-event-log", .data = &t8140_event_hw }, { } From edc50a79270258f206000cdef9598ba747489714 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:35:45 +1100 Subject: [PATCH 153/392] EDAC: apple_soc: add the T8112 event-log handler Claim the 16 T8112 (M2) event-log lines and enable the node. The T8112 error-handler stays unclaimed: unlike T6000 and T6041, its ADT node does not describe which line belongs to which source group. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8112.dtsi | 5 +++++ drivers/edac/apple_soc_edac.c | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 3b733a4e61b1b..f431618ee6d2c 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -2654,6 +2654,11 @@ #include "t8112-peripherals.dtsi" +/* Event-log interrupt group (apple_soc_edac). */ +&m2_event_log_handler { + status = "okay"; +}; + /* Both JPEG encoders are SoC blocks with no board wiring. */ &m2_dart_jpeg0 { status = "okay"; diff --git a/drivers/edac/apple_soc_edac.c b/drivers/edac/apple_soc_edac.c index 24aec4c6abcd0..0733a2712d4a4 100644 --- a/drivers/edac/apple_soc_edac.c +++ b/drivers/edac/apple_soc_edac.c @@ -141,6 +141,11 @@ static const struct apple_soc_edac_hw t6000_event_hw = { .event_log = true, }; +static const struct apple_soc_edac_hw t8112_event_hw = { + .num_irqs = 16, + .event_log = true, +}; + static const struct apple_soc_edac_hw t6041_event_hw = { .num_irqs = 21, .event_log = true, @@ -388,6 +393,7 @@ static const struct of_device_id apple_soc_edac_of_match[] = { { .compatible = "apple,t8140-error-handler", .data = &t8140_error_hw }, { .compatible = "apple,t6000-event-log", .data = &t6000_event_hw }, { .compatible = "apple,t6041-event-log", .data = &t6041_event_hw }, + { .compatible = "apple,t8112-event-log", .data = &t8112_event_hw }, { .compatible = "apple,t8140-event-log", .data = &t8140_event_hw }, { } }; From cf04ed335f2c4d0f22f6547b0ba69473fa32e7ed Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:12:42 +1100 Subject: [PATCH 154/392] arm64: dts: apple: t6000: mark the video decoder dma-coherent The T6000 AVD is IO-coherent in both directions. This was measured on an M1 Pro (J316s) with every cache operation skipped (V4L2_MEMORY_FLAG_NON_COHERENT plus V4L2_BUF_FLAG_NO_CACHE_CLEAN and NO_CACHE_INVALIDATE), one picture in flight, H.264 and HEVC 1080p: - CPU-dirtied CAPTURE buffers decode bit-exact; - CPU-cached CAPTURE buffers read back bit-exact without an invalidate; - bitstream OUTPUT buffers written through a cacheable mapping and queued without a clean decode bit-exact. Without dma-coherent every decoded frame is mapped uncached. On this SoC a consumer that reads every frame gets about 0.2 GB/s from it. Results, same Mac, avd-bench, every frame checked against the software decode: stock dma-coherent h264 1080p, Q16 1047-1090 fps 1103-1112 fps hevc 1080p, Q16 1207-1229 fps 1221-1236 fps h264 1080p, Q1 286-293 fps 300 fps read every frame 197 MB/s 24821 MB/s (35.3 ms/frame) (0.28 ms/frame) decode + read, h264 28.2 fps 1110.6 fps There were no DART faults and every result is bit-exact. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6000.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 725901e1ea630..c3c98ce5ec83b 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -111,6 +111,11 @@ status = "disabled"; }; +/* The video decoder is IO-coherent in both directions. */ +&avd { + dma-coherent; +}; + #include "t6000-peripherals.dtsi" #include "t6000-aux-spmi.dtsi" From 8ee6096f8b61eac1514e369e760b7ada7ac8e631 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:19:34 +1100 Subject: [PATCH 155/392] arm64: dts: apple: t8112-j413: add CPU thermal zones The MacBook Air (M2) is fanless, but its device tree had no thermal zone apart from the battery, so nothing throttled the CPU on heat before the SMC's own protection. Describe the SoC die temperatures the T8112 SMC reports as hwmon channels (die maximum, E cluster, four P-core sensors), make the SMC hwmon a thermal sensor provider and the CPUs cooling devices, and list the GPU and CPU die keys for the hwmon SoC die zone. The J413 gets the zones the J613 (MacBook Air, M3) uses: one per die sensor, each throttling both clusters with E weighted 3:1 and capped at state 3, with the J613 alert and control trips (58 C and 95 C). There is no critical trip: these keys read past 100 C for a second or two in boot load bursts, a critical trip there powers the Mac off, and the SMC keeps its own protection. The keys were mapped on a J413 by loading the E cluster, the P cluster and the GPU in turn: Te05 follows the E cluster, Tp01, Tp05, Tp09 and Tp0D follow the P cluster, and Tg0f, Tg0n and Tg0r follow the GPU. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-j413.dts | 227 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 46 +++++ 2 files changed, 273 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6bea58173d2ab..176e6a4784f2d 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -12,10 +12,12 @@ #include "t8112.dtsi" #include "t8112-jxxx.dtsi" #include +#include / { compatible = "apple,j413", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M2, 2022)"; + chassis-type = "laptop"; aliases { @@ -375,3 +377,228 @@ status = "disabled"; }; }; + +/ { + /* + * Fanless: every die sensor throttles both clusters, E weighted 3:1 + * and capped at state 3, as on the J613, with its alert and control + * trips. No critical trip: boot load bursts briefly read past 100 C, + * and the SMC keeps its own protection. + */ + thermal-zones { + cpu-e0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 1>; /* Te05 */ + sustainable-power = <6000>; + + trips { + cpu_e0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 2>; /* Tp01 */ + sustainable-power = <6000>; + + trips { + cpu_p0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 3>; /* Tp05 */ + sustainable-power = <6000>; + + trips { + cpu_p1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p2-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 4>; /* Tp09 */ + sustainable-power = <6000>; + + trips { + cpu_p2_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p2_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p3-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 5>; /* Tp0D */ + sustainable-power = <6000>; + + trips { + cpu_p3_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p3_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index f9ead58538938..3b3d9b499f053 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -75,6 +75,7 @@ next-level-cache = <&l2_cache_0>; i-cache-size = <0x20000>; d-cache-size = <0x10000>; + #cooling-cells = <2>; }; cpu_e1: cpu@1 { @@ -89,6 +90,7 @@ next-level-cache = <&l2_cache_0>; i-cache-size = <0x20000>; d-cache-size = <0x10000>; + #cooling-cells = <2>; }; cpu_e2: cpu@2 { @@ -103,6 +105,7 @@ next-level-cache = <&l2_cache_0>; i-cache-size = <0x20000>; d-cache-size = <0x10000>; + #cooling-cells = <2>; }; cpu_e3: cpu@3 { @@ -117,6 +120,7 @@ next-level-cache = <&l2_cache_0>; i-cache-size = <0x20000>; d-cache-size = <0x10000>; + #cooling-cells = <2>; }; cpu_p0: cpu@10100 { @@ -131,6 +135,7 @@ next-level-cache = <&l2_cache_1>; i-cache-size = <0x30000>; d-cache-size = <0x20000>; + #cooling-cells = <2>; }; cpu_p1: cpu@10101 { @@ -145,6 +150,7 @@ next-level-cache = <&l2_cache_1>; i-cache-size = <0x30000>; d-cache-size = <0x20000>; + #cooling-cells = <2>; }; cpu_p2: cpu@10102 { @@ -159,6 +165,7 @@ next-level-cache = <&l2_cache_1>; i-cache-size = <0x30000>; d-cache-size = <0x20000>; + #cooling-cells = <2>; }; cpu_p3: cpu@10103 { @@ -173,6 +180,7 @@ next-level-cache = <&l2_cache_1>; i-cache-size = <0x30000>; d-cache-size = <0x20000>; + #cooling-cells = <2>; }; l2_cache_0: l2-cache-0 { @@ -1538,6 +1546,44 @@ smc_hwmon: hwmon { compatible = "apple,smc-hwmon"; + #thermal-sensor-cells = <1>; + /* + * The GPU keys read a fixed low value while the GPU + * is idle, then the CPU die keys. + */ + apple,soc-die-keys = "Tg0f", "Tg0n", "Tg0r", "Te05", + "Tp01", "Tp05", "Tp09", "Tp0D"; + + /* SoC die temperatures, first: thermal zones index them. */ + temperature-TCMz { + apple,key-id = "TCMz"; + label = "SoC Die Temperature"; + }; + + temperature-Te05 { + apple,key-id = "Te05"; + label = "E-cluster Temperature"; + }; + + temperature-Tp01 { + apple,key-id = "Tp01"; + label = "P-core 0 Temperature"; + }; + + temperature-Tp05 { + apple,key-id = "Tp05"; + label = "P-core 1 Temperature"; + }; + + temperature-Tp09 { + apple,key-id = "Tp09"; + label = "P-core 2 Temperature"; + }; + + temperature-Tp0D { + apple,key-id = "Tp0D"; + label = "P-core 3 Temperature"; + }; }; smc_rtc: rtc { From 8460aaf8a4836c9cada9a6ea576b288c8f22acad Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:46:51 +1100 Subject: [PATCH 156/392] arm64: dts: apple: adt-coverage: map the J700 power states we describe 61 J700 PMGR records were waived as virtual votes with no power-state register. Each one has a register and a power-controller node in our tree: the ISP sub-states, the AOP and audio states, and SMC_FABRIC. Map them to their nodes and drop their waivers. They were matched by PMGR group and register offset, and the labels were checked against the ADT names. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../dts/apple/adt-coverage/t8140-j700.txt | 122 +++++++++--------- .../boot/dts/apple/adt-coverage/waivers.txt | 61 --------- 2 files changed, 61 insertions(+), 122 deletions(-) diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t8140-j700.txt b/arch/arm64/boot/dts/apple/adt-coverage/t8140-j700.txt index cc87cbfe22276..c21a03f535c60 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t8140-j700.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t8140-j700.txt @@ -179,19 +179,19 @@ /arm-io/pmgr/power-devices/ATC0_COMMON 0x3007003e8 -> /soc/power-management@300700000/power-controller@3e8 /arm-io/pmgr/power-devices/ATC0_USB 0x3007003f0 -> /soc/power-management@300700000/power-controller@3f0 /arm-io/pmgr/power-devices/SEP 0x300700c00 -> /soc/power-management@300700000/power-controller@c00 -/arm-io/pmgr/power-devices/ISP_CPU - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CPU_CORE0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CPU_CORE1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_FE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DPRX - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_SECURE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_STS0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_STS1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_PEARL - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_BE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CLR - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CVD - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CVM - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/ISP_CPU 0x300704000 -> /soc/power-management@300700000/power-controller@4000 +/arm-io/pmgr/power-devices/ISP_CPU_CORE0 0x300704008 -> /soc/power-management@300700000/power-controller@4008 +/arm-io/pmgr/power-devices/ISP_CPU_CORE1 0x300704010 -> /soc/power-management@300700000/power-controller@4010 +/arm-io/pmgr/power-devices/ISP_FE 0x300704018 -> /soc/power-management@300700000/power-controller@4018 +/arm-io/pmgr/power-devices/DPRX 0x300704020 -> /soc/power-management@300700000/power-controller@4020 +/arm-io/pmgr/power-devices/ISP_SECURE 0x300704028 -> /soc/power-management@300700000/power-controller@4028 +/arm-io/pmgr/power-devices/ISP_STS0 0x300704030 -> /soc/power-management@300700000/power-controller@4030 +/arm-io/pmgr/power-devices/ISP_STS1 0x300704038 -> /soc/power-management@300700000/power-controller@4038 +/arm-io/pmgr/power-devices/ISP_PEARL 0x300704040 -> /soc/power-management@300700000/power-controller@4040 +/arm-io/pmgr/power-devices/ISP_BE 0x300704048 -> /soc/power-management@300700000/power-controller@4048 +/arm-io/pmgr/power-devices/ISP_CLR 0x300704050 -> /soc/power-management@300700000/power-controller@4050 +/arm-io/pmgr/power-devices/ISP_CVD 0x300704058 -> /soc/power-management@300700000/power-controller@4058 +/arm-io/pmgr/power-devices/ISP_CVM 0x300704060 -> /soc/power-management@300700000/power-controller@4060 /arm-io/pmgr/power-devices/VENC0_DMA 0x300708000 -> /soc/power-management@300700000/power-controller@8000 /arm-io/pmgr/power-devices/VENC0_PIPE4 0x300708008 -> /soc/power-management@300700000/power-controller@8008 /arm-io/pmgr/power-devices/VENC0_PIPE5 0x300708010 -> /soc/power-management@300700000/power-controller@8010 @@ -225,56 +225,56 @@ /arm-io/pmgr/power-devices/NUB_FABRIC 0x3082800a0 -> /soc/power-management@308280000/power-controller@a0 /arm-io/pmgr/power-devices/NUB_SRAM 0x3082800a8 -> /soc/power-management@308280000/power-controller@a8 /arm-io/pmgr/power-devices/DEBUG_SWITCH 0x3082800b0 -> /soc/power-management@308280000/power-controller@b0 -/arm-io/pmgr/power-devices/AOP_FILTER - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_GPIO - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_BASE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_FR - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SHIM - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_PIODMA - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP2_PIODMA - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_I2CM0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_I2CM1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_I2CM2 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_I2CM3 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SCM - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SPI0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SPI1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SPMI0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SPMI1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_HE_NRCS0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_HE_NRCS1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_CPU - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP2_CPU - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_FR - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_UART0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_UART1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_UART2 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AOP_SRAM - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_P - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_SENSE_PDMC - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_BASE_S_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_BASE_NS_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_MCA - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA0_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA1_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA2_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA3_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA4_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_MCA5_M - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_C - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_S_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_NS_A - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_RX0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_RX1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_RX2 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_RX3 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_RX4 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/AUDIO_LEAP_TX0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/AOP_FILTER 0x308284000 -> /soc/power-management@308280000/power-controller@4000 +/arm-io/pmgr/power-devices/AOP_GPIO 0x308284008 -> /soc/power-management@308280000/power-controller@4008 +/arm-io/pmgr/power-devices/AOP_BASE 0x308284010 -> /soc/power-management@308280000/power-controller@4010 +/arm-io/pmgr/power-devices/AOP_FR 0x308284018 -> /soc/power-management@308280000/power-controller@4018 +/arm-io/pmgr/power-devices/AUDIO_A 0x308284020 -> /soc/power-management@308280000/power-controller@4020 +/arm-io/pmgr/power-devices/AOP_SHIM 0x308284028 -> /soc/power-management@308280000/power-controller@4028 +/arm-io/pmgr/power-devices/AOP_PIODMA 0x308284030 -> /soc/power-management@308280000/power-controller@4030 +/arm-io/pmgr/power-devices/AOP2_PIODMA 0x308284038 -> /soc/power-management@308280000/power-controller@4038 +/arm-io/pmgr/power-devices/AOP_I2CM0 0x308284040 -> /soc/power-management@308280000/power-controller@4040 +/arm-io/pmgr/power-devices/AOP_I2CM1 0x308284048 -> /soc/power-management@308280000/power-controller@4048 +/arm-io/pmgr/power-devices/AOP_I2CM2 0x308284050 -> /soc/power-management@308280000/power-controller@4050 +/arm-io/pmgr/power-devices/AOP_I2CM3 0x308284058 -> /soc/power-management@308280000/power-controller@4058 +/arm-io/pmgr/power-devices/AOP_SCM 0x308284060 -> /soc/power-management@308280000/power-controller@4060 +/arm-io/pmgr/power-devices/AOP_SPI0 0x308284068 -> /soc/power-management@308280000/power-controller@4068 +/arm-io/pmgr/power-devices/AOP_SPI1 0x308284070 -> /soc/power-management@308280000/power-controller@4070 +/arm-io/pmgr/power-devices/AOP_SPMI0 0x308284078 -> /soc/power-management@308280000/power-controller@4078 +/arm-io/pmgr/power-devices/AOP_SPMI1 0x308284080 -> /soc/power-management@308280000/power-controller@4080 +/arm-io/pmgr/power-devices/AOP_HE_NRCS0 0x308284088 -> /soc/power-management@308280000/power-controller@4088 +/arm-io/pmgr/power-devices/AOP_HE_NRCS1 0x308284090 -> /soc/power-management@308280000/power-controller@4090 +/arm-io/pmgr/power-devices/AOP_CPU 0x308284098 -> /soc/power-management@308280000/power-controller@4098 +/arm-io/pmgr/power-devices/AOP2_CPU 0x3082840a0 -> /soc/power-management@308280000/power-controller@40a0 +/arm-io/pmgr/power-devices/AUDIO_FR 0x3082840a8 -> /soc/power-management@308280000/power-controller@40a8 +/arm-io/pmgr/power-devices/AOP_UART0 0x3082840b0 -> /soc/power-management@308280000/power-controller@40b0 +/arm-io/pmgr/power-devices/AOP_UART1 0x3082840b8 -> /soc/power-management@308280000/power-controller@40b8 +/arm-io/pmgr/power-devices/AOP_UART2 0x3082840c0 -> /soc/power-management@308280000/power-controller@40c0 +/arm-io/pmgr/power-devices/AOP_SRAM 0x3082840c8 -> /soc/power-management@308280000/power-controller@40c8 +/arm-io/pmgr/power-devices/AUDIO_P 0x3082840d0 -> /soc/power-management@308280000/power-controller@40d0 +/arm-io/pmgr/power-devices/AUDIO_LEAP_A 0x3082840d8 -> /soc/power-management@308280000/power-controller@40d8 +/arm-io/pmgr/power-devices/AUDIO_SENSE_PDMC 0x3082840e0 -> /soc/power-management@308280000/power-controller@40e0 +/arm-io/pmgr/power-devices/AUDIO_BASE_S_A 0x3082840e8 -> /soc/power-management@308280000/power-controller@40e8 +/arm-io/pmgr/power-devices/AUDIO_BASE_NS_A 0x3082840f0 -> /soc/power-management@308280000/power-controller@40f0 +/arm-io/pmgr/power-devices/AUDIO_LEAP_MCA 0x3082840f8 -> /soc/power-management@308280000/power-controller@40f8 +/arm-io/pmgr/power-devices/AUDIO_MCA0_M 0x308284100 -> /soc/power-management@308280000/power-controller@4100 +/arm-io/pmgr/power-devices/AUDIO_MCA1_M 0x308284108 -> /soc/power-management@308280000/power-controller@4108 +/arm-io/pmgr/power-devices/AUDIO_MCA2_M 0x308284110 -> /soc/power-management@308280000/power-controller@4110 +/arm-io/pmgr/power-devices/AUDIO_MCA3_M 0x308284118 -> /soc/power-management@308280000/power-controller@4118 +/arm-io/pmgr/power-devices/AUDIO_MCA4_M 0x308284120 -> /soc/power-management@308280000/power-controller@4120 +/arm-io/pmgr/power-devices/AUDIO_MCA5_M 0x308284128 -> /soc/power-management@308280000/power-controller@4128 +/arm-io/pmgr/power-devices/AUDIO_LEAP_C 0x308284130 -> /soc/power-management@308280000/power-controller@4130 +/arm-io/pmgr/power-devices/AUDIO_LEAP_S_A 0x308284138 -> /soc/power-management@308280000/power-controller@4138 +/arm-io/pmgr/power-devices/AUDIO_LEAP_NS_A 0x308284140 -> /soc/power-management@308280000/power-controller@4140 +/arm-io/pmgr/power-devices/AUDIO_LEAP_RX0 0x308284148 -> /soc/power-management@308280000/power-controller@4148 +/arm-io/pmgr/power-devices/AUDIO_LEAP_RX1 0x308284150 -> /soc/power-management@308280000/power-controller@4150 +/arm-io/pmgr/power-devices/AUDIO_LEAP_RX2 0x308284158 -> /soc/power-management@308280000/power-controller@4158 +/arm-io/pmgr/power-devices/AUDIO_LEAP_RX3 0x308284160 -> /soc/power-management@308280000/power-controller@4160 +/arm-io/pmgr/power-devices/AUDIO_LEAP_RX4 0x308284168 -> /soc/power-management@308280000/power-controller@4168 +/arm-io/pmgr/power-devices/AUDIO_LEAP_TX0 0x308284170 -> /soc/power-management@308280000/power-controller@4170 /arm-io/pmgr/power-devices/PDM0_AUDIO_REF - -> WAIVED: Virtual PMGR vote; no independent power-state register. /arm-io/pmgr/power-devices/PDM1_AUDIO_REF - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/SMC_FABRIC - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/SMC_FABRIC 0x308288000 -> /soc/power-management@308280000/power-controller@8000 /arm-io/pmgr/power-devices/SMC_GPIO - -> WAIVED: Virtual PMGR vote; no independent power-state register. /arm-io/pmgr/power-devices/SMC_AON - -> WAIVED: Virtual PMGR vote; no independent power-state register. /arm-io/pmgr/power-devices/SMC_UART0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. diff --git a/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt b/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt index 84e0a31edc326..097e09dd16bbe 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt @@ -608,30 +608,6 @@ t8140-j700 /arm-io/pmgr/power-devices/AMCC0_P3 | Virtual PMGR vote; no independe t8140-j700 /arm-io/pmgr/power-devices/ANE-SYS-V | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ANE-SYS-V-PMP | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ANE_CPU | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP2_CPU | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP2_PIODMA | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_BASE | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_CPU | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_FILTER | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_FR | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_GPIO | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_HE_NRCS0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_HE_NRCS1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_I2CM0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_I2CM1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_I2CM2 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_I2CM3 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_PIODMA | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SCM | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SHIM | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SPI0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SPI1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SPMI0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SPMI1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_SRAM | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_UART0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_UART1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AOP_UART2 | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/APCIE_SYS_ST-V | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ATC0-DCS-F1 | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ATC0-DCS-F2 | Virtual PMGR vote; no independent power-state register. @@ -642,29 +618,6 @@ t8140-j700 /arm-io/pmgr/power-devices/ATC0-USB-AUD-V | Virtual PMGR vote; no ind t8140-j700 /arm-io/pmgr/power-devices/ATC0-USB-DART | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/AUDIO-LLT-V | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/AUDIO-P-V | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_BASE_NS_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_BASE_S_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_FR | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_C | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_MCA | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_NS_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_RX0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_RX1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_RX2 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_RX3 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_RX4 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_S_A | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_LEAP_TX0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA0_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA1_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA2_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA3_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA4_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_MCA5_M | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_P | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/AUDIO_SENSE_PDMC | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/AUSB0_AONUSB-V | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/AVD-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/AVD-SOC-VNOM | Virtual PMGR vote; no independent power-state register. @@ -695,7 +648,6 @@ t8140-j700 /arm-io/pmgr/power-devices/DISP_INT_M3 | Virtual PMGR vote; no indepe t8140-j700 /arm-io/pmgr/power-devices/DISP_PPP | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/DP | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/DPEXT0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/DPRX | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/DPTX_PHY | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/DSG | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ECPU0_SRAM | Virtual PMGR vote; no independent power-state register. @@ -714,18 +666,6 @@ t8140-j700 /arm-io/pmgr/power-devices/ISP-CORE-V | Virtual PMGR vote; no indepen t8140-j700 /arm-io/pmgr/power-devices/ISP-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ISP-SOC-VNOM | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/ISP-SYS-V | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_BE | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CLR | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CPU | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CPU_CORE0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CPU_CORE1 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CVD | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_CVM | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_FE | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_PEARL | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_SECURE | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_STS0 | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/ISP_STS1 | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/JPG-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/JPG-SOC-VNOM | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/JPG0-V | Virtual PMGR vote; no independent power-state register. @@ -765,7 +705,6 @@ t8140-j700 /arm-io/pmgr/power-devices/SIO | Virtual PMGR vote; no independent po t8140-j700 /arm-io/pmgr/power-devices/SIO-CPU-V | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/SMC_AON | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/SMC_CPU | Virtual PMGR vote; no independent power-state register. -t8140-j700 /arm-io/pmgr/power-devices/SMC_FABRIC | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/SMC_FPWM0 | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/SMC_FPWM1 | Virtual PMGR vote; no independent power-state register. t8140-j700 /arm-io/pmgr/power-devices/SMC_FPWM2 | Virtual PMGR vote; no independent power-state register. From cee332dfa1be1d60362773835e743652023dfef7 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:53:28 +1100 Subject: [PATCH 157/392] arm64: dts: apple: t6000: leave the JPEG performance request out Measured on a J316s with the J314s PMP profile (same session): with level 2 in request class 0x21 of entry 0x114, JPEG encode at 1080p dropped from 405-432 fps to 287-303 fps, and 4K moved only from 106-107 to 111-114 fps (119 with the PMP off). AVD with its request runs 72-73 fps at 1080p and 18 fps at 4K, against 62-63 and 15-16 with the PMP off, so keep that one and drop the JPEG store until the JPEG request is understood. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000.dtsi | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 291c854d8cabf..b5d973a1fe231 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -87,20 +87,14 @@ #ifdef APPLE_USE_PMP /* - * AVD and the JPEG codecs (one shared soc-device) also send the PMP a - * performance request while they are powered: level 2 (VMAX) in request - * class 0x21 of their SOC-DEV-DVFS entries, 0x112 and 0x114. The PMP then - * holds the SoC rail and fabric up for them. + * AVD also sends the PMP a performance request while it is powered: level 2 + * (VMAX) in request class 0x21 of its SOC-DEV-DVFS entry 0x112. The PMP then + * holds the SoC rail and fabric up for it. */ &pmp_report_avd_sys { apple,dvfs = <0x112 2>; apple,dvfs-request-class = <0x21>; }; - -&pmp_report_jpg { - apple,dvfs = <0x114 2>; - apple,dvfs-request-class = <0x21>; -}; #endif &ane { From 46f876e27b994f092dde5ac99c353d99fc0d5099 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:14:55 +1100 Subject: [PATCH 158/392] tools: asahi: give the T6030 graph harness the entry release word The report entry now keeps the word it stores to release its performance request (dvfs_off) next to the active one; the harness copy of the entry struct follows. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- tools/asahi/t6030-pmp-graph-tests.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index f9f758cbf15e6..840d961ff8cbb 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -199,6 +199,7 @@ struct apple_pmp_report_entry { bool no_ack, dvfs, removed; bool sleep_release, released; u32 dvfs_entry; + u64 dvfs_off; u64 dvfs_on; }; #define PMP_DVFS_VALID BIT_ULL(61) From 04c6eca23ca9f8dbfbcf9b1d74c930877b1c403e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:25:50 +1100 Subject: [PATCH 159/392] arm64: dts: apple: t6000-j316s: enable PMP Let m1n1 start the PMP on the 16-inch M1 Pro, as on the J314s: the ANE, GPU and AVD now send it their performance requests. Same session, PMP off against on (aura-try RAM-only trial, this DTB): glmark2 800x600 offscreen 10002 - 10193 -> 12079 - 12187 (+20%) AVD 1080p / 4K decode 62 - 63 / 15 - 16 fps -> 71 - 72 / 17 - 18 fps ANE MobileNet 4.85 ms -> 3.67 ms idle, Total System Power 3.93 W -> 3.76 W JPEG encode 1080p 392 - 411 fps -> 387 - 405 fps JPEG encode 4K 118 - 119 fps -> 107 fps (-10%) Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000-j316s.dts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 5fca6c4d4a8c9..9e4bd2ae50b03 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -9,6 +9,12 @@ /dts-v1/; +/* + * PMP manages system power states, as on the J314s: the ANE, GPU, AVD and + * JPEG send it their performance requests, and the board idles lower. + */ +#define APPLE_USE_PMP + #include "t6000.dtsi" #include "t600x-j314-j316.dtsi" From 3fd48163f275752f0cc44e396c40e68dadd15d0b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:08:00 +1100 Subject: [PATCH 160/392] arm64: dts: apple: t6000: JPEG performance request to the PMP at level 1 With APPLE_USE_PMP the JPEG codecs sent the PMP no performance request, and 4K NV12 to JPEG encode ran 106-107 fps against 118-119 with the PMP off (J316s with the J314s PMP profile, same session). Level 2 in request class 0x21 of entry 0x114 made 1080p slower (287-303 fps, from 392-411); level 1 restores both: 1080p 383-408 fps and 4K 117-118 fps. GPU, AVD and ANE rows are unchanged (glmark2 12168-12329, AVD 1080p 71 fps and 4K 17-18 fps, ANE 3.68 ms), and idle Total System Power is 3.90 W (3.93 W with the PMP off). Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000.dtsi | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 73fc025895aa8..6b11b4ac9a4d8 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -87,14 +87,20 @@ #ifdef APPLE_USE_PMP /* - * AVD also sends the PMP a performance request while it is powered: level 2 - * (VMAX) in request class 0x21 of its SOC-DEV-DVFS entry 0x112. The PMP then - * holds the SoC rail and fabric up for it. + * AVD and the JPEG codecs (one soc-device for both) also send the PMP a + * performance request while they are powered, in request class 0x21 of + * their SOC-DEV-DVFS entries: AVD level 2 (VMAX) at 0x112, JPEG level 1 + * (VNOM) at 0x114. The PMP then holds the SoC rail and fabric up for them. */ &pmp_report_avd_sys { apple,dvfs = <0x112 2>; apple,dvfs-request-class = <0x21>; }; + +&pmp_report_jpg { + apple,dvfs = <0x114 1>; + apple,dvfs-request-class = <0x21>; +}; #endif &ane { From 9330cb74656ac4e840a0ecc5c44477232e6929ad Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:17:19 +1100 Subject: [PATCH 161/392] soc: apple: aop: keep the firmware nCal on T8112 and T6000 The aop driver patches the AOP firmware boot arguments with nCal 0 on T8112 and T6000. The J413 (M2) image carries nCal 1. With 0 the firmware never announces its accelerometer and gyroscope services, so the aop_motion driver has nothing to bind to. With the image value kept, both services appear and report once the bootloader hands over their calibration. Make nCal optional per SoC. None leaves the image value in place. T8112 and T6000 use None. Every other SoC keeps the value it had: T6040/T8132 write 1, the others write 0. Tested on the M2 (J413) with aura#124 and the J413 accel/gyro children enabled. The firmware image value is 1. Accelerometer at rest: |a| 9.796 m/s^2 over 50 distinct samples. Gyroscope: 0.26 deg/s. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/soc/apple/aop.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index fdb2f1b8e825f..7f446caf407a8 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -1604,7 +1604,7 @@ impl AopData { fn patch_bootargs( &self, aop_mmio: &Mmio<'_, Region>, - patches: &[(u32, u64)], + patches: &[(u32, Option)], word_access: bool, ) -> Result<()> { let offset = aop_mmio.read32(BOOTARGS_OFFSET) as usize; @@ -1648,6 +1648,10 @@ impl AopData { if *k != key { continue; } + // `None` keeps the value the firmware image carries. + let Some(v) = v else { + break; + }; if len > mem::size_of::() { return Err(EINVAL); } @@ -2562,8 +2566,8 @@ struct AopDriver(Arc); struct AopHwConfig { ec0p: u64, - /// The `nCal` boot argument. - ncal: u64, + /// The `nCal` boot argument, or `None` to keep the firmware's own value. + ncal: Option, alig: u64, aopt: u64, /// Complete the firmware's boot arguments before starting it. @@ -2588,7 +2592,7 @@ struct AopHwConfig { const HW_CFG_T8103: AopHwConfig = AopHwConfig { ec0p: 0x020000, - ncal: 0, + ncal: Some(0), aopt: 1, alig: 128, patch_bootargs: true, @@ -2600,9 +2604,11 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { bootargs_word_access: false, halt_on_failed_start: false, }; +/// `nCal` keeps the firmware's own value, as on T6040: the J413 image carries 1, +/// and with 0 the firmware never announces its accelerometer and gyroscope. const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, - ncal: 0, + ncal: None, aopt: 0, alig: 128, patch_bootargs: true, @@ -2614,9 +2620,10 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { bootargs_word_access: false, halt_on_failed_start: false, }; +/// `nCal` keeps the firmware's own value, as on T8112. const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, - ncal: 0, + ncal: None, aopt: 0, alig: 64, patch_bootargs: true, @@ -2630,7 +2637,7 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, - ncal: 0, + ncal: Some(0), aopt: 0, alig: 64, patch_bootargs: true, @@ -2649,7 +2656,7 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { /// and report interval but never send a report. const HW_CFG_T6040: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, - ncal: 1, + ncal: Some(1), aopt: 0, alig: 64, patch_bootargs: true, @@ -2672,7 +2679,7 @@ const HW_CFG_T8132: AopHwConfig = HW_CFG_T6040; /// are started if they will. const HW_CFG_T8140: AopHwConfig = AopHwConfig { ec0p: 0, - ncal: 0, + ncal: Some(0), aopt: 0, alig: 0, patch_bootargs: false, @@ -2737,10 +2744,10 @@ impl platform::Driver for AopDriver { data.patch_bootargs( &aop_mmio, &[ - (from_fourcc(b"EC0p"), cfg.ec0p), + (from_fourcc(b"EC0p"), Some(cfg.ec0p)), (from_fourcc(b"nCal"), cfg.ncal), - (from_fourcc(b"alig"), cfg.alig), - (from_fourcc(b"AOPt"), cfg.aopt), + (from_fourcc(b"alig"), Some(cfg.alig)), + (from_fourcc(b"AOPt"), Some(cfg.aopt)), ], cfg.bootargs_word_access, )?; From fa13cbdc1103dc5a66ed109e033a4a6ec2307fd0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 8 Nov 2026 10:39:18 -0500 Subject: [PATCH 162/392] drm/apple: timestamp fixed ProMotion on legacy firmware Fixed 120 Hz internal ProMotion modes can have explicit zero VRR bounds. The parser sets needs_timestamps for these modes, but the legacy modeset path checks vrr instead. This leaves their swaps without the timestamps needed for 120 Hz presentation. Use needs_timestamps for the legacy path too, keeping the main-display guard so external displays retain their existing policy. Fixes: 9df36a2c42b6 ("drm/apple: keep ProMotion deadlines independent of VRR admission") Signed-off-by: DjDeveloperr --- drivers/gpu/drm/apple/iomfb_template.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ad87814e9bb9b..1a648d2c613fd 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -3591,7 +3591,7 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->output_pq = dcp->pq_requested && mode->pq.score >= 0; /* Built-in ProMotion panels require timestamps even in fixed-120 mode. */ - dcp->use_timestamps = mode->vrr && dcp->main_display; + dcp->use_timestamps = mode->needs_timestamps && dcp->main_display; cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) { From 79655ca290a29e06a73d86162a44c17e8e6d7e32 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:05:49 +1100 Subject: [PATCH 163/392] tools/asahi: aop-bootargs-tests: cover the optional nCal AopHwConfig.ncal is now an Option, and patch_bootargs() takes an optional value per key, so the generated harness no longer compiled. Pass Some() values to the existing cases, and add these: - a None key keeps the firmware value while the other keys are still patched; - the per-SoC table, extracted from aop.rs: T8112 and T6000 keep the image nCal, T6040 and T8132 write 1, and the others write 0. Each case also runs through patch_bootargs(). Negative controls: None written as 0, T8112 forced to 0, T6040 forced to 0 and T6000 forced to 1 must each fail. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- tools/asahi/aop-bootargs-tests.py | 52 ++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/tools/asahi/aop-bootargs-tests.py b/tools/asahi/aop-bootargs-tests.py index aba3842077664..d2e24ecb07eb2 100644 --- a/tools/asahi/aop-bootargs-tests.py +++ b/tools/asahi/aop-bootargs-tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Exercise production bootarg patching with synthetic MMIO.""" +"""Exercise production bootarg patching and the per-SoC nCal choice with synthetic MMIO.""" import argparse import os from pathlib import Path @@ -40,7 +40,10 @@ if offset+table.len()<=bytes.len(){bytes[offset..offset+table.len()].copy_from_slice(table);} Device{bytes:RefCell::new(bytes),writes:RefCell::new(Vec::new()),fail:Cell::new(false)} } -fn patch(dev:&Device,word:bool)->Result{Aop.patch_bootargs(&Mmio{dev,_p:PhantomData::>},&[(1,0x12345678),(2,0x87654321)],word)} +fn patch(dev:&Device,word:bool)->Result{Aop.patch_bootargs(&Mmio{dev,_p:PhantomData::>},&[(1,Some(0x12345678)),(2,Some(0x87654321))],word)} +fn patch_with(dev:&Device,patches:&[(u32,Option)])->Result{Aop.patch_bootargs(&Mmio{dev,_p:PhantomData::>},patches,true)} +fn from_fourcc(b:&[u8;4])->u32{u32::from_le_bytes(*b)} +CONFIGS fn main(){ let table:[u8;24]=[1,0,0,0,4,0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,0,0,0,0]; for offset in 32..36 {for word in [false,true] { @@ -56,7 +59,21 @@ let d=fixture(35,&table);d.fail.set(true);assert_eq!(patch(&d,true),Err(EIO),"readback error ignored"); assert_eq!(patch(&fixture(35,&[]),true),Err(EINVAL));assert_eq!(patch(&fixture(35,&[]),false),Ok(())); assert_eq!(patch(&fixture(120,&table),true),Err(EINVAL)); - println!("PASS aligned/unaligned byte and word access, unchanged writes, malformed tables, bounds and failed readback"); + // None keeps the firmware value of its key and still patches the others. + let d=fixture(32,&table);let mut expected=d.bytes.borrow().clone();expected[32+8..32+12].copy_from_slice(&0x12345678u32.to_le_bytes()); + let fw=0x0badf00du32;d.bytes.borrow_mut()[32+20..32+24].copy_from_slice(&fw.to_le_bytes());expected[32+20..32+24].copy_from_slice(&fw.to_le_bytes()); + assert_eq!(patch_with(&d,&[(1,Some(0x12345678)),(2,None)]),Ok(()));assert_eq!(*d.bytes.borrow(),expected,"None overwrote the firmware value"); + // Per-SoC nCal: T8112 and T6000 keep the image value, T6040/T8132 write 1, the others write 0. + let ncal=from_fourcc(b"nCal"); + for (name,cfg,want) in [("T8103",&HW_CFG_T8103,None),("T8112",&HW_CFG_T8112,Some(1u32)),("T6000",&HW_CFG_T6000,Some(1)),("T6020",&HW_CFG_T6020,None),("T6040",&HW_CFG_T6040,None),("T8132",&HW_CFG_T8132,None),("T8140",&HW_CFG_T8140,None)] { + let expect=match name {"T8112"|"T6000"=>None,"T6040"|"T8132"=>Some(1u64),_=>Some(0u64)}; + assert_eq!(cfg.ncal,expect,"{} nCal choice",name); + let mut t=[0u8;12];t[0..4].copy_from_slice(&ncal.to_le_bytes());t[4..8].copy_from_slice(&4u32.to_le_bytes());t[8..12].copy_from_slice(&1u32.to_le_bytes()); + let d=fixture(32,&t);assert_eq!(patch_with(&d,&[(ncal,cfg.ncal)]),Ok(())); + let got=u32::from_le_bytes(d.bytes.borrow()[40..44].try_into().unwrap()); + let wanted=want.unwrap_or_else(||cfg.ncal.unwrap() as u32);assert_eq!(got,wanted,"{} nCal written",name); + } + println!("PASS aligned/unaligned byte and word access, unchanged writes, malformed tables, bounds, failed readback, None keeps the firmware value, per-SoC nCal"); } ''' @@ -72,13 +89,25 @@ def method(source): return source[start:end] +def configs(source): + start = source.index('struct AopHwConfig {') + end = source.index('kernel::of_device_table!', start) + return source[start:end] + + +def compile_run(rustc, p, binary, code): + p.write_text(code) + subprocess.run([rustc, '--edition=2021', str(p), '-o', str(binary)], check=True) + return subprocess.run([str(binary)], capture_output=True, text=True) + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[2] / 'drivers/soc/apple/aop.rs') args = parser.parse_args() production = method(args.source.read_text()) macros = ''.join('macro_rules! ' + name + ' {($($t:tt)*)=>{()};}\n' for name in ['dev_info', 'dev_err'] if name + '!' in production) - code = HARNESS.replace('METHOD', production).replace('struct Aop;', macros + 'struct Aop;') + code = HARNESS.replace('METHOD', production).replace('CONFIGS', configs(args.source.read_text())).replace('struct Aop;', macros + 'struct Aop;') with tempfile.TemporaryDirectory(prefix='aop-bootargs-') as tmp: p = Path(tmp) / 'test.rs' binary = p.with_suffix('') @@ -92,6 +121,21 @@ def main(): result = subprocess.run([str(binary)], capture_output=True, text=True) assert result.returncode != 0 and 'readback error ignored' in result.stderr print('PASS negative control: missing write verification') + keep = 'let Some(v) = v else {' + assert code.count(keep) == 1 + rustc = os.environ.get('RUSTC', 'rustc') + bad = code.replace(keep, 'let v = &v.unwrap_or(0); if false {') + result = compile_run(rustc, p, binary, bad) + assert result.returncode != 0 and 'None overwrote the firmware value' in result.stderr, result.stderr + print('PASS negative control: None written as 0') + for soc, val in (('T8112', 'Some(0)'), ('T6040', 'Some(0)'), ('T6000', 'Some(1)')): + start = code.index('const HW_CFG_%s: AopHwConfig' % soc) + at = code.index(' ncal: ', start) + line_end = code.index('\n', at) + bad = code[:at] + ' ncal: %s,' % val + code[line_end:] + result = compile_run(rustc, p, binary, bad) + assert result.returncode != 0 and '%s nCal choice' % soc in result.stderr, (soc, result.stderr) + print('PASS negative control: %s nCal %s' % (soc, val)) if __name__ == '__main__': From 7fd168f0148e48459ad201f4968ba4a1c1bc35cd Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:04:38 +1100 Subject: [PATCH 164/392] arm64: dts: apple: adt-coverage: fix the j413 and j316s PMGR waivers The J413 and J316s inventories waived some PMGR records as virtual votes with no power-state register. 22 J413 records and 13 J316s records do have a register. They were found by PMGR group and register offset, as for the J700. Three J413 records have a power-controller node in our tree: ISP_CPU_CORE0 and ISP_CPU_CORE1 are the isp_set1 and isp_set2 registers, and ANE_CPU is ane_sys_cpu. Map them to those nodes. The rest are display and SEP states: DISP0_* and DP on the DCP, DISPEXT*, DPEXT and DPTX_PHY on the DCPEXT, and SEP_EISP on the SEP. The ADT marks each one no_ps, so the macOS PMGR driver does not switch them either, and the coprocessor firmware does. Keep them waived, with that reason and their register address. The other 145 J413 and 155 J316s waivers are real virtual votes and stay as they are. MISSING stays 0 on both boards. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../dts/apple/adt-coverage/t6000-j316s.txt | 26 +++---- .../dts/apple/adt-coverage/t8112-j413.txt | 44 ++++++------ .../boot/dts/apple/adt-coverage/waivers.txt | 67 +++++++++---------- 3 files changed, 67 insertions(+), 70 deletions(-) diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt b/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt index 43aa7b24bb729..5baa84eeaa229 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt @@ -187,26 +187,26 @@ /arm-io/pmgr/power-devices/DISPEXT1_FE 0x28e080260 -> /soc/power-management@28e080000/power-controller@260 /arm-io/pmgr/power-devices/ANE_SYS 0x28e080268 -> /soc/power-management@28e080000/power-controller@268 /arm-io/pmgr/power-devices/AVD_SYS 0x28e080270 -> /soc/power-management@28e080000/power-controller@270 -/arm-io/pmgr/power-devices/DISPEXT0_BE - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT0_BE 0x28e080278 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/DISPEXT0_CPU0 0x28e080280 -> /soc/power-management@28e080000/power-controller@280 -/arm-io/pmgr/power-devices/DISPEXT0_GP0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT0_GP1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT0_PPP - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT1_BE - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT0_GP0 0x28e080288 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT0_GP1 0x28e080290 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT0_PPP 0x28e080298 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT1_BE 0x28e0802a0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/DISPEXT1_CPU0 0x28e0802a8 -> /soc/power-management@28e080000/power-controller@2a8 -/arm-io/pmgr/power-devices/DISPEXT1_GP0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT1_GP1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT1_PPP - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT1_GP0 0x28e0802b0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT1_GP1 0x28e0802b8 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT1_PPP 0x28e0802c0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/ANE_SYS_CPU 0x28e0802c8 -> /soc/power-management@28e080000/power-controller@2c8 -/arm-io/pmgr/power-devices/DISPEXT0_DP - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT0_DP 0x28e0802d0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/PMP 0x28e0802d8 -> /soc/power-management@28e080000/power-controller@2d8 /arm-io/pmgr/power-devices/PMS_SRAM 0x28e0802e0 -> /soc/power-management@28e080000/power-controller@2e0 /arm-io/pmgr/power-devices/APCIE_ST_SYS 0x28e0802e8 -> /soc/power-management@28e080000/power-controller@2e8 /arm-io/pmgr/power-devices/APCIE_ST1_SYS 0x28e0802f0 -> /soc/power-management@28e080000/power-controller@2f0 /arm-io/pmgr/power-devices/ATC2_COMMON 0x28e0802f8 -> /soc/power-management@28e080000/power-controller@2f8 /arm-io/pmgr/power-devices/ATC3_COMMON 0x28e080300 -> /soc/power-management@28e080000/power-controller@300 -/arm-io/pmgr/power-devices/DISPEXT0_DSC - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT1_DP - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT0_DSC 0x28e080308 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT1_DP 0x28e080310 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/USB 0x28e080318 -> /soc/power-management@28e080000/power-controller@318 /arm-io/pmgr/power-devices/APCIE_GP_SYS 0x28e080320 -> /soc/power-management@28e080000/power-controller@320 /arm-io/pmgr/power-devices/ATC0_CIO 0x28e080328 -> /soc/power-management@28e080000/power-controller@328 @@ -228,7 +228,7 @@ /arm-io/pmgr/power-devices/ATC3_CIO_PCIE 0x28e0803a8 -> /soc/power-management@28e080000/power-controller@3a8 /arm-io/pmgr/power-devices/ATC3_CIO_USB 0x28e0803b0 -> /soc/power-management@28e080000/power-controller@3b0 /arm-io/pmgr/power-devices/TRACE_FAB 0x28e0803b8 -> /soc/power-management@28e080000/power-controller@3b8 -/arm-io/pmgr/power-devices/DISPEXT1_DSC - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT1_DSC 0x28e0803c0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/CLVR_SPMI0 0x28e580100 -> /soc/power-management@28e580000/power-controller@100 /arm-io/pmgr/power-devices/CLVR_SPMI1 0x28e580108 -> /soc/power-management@28e580000/power-controller@108 /arm-io/pmgr/power-devices/CLVR_SPMI2 0x28e580110 -> /soc/power-management@28e580000/power-controller@110 @@ -320,7 +320,7 @@ /arm-io/pmgr/power-devices/GFX 0x28e5803c0 -> /soc/power-management@28e580000/power-controller@3c0 /arm-io/pmgr/power-devices/DISP0_BRC - -> WAIVED: Virtual PMGR vote; no independent power-state register. /arm-io/pmgr/power-devices/SEP 0x28e580c00 -> /soc/power-management@28e580000/power-controller@c00 -/arm-io/pmgr/power-devices/SEP_EISP - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/SEP_EISP 0x28e580c08 -> WAIVED: Power state with a register, switched by the SEP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/VENC_DMA 0x28e588000 -> /soc/power-management@28e580000/power-controller@8000 /arm-io/pmgr/power-devices/VENC_PIPE4 0x28e588008 -> /soc/power-management@28e580000/power-controller@8008 /arm-io/pmgr/power-devices/VENC_PIPE5 0x28e588010 -> /soc/power-management@28e580000/power-controller@8010 diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t8112-j413.txt b/arch/arm64/boot/dts/apple/adt-coverage/t8112-j413.txt index 8f679d077a214..ee97381307b1f 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t8112-j413.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t8112-j413.txt @@ -198,15 +198,15 @@ /arm-io/pmgr/power-devices/DISP0_FE 0x23b700378 -> /soc/power-management@23b700000/power-controller@378 /arm-io/pmgr/power-devices/DISPEXT_SYS 0x23b700380 -> /soc/power-management@23b700000/power-controller@380 /arm-io/pmgr/power-devices/DISPEXT_FE 0x23b700388 -> /soc/power-management@23b700000/power-controller@388 -/arm-io/pmgr/power-devices/DISPEXT_BE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT_GP0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT_GP1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT_GP2 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT_PPP - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DPEXT - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISPEXT_DSC - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISPEXT_BE 0x23b700390 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT_GP0 0x23b700398 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT_GP1 0x23b7003a0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT_GP2 0x23b7003a8 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT_PPP 0x23b7003b0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DPEXT 0x23b7003b8 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISPEXT_DSC 0x23b7003c0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/DISPEXT_CPU0 0x23b7003c8 -> /soc/power-management@23b700000/power-controller@3c8 -/arm-io/pmgr/power-devices/DPTX_PHY - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DPTX_PHY 0x23b7003d0 -> WAIVED: Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/DPTX_EXT_PHY 0x23b7003d8 -> /soc/power-management@23b700000/power-controller@3d8 /arm-io/pmgr/power-devices/DISPDFR_FE 0x23b7003e0 -> /soc/power-management@23b700000/power-controller@3e0 /arm-io/pmgr/power-devices/DISPDFR_BE 0x23b7003e8 -> /soc/power-management@23b700000/power-controller@3e8 @@ -236,26 +236,26 @@ /arm-io/pmgr/power-devices/ANE_SYS 0x23b7004a8 -> /soc/power-management@23b700000/power-controller@4a8 /arm-io/pmgr/power-devices/SCODEC 0x23b7004b0 -> /soc/power-management@23b700000/power-controller@4b0 /arm-io/pmgr/power-devices/SEP 0x23b700c00 -> /soc/power-management@23b700000/power-controller@c00 -/arm-io/pmgr/power-devices/SEP_EISP - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CPU_CORE0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/ISP_CPU_CORE1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/SEP_EISP 0x23b700c08 -> WAIVED: Power state with a register, switched by the SEP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/ISP_CPU_CORE0 0x23b704008 -> /soc/power-management@23b700000/power-controller@4008 +/arm-io/pmgr/power-devices/ISP_CPU_CORE1 0x23b704010 -> /soc/power-management@23b700000/power-controller@4010 /arm-io/pmgr/power-devices/VENC_DMA 0x23b708000 -> /soc/power-management@23b700000/power-controller@8000 /arm-io/pmgr/power-devices/VENC_PIPE4 0x23b708008 -> /soc/power-management@23b700000/power-controller@8008 /arm-io/pmgr/power-devices/VENC_PIPE5 0x23b708010 -> /soc/power-management@23b700000/power-controller@8010 /arm-io/pmgr/power-devices/VENC_ME0 0x23b708018 -> /soc/power-management@23b700000/power-controller@8018 /arm-io/pmgr/power-devices/VENC_ME1 0x23b708020 -> /soc/power-management@23b700000/power-controller@8020 -/arm-io/pmgr/power-devices/ANE_CPU - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/ANE_CPU 0x23b70c008 -> /soc/power-management@23b700000/power-controller@c008 /arm-io/pmgr/power-devices/DISP0_CPU0 0x23b710000 -> /soc/power-management@23b700000/power-controller@10000 -/arm-io/pmgr/power-devices/DISP0_BE - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DP - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_SPI - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_SPMI - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_GP0 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_GP1 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_GP2 - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_PPP - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_BRC - -> WAIVED: Virtual PMGR vote; no independent power-state register. -/arm-io/pmgr/power-devices/DISP0_BLC - -> WAIVED: Virtual PMGR vote; no independent power-state register. +/arm-io/pmgr/power-devices/DISP0_BE 0x23b710008 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DP 0x23b710010 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_SPI 0x23b710018 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_SPMI 0x23b710020 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_GP0 0x23b710028 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_GP1 0x23b710030 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_GP2 0x23b710038 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_PPP 0x23b710040 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_BRC 0x23b710048 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +/arm-io/pmgr/power-devices/DISP0_BLC 0x23b710050 -> WAIVED: Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. /arm-io/pmgr/power-devices/DEBUG_GATED 0x23d280058 -> /soc/power-management@23d280000/power-controller@58 /arm-io/pmgr/power-devices/NUB_SPMI0 0x23d280060 -> /soc/power-management@23d280000/power-controller@60 /arm-io/pmgr/power-devices/NUB_SPMI1 0x23d280068 -> /soc/power-management@23d280000/power-controller@68 diff --git a/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt b/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt index 097e09dd16bbe..d2f60ec751893 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/waivers.txt @@ -85,20 +85,20 @@ t6000-j316s /arm-io/pmgr/power-devices/DISP0_GP1 | Virtual PMGR vote; no indepen t6000-j316s /arm-io/pmgr/power-devices/DISP0_PPP | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/DISP0_SPI | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/DISP0_SPMI | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_BE | Virtual PMGR vote; no independent power-state register. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_BE | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_CPU0-V | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_DP | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_DSC | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_GP0 | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_GP1 | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_PPP | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_BE | Virtual PMGR vote; no independent power-state register. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_DP | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_DSC | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_GP0 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_GP1 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT0_PPP | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_BE | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_CPU0-V | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_DP | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_DSC | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_GP0 | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_GP1 | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_PPP | Virtual PMGR vote; no independent power-state register. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_DP | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_DSC | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_GP0 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_GP1 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t6000-j316s /arm-io/pmgr/power-devices/DISPEXT1_PPP | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t6000-j316s /arm-io/pmgr/power-devices/DP0-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/DP1-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/DP2-SOC-VMAX | Virtual PMGR vote; no independent power-state register. @@ -135,7 +135,7 @@ t6000-j316s /arm-io/pmgr/power-devices/SCODEC-SOC-VNOM | Virtual PMGR vote; no i t6000-j316s /arm-io/pmgr/power-devices/SCODEC-V | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/SEP-PEARL-V | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/SEP-V | Virtual PMGR vote; no independent power-state register. -t6000-j316s /arm-io/pmgr/power-devices/SEP_EISP | Virtual PMGR vote; no independent power-state register. +t6000-j316s /arm-io/pmgr/power-devices/SEP_EISP | Power state with a register, switched by the SEP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t6000-j316s /arm-io/pmgr/power-devices/SIO-CPU-V | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/SIO-DART | Virtual PMGR vote; no independent power-state register. t6000-j316s /arm-io/pmgr/power-devices/SMC_AON | Virtual PMGR vote; no independent power-state register. @@ -410,7 +410,6 @@ t6040-j616s /arm-io/pmgr/power-devices/VENC1-SYS-V | Virtual PMGR vote; no indep t6040-j616s /sacm | Logical firmware service or diagnostic view; represented through coprocessor discovery, not an independent AP device. t8112-j413 /arm-io/pmgr/power-devices/ANE-SYS-DART | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/ANE-SYS-V | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/ANE_CPU | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/AOP_ADMA0 | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/AOP_AUDIO_ADMA0 | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/AOP_AUDIO_SHIM | Virtual PMGR vote; no independent power-state register. @@ -474,28 +473,28 @@ t8112-j413 /arm-io/pmgr/power-devices/DFR-MIE-DPB | Virtual PMGR vote; no indepe t8112-j413 /arm-io/pmgr/power-devices/DISP-CARVE-OUT | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/DISP-FB-ACTIVE | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/DISP-MIE-DPB | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_BE | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_BLC | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_BRC | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP0 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP1 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP2 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_PPP | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_SPI | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISP0_SPMI | Virtual PMGR vote; no independent power-state register. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_BE | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_BLC | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_BRC | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP0 | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP1 | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_GP2 | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_PPP | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_SPI | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISP0_SPMI | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t8112-j413 /arm-io/pmgr/power-devices/DISPDFR-V | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_BE | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_DSC | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP0 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP1 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP2 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_PPP | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DP | Virtual PMGR vote; no independent power-state register. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_BE | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_DSC | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP0 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP1 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_GP2 | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DISPEXT_PPP | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DP | Power state with a register, switched by the DCP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t8112-j413 /arm-io/pmgr/power-devices/DP-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/DP0_VMAX-V | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/DP1_VMAX-V | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DPEXT | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/DPTX_PHY | Virtual PMGR vote; no independent power-state register. +t8112-j413 /arm-io/pmgr/power-devices/DPEXT | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. +t8112-j413 /arm-io/pmgr/power-devices/DPTX_PHY | Power state with a register, switched by the DCPEXT firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t8112-j413 /arm-io/pmgr/power-devices/ECPM_NRG | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/ECPU0_SRAM | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/ECPU1_SRAM | Virtual PMGR vote; no independent power-state register. @@ -510,8 +509,6 @@ t8112-j413 /arm-io/pmgr/power-devices/ISP-SOC-VMAX | Virtual PMGR vote; no indep t8112-j413 /arm-io/pmgr/power-devices/ISP-SOC-VNOM | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/ISP-SYS-DART | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/ISP-SYS-V | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/ISP_CPU_CORE0 | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/ISP_CPU_CORE1 | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/JPG-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/JPG-SOC-VNOM | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/JPG0-DART | Virtual PMGR vote; no independent power-state register. @@ -544,7 +541,7 @@ t8112-j413 /arm-io/pmgr/power-devices/SCDC_VNOM-V | Virtual PMGR vote; no indepe t8112-j413 /arm-io/pmgr/power-devices/SDEC-SOC-VMAX | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/SDEC-SOC-VNOM | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/SEP-PEARL-V | Virtual PMGR vote; no independent power-state register. -t8112-j413 /arm-io/pmgr/power-devices/SEP_EISP | Virtual PMGR vote; no independent power-state register. +t8112-j413 /arm-io/pmgr/power-devices/SEP_EISP | Power state with a register, switched by the SEP firmware. The ADT marks it no_ps, so the macOS PMGR driver leaves it alone too; no DT power controller. t8112-j413 /arm-io/pmgr/power-devices/SIO-CPU-DART | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/SIO-CPU-V | Virtual PMGR vote; no independent power-state register. t8112-j413 /arm-io/pmgr/power-devices/SMC_AON | Virtual PMGR vote; no independent power-state register. From 7a44c0a85d8fa66b2da2f38c3b42b67c3ba49963 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:33:44 +1100 Subject: [PATCH 165/392] spmi: apple: take the interrupt count from the SoC The controller numbers its peripheral interrupts first and puts the RX FIFO notification right after them. On t8103, t8112 and t8140 that is 256 peripheral interrupts with the notification at 256, which the driver hard-codes. The M1 Pro, Max and Ultra (t6000) controller has 192 peripheral interrupts and raises the notification as interrupt 192, as the firmware device tree for the nub SPMI bus lists it. With the nub SPMI controller described as an interrupt controller, the driver unmasked bit 256 and waited for a completion that never came, so every transfer on an M1 Pro timed out ("failed to wait for RX FIFO not empty") and the SMC RTC, the reboot nvmem cells and the PMU RTC all failed to read. Add the interrupt count to the per-SoC data, use it for the notification index, the mask/ack ranges and the domain bounds, and match apple,t6000-spmi with a 192-interrupt layout. Other SoCs keep 256. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/spmi/spmi-apple-controller.c | 48 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index fbdb9ba0e7f2a..58ada41ed96ec 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -34,10 +34,12 @@ #define SPMI_IRQ_MASK_BASE 0x20 #define SPMI_IRQ_ACK_BASE 0x60 -#define SPMI_NUM_PERIPHERAL_IRQS 256 -#define SPMI_NUM_IRQS (SPMI_NUM_PERIPHERAL_IRQS + 32) - -#define SPMI_IRQ_NOTIFY 256 +/* + * Peripheral interrupts come first. The RX FIFO notification sits right + * after them, followed by the controller's own fault interrupts, so the + * notification index is the number of peripheral interrupts. + */ +#define SPMI_MAX_PERIPHERAL_IRQS 256 /* SPMI_RSP_REG reply word */ #define SPMI_REPLY_FRAME_PARITY_STATUS GENMASK(31, 16) @@ -57,6 +59,7 @@ struct apple_spmi_hw { u32 reply; u32 rx_empty; bool legacy_irqs; + u32 num_irqs; }; static const struct apple_spmi_hw apple_spmi_gen1 = { @@ -65,6 +68,17 @@ static const struct apple_spmi_hw apple_spmi_gen1 = { .reply = SPMI_RSP_REG, .rx_empty = SPMI_RX_FIFO_EMPTY, .legacy_irqs = true, + .num_irqs = 256, +}; + +/* M1 Pro, Max and Ultra: 192 peripheral interrupts, notification at 192 */ +static const struct apple_spmi_hw apple_spmi_t6000 = { + .status = SPMI_STATUS_REG, + .command = SPMI_CMD_REG, + .reply = SPMI_RSP_REG, + .rx_empty = SPMI_RX_FIFO_EMPTY, + .legacy_irqs = true, + .num_irqs = 192, }; static const struct apple_spmi_hw apple_spmi_gen4 = { @@ -81,7 +95,7 @@ struct apple_spmi { struct completion fifo_rx; struct irq_domain *irqd; raw_spinlock_t irq_mask_lock; - DECLARE_BITMAP(irq_mask_cache, SPMI_NUM_PERIPHERAL_IRQS); + DECLARE_BITMAP(irq_mask_cache, SPMI_MAX_PERIPHERAL_IRQS); int irq; bool notify_irq; bool prev_fail; @@ -190,7 +204,7 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, u16 param, if (!spmi->hw->legacy_irqs) return -EIO; writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_ACT_REG); - apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + apple_spmi_irq_ack_raw(spmi, spmi->hw->num_irqs); spmi->prev_fail = false; } reinit_completion(&spmi->fifo_rx); @@ -335,12 +349,13 @@ static int apple_spmi_irq_domain_translate(struct irq_domain *irqd, unsigned long *hwirq, unsigned int *type) { + struct apple_spmi *spmi = irqd->host_data; u32 *args = fwspec->param; if (fwspec->param_count != 2) return -EINVAL; - if (args[0] >= SPMI_NUM_PERIPHERAL_IRQS) + if (args[0] >= spmi->hw->num_irqs) return -EINVAL; *hwirq = args[0]; *type = args[1] & IRQ_TYPE_SENSE_MASK; @@ -352,6 +367,7 @@ static int apple_spmi_irq_domain_alloc(struct irq_domain *irqd, unsigned int vir { unsigned int type = IRQ_TYPE_NONE; struct irq_fwspec *fwspec = arg; + struct apple_spmi *spmi = irqd->host_data; irq_hw_number_t hwirq; int i, ret; @@ -359,7 +375,7 @@ static int apple_spmi_irq_domain_alloc(struct irq_domain *irqd, unsigned int vir if (ret) return ret; - if (hwirq + nr_irqs > SPMI_NUM_PERIPHERAL_IRQS) + if (hwirq + nr_irqs > spmi->hw->num_irqs) return -EINVAL; for (i = 0; i < nr_irqs; i++) { @@ -394,18 +410,19 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) { struct apple_spmi *spmi = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); + u32 notify = spmi->hw->num_irqs; bool handled = false; unsigned long val, offset, bit; chained_irq_enter(chip, desc); - val = readl(spmi->regs + SPMI_IRQ_ACK_BASE + (SPMI_IRQ_NOTIFY / 32) * 4); - if (val & BIT(SPMI_IRQ_NOTIFY % 32)) { - apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); + val = readl(spmi->regs + SPMI_IRQ_ACK_BASE + (notify / 32) * 4); + if (val & BIT(notify % 32)) { + apple_spmi_irq_ack_raw(spmi, notify); complete(&spmi->fifo_rx); handled = true; } - for (offset = 0; offset < SPMI_NUM_PERIPHERAL_IRQS / 8; offset += sizeof(val)) { + for (offset = 0; offset < notify / 8; offset += sizeof(val)) { val = readq(spmi->regs + SPMI_IRQ_ACK_BASE + offset); /** * because of other masters in the bus, we're going to get a multitude of @@ -428,7 +445,7 @@ static void apple_spmi_teardown_irq(void *data) { struct apple_spmi *spmi = data; - for (size_t offset = 0; offset < SPMI_NUM_IRQS / 8; offset += 4) + for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); synchronize_irq(spmi->irq); @@ -448,7 +465,7 @@ static int apple_spmi_init_irq(struct platform_device *pdev, raw_spin_lock_init(&spmi->irq_mask_lock); - for (size_t offset = 0; offset < SPMI_NUM_IRQS / 8; offset += 4) { + for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) { writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); } @@ -462,7 +479,7 @@ static int apple_spmi_init_irq(struct platform_device *pdev, return ret; irq_set_chained_handler_and_data(spmi->irq, apple_spmi_irq_handler, spmi); - apple_spmi_irq_unmask_raw(spmi, SPMI_IRQ_NOTIFY); + apple_spmi_irq_unmask_raw(spmi, spmi->hw->num_irqs); spmi->notify_irq = true; return 0; @@ -519,6 +536,7 @@ static int apple_spmi_probe(struct platform_device *pdev) static const struct of_device_id apple_spmi_match_table[] = { { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, + { .compatible = "apple,t6000-spmi", .data = &apple_spmi_t6000 }, { .compatible = "apple,spmi", .data = &apple_spmi_gen1 }, {} }; From c85f88bc3866e5926c20fb3537fa150bd357490e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:33:58 +1100 Subject: [PATCH 166/392] arm64: dts: apple: t600x: describe the PMU RTC alarm Make the nub SPMI controller an interrupt controller and add the Maverick PMU RTC under the PMIC, so rtcwake can wake M1 Pro, Max and Ultra systems from s2idle. The PMU raises SPMI interrupt 2 for the alarm. Keep rtc0 on the SMC clock and give the alarm rtc1 on every t600x board. The t602x boards share the board files but not this PMU, so the aliases go in the t600x board files. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 2 ++ arch/arm64/boot/dts/apple/t6000-j316s.dts | 2 ++ arch/arm64/boot/dts/apple/t6001-j314c.dts | 2 ++ arch/arm64/boot/dts/apple/t6001-j316c.dts | 2 ++ arch/arm64/boot/dts/apple/t6001-j375c.dts | 5 +++++ arch/arm64/boot/dts/apple/t6002-j375d.dts | 2 ++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 15 ++++++++++++++- 7 files changed, 29 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index a3c26b7819d43..2286e31c8f59a 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -24,6 +24,8 @@ model = "Apple MacBook Pro (14-inch, M1 Pro, 2021)"; aliases { + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; sep = &sep; }; }; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 9e4bd2ae50b03..b08c10046e83c 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -23,6 +23,8 @@ model = "Apple MacBook Pro (16-inch, M1 Pro, 2021)"; aliases { + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; sep = &sep; }; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index 23be760ba6e49..1f7b1e4d2827c 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -17,6 +17,8 @@ model = "Apple MacBook Pro (14-inch, M1 Max, 2021)"; aliases { + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; sep = &sep; }; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index d171d9c1200b8..f2aa92e7a79f3 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -17,6 +17,8 @@ model = "Apple MacBook Pro (16-inch, M1 Max, 2021)"; aliases { + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; sep = &sep; }; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j375c.dts b/arch/arm64/boot/dts/apple/t6001-j375c.dts index 35cd50028b75b..dd0864540b9ba 100644 --- a/arch/arm64/boot/dts/apple/t6001-j375c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j375c.dts @@ -15,6 +15,11 @@ / { compatible = "apple,j375c", "apple,t6001", "apple,arm-platform"; model = "Apple Mac Studio (M1 Max, 2022)"; + + aliases { + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; + }; }; &wifi0 { diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index a16df355852ef..a148a7325d9d4 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -18,6 +18,8 @@ aliases { atcphy4 = &atcphy0_die1; atcphy5 = &atcphy1_die1; + rtc0 = &smc_rtc; + rtc1 = &pmu_rtc; usb4-4-acio = &usb4_0_acio_die1; usb4-4-nhi = &usb4_0_nhi_die1; usb4-5-acio = &usb4_1_acio_die1; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 6f028f884a703..5e1df81ed234e 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -363,7 +363,7 @@ compatible = "apple,smc-hwmon"; }; - rtc { + smc_rtc: rtc { compatible = "apple,smc-rtc"; nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; @@ -409,11 +409,24 @@ reg = <0x2 0x920a1300 0x0 0x100>; #address-cells = <2>; #size-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupt-parent = <&aic>; + interrupts = ; pmic1: pmic@f { compatible = "apple,maverick-pmic", "apple,spmi-nvmem"; reg = <0xf SPMI_USID>; + pmu_rtc: rtc { + compatible = "apple,t6000-pmu-rtc"; + interrupt-parent = <&nub_spmi0>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + nvmem-layout { compatible = "fixed-layout"; #address-cells = <1>; From 37888b11fb7a6a0faec665cdb95f6fd561847392 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:46:09 +1100 Subject: [PATCH 167/392] media: apple: add a probe-only driver for the scaler/CSC engine Every Apple silicon SoC has one or more MSR engines: memory-to-memory scalers with a programmable colour-space converter, chroma resampling, flip/rotate and alpha fill. macOS reaches them through IOSurfaceAccelerator and VideoToolbox; Linux has had no driver, so scaling and YUV/RGB conversion fall back to the CPU or GPU. Hardware-version 10 (M1) through 22 (A18 Pro) program the engine through MMIO and signal completion with an interrupt. This first step binds the existing apple,scaler nodes for T6041 (M4 Pro/Max) and T8140 (A18 Pro): it attaches the MSR power domains with runtime-PM links, requires the DART, reads the identity and idle state, clears and masks the interrupt and keeps the engine idle. The video device and job submission follow once the RGB path is qualified on hardware. The DT nodes stay disabled, so nothing changes on existing systems. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- MAINTAINERS | 8 + arch/arm64/configs/asahi.config | 1 + drivers/media/platform/apple/Kconfig | 1 + drivers/media/platform/apple/Makefile | 1 + drivers/media/platform/apple/scaler/Kconfig | 16 ++ drivers/media/platform/apple/scaler/Makefile | 4 + .../platform/apple/scaler/apple-scaler-drv.c | 227 ++++++++++++++++++ .../platform/apple/scaler/apple-scaler-regs.h | 37 +++ 8 files changed, 295 insertions(+) create mode 100644 drivers/media/platform/apple/scaler/Kconfig create mode 100644 drivers/media/platform/apple/scaler/Makefile create mode 100644 drivers/media/platform/apple/scaler/apple-scaler-drv.c create mode 100644 drivers/media/platform/apple/scaler/apple-scaler-regs.h diff --git a/MAINTAINERS b/MAINTAINERS index 78e5d506eff3a..fc20e250d5476 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2844,6 +2844,14 @@ S: Maintained F: Documentation/devicetree/bindings/spi/apple,qspi.yaml F: drivers/spi/spi-apple-qspi.c +ARM/APPLE SCALER DRIVER +M: Ryan Murray +L: linux-media@vger.kernel.org +S: Maintained +W: https://aurora-silicon.github.io +F: Documentation/devicetree/bindings/media/apple,scaler.yaml +F: drivers/media/platform/apple/scaler/ + ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 611fc646ffbcf..76919c3e257d8 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -60,6 +60,7 @@ CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_VIDEO_APPLE_PRORES=m CONFIG_VIDEO_APPLE_JPEG=m +CONFIG_VIDEO_APPLE_SCALER=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m CONFIG_DRM_ASAHI_G15=y diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig index 2d7f7f9e176b4..83c6538fe6547 100644 --- a/drivers/media/platform/apple/Kconfig +++ b/drivers/media/platform/apple/Kconfig @@ -7,3 +7,4 @@ source "drivers/media/platform/apple/avd/Kconfig" source "drivers/media/platform/apple/ave/Kconfig" source "drivers/media/platform/apple/jpeg/Kconfig" source "drivers/media/platform/apple/prores/Kconfig" +source "drivers/media/platform/apple/scaler/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile index 0d5814d987671..eb0b9cb65952c 100644 --- a/drivers/media/platform/apple/Makefile +++ b/drivers/media/platform/apple/Makefile @@ -5,3 +5,4 @@ obj-y += avd/ obj-y += ave/ obj-y += jpeg/ obj-y += prores/ +obj-y += scaler/ diff --git a/drivers/media/platform/apple/scaler/Kconfig b/drivers/media/platform/apple/scaler/Kconfig new file mode 100644 index 0000000000000..a2b72a3fec804 --- /dev/null +++ b/drivers/media/platform/apple/scaler/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_SCALER + tristate "Apple Silicon scaler and colour-space converter" + depends on VIDEO_DEV + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + depends on OF_ADDRESS && IOMMU_API && PM + help + Driver for the memory-to-memory image scaler and colour-space + converter (MSR) in Apple silicon SoCs. This first version + identifies the engine, attaches its power domains, IOMMU and + interrupt, and keeps it idle; it does not yet expose a video + device. + + To compile this driver as a module, choose M here: the module + will be called apple-scaler. diff --git a/drivers/media/platform/apple/scaler/Makefile b/drivers/media/platform/apple/scaler/Makefile new file mode 100644 index 0000000000000..ab203c242b6d4 --- /dev/null +++ b/drivers/media/platform/apple/scaler/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_SCALER) += apple-scaler.o +apple-scaler-y := apple-scaler-drv.o diff --git a/drivers/media/platform/apple/scaler/apple-scaler-drv.c b/drivers/media/platform/apple/scaler/apple-scaler-drv.c new file mode 100644 index 0000000000000..37fd72719bac1 --- /dev/null +++ b/drivers/media/platform/apple/scaler/apple-scaler-drv.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple silicon scaler and colour-space converter (MSR). + * + * Every Apple silicon SoC has one or more MSR engines: memory-to-memory + * scalers with a programmable colour-space converter, chroma resampling, + * flip/rotate and alpha fill. Hardware-version 10 (M1) through 22 (A18 Pro) + * are programmed directly through MMIO and report completion with an + * interrupt; later generations front the engine with firmware and are not + * handled here. + * + * This version only brings the engine up: it attaches the power domains, + * checks the IOMMU, identifies the hardware, takes the completion + * interrupt and keeps the engine idle. Jobs and the V4L2 mem2mem device + * come in later patches. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple-scaler-regs.h" + +#define ASC_NAME "apple-scaler" +#define ASC_MIN_REG_SIZE 0x28000 + +/** + * struct apple_scaler_variant - per-SoC engine description + * @hw_version: MSR hardware-version of this SoC (from the hardware description) + * @num_reg: number of register apertures the binding describes + */ +struct apple_scaler_variant { + u32 hw_version; + unsigned int num_reg; +}; + +struct apple_scaler { + struct device *dev; + void __iomem *regs; + const struct apple_scaler_variant *variant; + struct dev_pm_domain_list *pd_list; + struct reset_control *rst; + int irq; + u32 id; +}; + +static u32 asc_read(struct apple_scaler *asc, u32 reg) +{ + return readl_relaxed(asc->regs + reg); +} + +static void asc_write(struct apple_scaler *asc, u32 reg, u32 val) +{ + writel_relaxed(val, asc->regs + reg); +} + +/* + * No job is ever started yet, so any interrupt is unexpected: acknowledge + * it and report it. The handler only touches the engine while it is + * runtime-active, because a powered-down MSR must not be read. + */ +static irqreturn_t asc_irq(int irq, void *data) +{ + struct apple_scaler *asc = data; + u32 status; + + if (pm_runtime_get_if_active(asc->dev) <= 0) + return IRQ_NONE; + + status = asc_read(asc, ASC_IRQ_STATUS); + if (status) { + asc_write(asc, ASC_IRQ_STATUS, status); + dev_warn_ratelimited(asc->dev, "unexpected interrupt, status %#x\n", + status); + } + pm_runtime_put_autosuspend(asc->dev); + return status ? IRQ_HANDLED : IRQ_NONE; +} + +static int asc_identify(struct apple_scaler *asc) +{ + u32 running, status; + int ret; + + ret = pm_runtime_resume_and_get(asc->dev); + if (ret < 0) + return dev_err_probe(asc->dev, ret, "failed to power up\n"); + + asc->id = asc_read(asc, ASC_HW_VERSION); + running = asc_read(asc, ASC_IS_RUNNING); + status = asc_read(asc, ASC_IRQ_STATUS); + /* Nothing may be enabled until a job path owns the engine. */ + asc_write(asc, ASC_IRQ_ENABLE, 0); + if (status) + asc_write(asc, ASC_IRQ_STATUS, status); + + pm_runtime_put_sync_suspend(asc->dev); + + if (running) + dev_warn(asc->dev, "engine busy at probe (%#x)\n", running); + dev_info(asc->dev, "MSR hardware-version %u, id %#x, stale status %#x\n", + asc->variant->hw_version, asc->id, status); + return 0; +} + +static int asc_runtime_suspend(struct device *dev) +{ + return 0; +} + +static int asc_runtime_resume(struct device *dev) +{ + /* The engine loses its state when powered down; jobs program it fully. */ + return 0; +} + +static const struct dev_pm_ops asc_pm_ops = { + RUNTIME_PM_OPS(asc_runtime_suspend, asc_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +static int asc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_scaler *asc; + struct resource *res; + int ret; + + asc = devm_kzalloc(dev, sizeof(*asc), GFP_KERNEL); + if (!asc) + return -ENOMEM; + asc->dev = dev; + asc->variant = of_device_get_match_data(dev); + if (!asc->variant) + return -ENODEV; + platform_set_drvdata(pdev, asc); + + if (!device_iommu_mapped(dev)) + return dev_err_probe(dev, -ENODEV, "no IOMMU\n"); + + /* + * MSR-V and the ASE core domain below it; link both so they are on + * exactly while the engine is runtime-active. + */ + if (of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells") > 1) { + static const struct dev_pm_domain_attach_data pd_data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + + ret = devm_pm_domain_attach_list(dev, &pd_data, &asc->pd_list); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to attach power domains\n"); + } else if (!dev->pm_domain) { + return dev_err_probe(dev, -ENODEV, "requires a power domain\n"); + } + + if (platform_get_resource(pdev, IORESOURCE_MEM, asc->variant->num_reg - 1) == NULL) + return dev_err_probe(dev, -EINVAL, "expected %u register ranges\n", + asc->variant->num_reg); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (resource_size(res) < ASC_MIN_REG_SIZE) + return dev_err_probe(dev, -EINVAL, "MSR aperture too small\n"); + asc->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(asc->regs)) + return PTR_ERR(asc->regs); + + asc->rst = devm_reset_control_get_optional_exclusive(dev, NULL); + if (IS_ERR(asc->rst)) + return dev_err_probe(dev, PTR_ERR(asc->rst), "failed to get reset\n"); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + ret = asc_identify(asc); + if (ret) + return ret; + + /* Interrupt 0 is the MSR (completion/error); 1 is the PIODMA engine. */ + asc->irq = platform_get_irq(pdev, 0); + if (asc->irq < 0) + return asc->irq; + return devm_request_irq(dev, asc->irq, asc_irq, 0, dev_name(dev), asc); +} + +static const struct apple_scaler_variant apple_scaler_t6041 = { + .hw_version = 21, + .num_reg = 4, +}; + +static const struct apple_scaler_variant apple_scaler_t8140 = { + .hw_version = 22, + .num_reg = 4, +}; + +static const struct of_device_id asc_of_match[] = { + { .compatible = "apple,t6041-scaler", .data = &apple_scaler_t6041 }, + { .compatible = "apple,t8140-scaler", .data = &apple_scaler_t8140 }, + { } +}; +MODULE_DEVICE_TABLE(of, asc_of_match); + +static struct platform_driver asc_driver = { + .probe = asc_probe, + .driver = { + .name = ASC_NAME, + .of_match_table = asc_of_match, + .pm = pm_ptr(&asc_pm_ops), + }, +}; +module_platform_driver(asc_driver); + +MODULE_DESCRIPTION("Apple silicon scaler and colour-space converter"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-regs.h b/drivers/media/platform/apple/scaler/apple-scaler-regs.h new file mode 100644 index 0000000000000..7402b9a97752c --- /dev/null +++ b/drivers/media/platform/apple/scaler/apple-scaler-regs.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple silicon scaler/CSC (MSR) register map, global block. + * + * Offsets are bytes from the start of the MSR aperture (reg index 0). + * The global control registers are common to every direct-MMIO MSR + * generation (hardware-version 10 through 22). + */ +#ifndef APPLE_SCALER_REGS_H +#define APPLE_SCALER_REGS_H + +#include + +#define ASC_HW_VERSION 0x0000 +#define ASC_RESET 0x0004 +#define ASC_RESET_APIODMA 0x0008 +#define ASC_IS_RUNNING 0x000c +#define ASC_REG_FIFO_AVAIL 0x0014 +#define ASC_IRQ_ENABLE 0x0018 +#define ASC_IRQ_STATUS 0x001c +#define ASC_IRQ_DONE BIT(0) +#define ASC_IRQ_DEBUG_ERROR BIT(1) +#define ASC_IRQ_READ_ERROR BIT(3) +#define ASC_IRQ_WRITE_ERROR BIT(4) +#define ASC_IRQ_DECOMPRESS_ERROR BIT(9) +#define ASC_IRQ_CONTEXT_SWITCH BIT(10) +#define ASC_IRQ_AXI_ERROR BIT(12) +#define ASC_IRQ_ERRORS (ASC_IRQ_DEBUG_ERROR | ASC_IRQ_READ_ERROR | \ + ASC_IRQ_WRITE_ERROR | ASC_IRQ_DECOMPRESS_ERROR | \ + ASC_IRQ_AXI_ERROR) +#define ASC_FRAME_COUNT 0x0020 +#define ASC_START 0x0080 +#define ASC_PRE_START 0x0098 +#define ASC_DEBUG_STATUS 0x009c +#define ASC_TRANSFORM_ID 0x0110 + +#endif /* APPLE_SCALER_REGS_H */ From eabe8ca3a3a5482ef47b5f26e63409b721d3cb6b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:50:20 +1100 Subject: [PATCH 168/392] media: apple: scaler: add the format, CSC and ratio encodings The job path needs three pieces of register encoding that do not depend on the hardware being present, so add them with KUnit coverage first: - the formats the engine is known to take: 32-bit B,G,R,A (ARGB32 and XRGB32 in V4L2 terms), NV12/NV12M and P010, with their layout class, format code, subsampling descriptor and component order; - the colour-space converter: a 3x3 Q3.12 matrix with input and output offsets (2^-24 of full scale, 26-bit signed), derived from the Kr/Kb luma coefficients of BT.601, BT.709 and BT.2020 for limited and full range, in both directions; - the scaler ratio register, Q22 with the two lowest bits clear. The tests compare against the register values used for the same conversions in a capture of the engine on macOS 27 on an M4 Pro: all six matrices and the offsets match bit for bit. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .../media/platform/apple/scaler/.kunitconfig | 14 ++ drivers/media/platform/apple/scaler/Kconfig | 9 + drivers/media/platform/apple/scaler/Makefile | 3 +- .../platform/apple/scaler/apple-scaler-hw.c | 192 ++++++++++++++++++ .../platform/apple/scaler/apple-scaler-hw.h | 84 ++++++++ .../platform/apple/scaler/apple-scaler-test.c | 158 ++++++++++++++ 6 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 drivers/media/platform/apple/scaler/.kunitconfig create mode 100644 drivers/media/platform/apple/scaler/apple-scaler-hw.c create mode 100644 drivers/media/platform/apple/scaler/apple-scaler-hw.h create mode 100644 drivers/media/platform/apple/scaler/apple-scaler-test.c diff --git a/drivers/media/platform/apple/scaler/.kunitconfig b/drivers/media/platform/apple/scaler/.kunitconfig new file mode 100644 index 0000000000000..84cb41aee612d --- /dev/null +++ b/drivers/media/platform/apple/scaler/.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_ARCH_APPLE=y +CONFIG_PM=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_OF=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_IOMMU_DMA=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_SUPPORT_FILTER=n +CONFIG_VIDEO_DEV=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_SCALER=y +CONFIG_VIDEO_APPLE_SCALER_KUNIT_TEST=y diff --git a/drivers/media/platform/apple/scaler/Kconfig b/drivers/media/platform/apple/scaler/Kconfig index a2b72a3fec804..799707d9938d1 100644 --- a/drivers/media/platform/apple/scaler/Kconfig +++ b/drivers/media/platform/apple/scaler/Kconfig @@ -14,3 +14,12 @@ config VIDEO_APPLE_SCALER To compile this driver as a module, choose M here: the module will be called apple-scaler. + +config VIDEO_APPLE_SCALER_KUNIT_TEST + tristate "KUnit tests for the Apple scaler" if !KUNIT_ALL_TESTS + depends on KUNIT && VIDEO_APPLE_SCALER + default KUNIT_ALL_TESTS + help + Check the colour-conversion matrices, offsets, scale ratios and + format encodings the scaler driver programs. No hardware is + needed. diff --git a/drivers/media/platform/apple/scaler/Makefile b/drivers/media/platform/apple/scaler/Makefile index ab203c242b6d4..d6a9dcbe9b33e 100644 --- a/drivers/media/platform/apple/scaler/Makefile +++ b/drivers/media/platform/apple/scaler/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_VIDEO_APPLE_SCALER) += apple-scaler.o -apple-scaler-y := apple-scaler-drv.o +apple-scaler-y := apple-scaler-drv.o apple-scaler-hw.o +obj-$(CONFIG_VIDEO_APPLE_SCALER_KUNIT_TEST) += apple-scaler-test.o diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.c b/drivers/media/platform/apple/scaler/apple-scaler-hw.c new file mode 100644 index 0000000000000..2cbff1aee9e3d --- /dev/null +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple silicon scaler/CSC (MSR): format table, colour matrices and + * scale ratios in register encoding. + * + * The colour matrices are derived here from the luma coefficients the + * standards define (Kr, Kb), not looked up, so every encoding/range pair + * comes from one formula. The format codes and component orders are the + * values the engine is driven with for those layouts. + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "apple-scaler-hw.h" + +#define ASC_SWIZZLE_IDENTITY 0x03020100 +#define ASC_SWIZZLE_CBCR 0x03000102 + +static const struct asc_format asc_formats[] = { + { + /* Bytes B, G, R, A in memory. */ + .fourcc = V4L2_PIX_FMT_ARGB32, .layout = 0x1, .code = 0x1e, + .sampling = 0x0, .swizzle = ASC_SWIZZLE_IDENTITY, .planes = 1, + .bpp = { 4 }, .hsub = 1, .vsub = 1, .alpha = true, + }, { + .fourcc = V4L2_PIX_FMT_XRGB32, .layout = 0x1, .code = 0x1e, + .sampling = 0x0, .swizzle = ASC_SWIZZLE_IDENTITY, .planes = 1, + .bpp = { 4 }, .hsub = 1, .vsub = 1, + }, { + .fourcc = V4L2_PIX_FMT_NV12M, .layout = 0xc, .code = 0x1, + .sampling = 0x6, .swizzle = ASC_SWIZZLE_CBCR, .planes = 2, + .bpp = { 1, 2 }, .hsub = 2, .vsub = 2, .yuv = true, + }, { + .fourcc = V4L2_PIX_FMT_NV12, .layout = 0xc, .code = 0x1, + .sampling = 0x6, .swizzle = ASC_SWIZZLE_CBCR, .planes = 2, + .bpp = { 1, 2 }, .hsub = 2, .vsub = 2, .yuv = true, + }, { + /* 10 bits in the top of each 16-bit sample. */ + .fourcc = V4L2_PIX_FMT_P010, .layout = 0xc, .code = 0x2, + .sampling = 0x12, .swizzle = ASC_SWIZZLE_CBCR, .planes = 2, + .bpp = { 2, 4 }, .hsub = 2, .vsub = 2, .yuv = true, + }, +}; + +const struct asc_format *asc_format_by_index(unsigned int index) +{ + return index < ARRAY_SIZE(asc_formats) ? &asc_formats[index] : NULL; +} +EXPORT_SYMBOL_IF_KUNIT(asc_format_by_index); + +const struct asc_format *asc_find_format(u32 fourcc) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(asc_formats); i++) + if (asc_formats[i].fourcc == fourcc) + return &asc_formats[i]; + return NULL; +} +EXPORT_SYMBOL_IF_KUNIT(asc_find_format); + +/* Luma coefficients Kr and Kb in units of 1/10000. */ +struct asc_luma { + s64 kr, kb; +}; + +static int asc_luma_coefs(enum v4l2_ycbcr_encoding enc, struct asc_luma *l) +{ + switch (enc) { + case V4L2_YCBCR_ENC_601: + case V4L2_YCBCR_ENC_XV601: + *l = (struct asc_luma){ 2990, 1140 }; + return 0; + case V4L2_YCBCR_ENC_709: + case V4L2_YCBCR_ENC_XV709: + *l = (struct asc_luma){ 2126, 722 }; + return 0; + case V4L2_YCBCR_ENC_BT2020: + *l = (struct asc_luma){ 2627, 593 }; + return 0; + default: + return -EINVAL; + } +} + +/* round(num / den) for signed num and positive den */ +static s64 asc_div_round(s64 num, s64 den) +{ + return num >= 0 ? div64_s64(num + den / 2, den) : -div64_s64(-num + den / 2, den); +} + +/* A coefficient num/den (with the range scale sn/sd applied) in Q3.12. */ +static s16 asc_q12(s64 num, s64 den, s64 sn, s64 sd) +{ + return (s16)asc_div_round(num * sn << ASC_CSC_COEF_SHIFT, den * sd); +} + +/* + * Offsets are in units of 2^-24 of full scale. The engine works on a + * 16-bit internal scale, so the limited-range black level (8-bit code 16) + * is 0x1010 there; the chroma midpoint is 0x8020 (10-bit code 512). These + * are the levels the engine is driven with on macOS for every encoding + * and depth. + */ +#define ASC_LUMA_BLACK (0x1010 << 8) +#define ASC_CHROMA_MID (0x8020 << 8) + +/** + * asc_csc_compute - colour conversion between Y'CbCr and R'G'B' + * @enc: Y'CbCr encoding (luma coefficients) + * @quant: range of the Y'CbCr side (the R'G'B' side is full range) + * @yuv_to_rgb: direction + * @csc: result in register encoding + * + * Return: 0, or -EINVAL for an encoding the matrix cannot express. + */ +int asc_csc_compute(enum v4l2_ycbcr_encoding enc, + enum v4l2_quantization quant, bool yuv_to_rgb, + struct asc_csc *csc) +{ + const s64 one = 10000; + bool limited = quant != V4L2_QUANTIZATION_FULL_RANGE; + /* Range scales: luma 219/255 and chroma 224/255 for limited range. */ + s64 yn = limited ? 219 : 1, yd = limited ? 255 : 1; + s64 cn = limited ? 224 : 1, cd = limited ? 255 : 1; + struct asc_luma l; + s64 kg; + int i; + + if (asc_luma_coefs(enc, &l)) + return -EINVAL; + kg = one - l.kr - l.kb; + memset(csc, 0, sizeof(*csc)); + + if (yuv_to_rgb) { + /* R = Y + 2(1-Kr) Cr, B = Y + 2(1-Kb) Cb, G from Y = Kr R + Kg G + Kb B */ + for (i = 0; i < 3; i++) + csc->coef[i][0] = asc_q12(1, 1, yd, yn); + csc->coef[0][2] = asc_q12(2 * (one - l.kr), one, cd, cn); + csc->coef[1][1] = asc_q12(-2 * l.kb * (one - l.kb), one * kg, cd, cn); + csc->coef[1][2] = asc_q12(-2 * l.kr * (one - l.kr), one * kg, cd, cn); + csc->coef[2][1] = asc_q12(2 * (one - l.kb), one, cd, cn); + csc->pre[0] = limited ? -ASC_LUMA_BLACK : 0; + csc->pre[1] = -ASC_CHROMA_MID; + csc->pre[2] = -ASC_CHROMA_MID; + } else { + /* Y = Kr R + Kg G + Kb B, Cb = (B - Y) / 2(1-Kb), Cr = (R - Y) / 2(1-Kr) */ + csc->coef[0][0] = asc_q12(l.kr, one, yn, yd); + csc->coef[0][1] = asc_q12(kg, one, yn, yd); + csc->coef[0][2] = asc_q12(l.kb, one, yn, yd); + csc->coef[1][0] = asc_q12(-l.kr, 2 * (one - l.kb), cn, cd); + csc->coef[1][1] = asc_q12(-kg, 2 * (one - l.kb), cn, cd); + csc->coef[1][2] = asc_q12(1, 2, cn, cd); + csc->coef[2][0] = asc_q12(1, 2, cn, cd); + csc->coef[2][1] = asc_q12(-kg, 2 * (one - l.kr), cn, cd); + csc->coef[2][2] = asc_q12(-l.kb, 2 * (one - l.kr), cn, cd); + csc->post[0] = limited ? ASC_LUMA_BLACK : 0; + csc->post[1] = ASC_CHROMA_MID; + csc->post[2] = ASC_CHROMA_MID; + } + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(asc_csc_compute); + +u32 asc_csc_offset_reg(s32 offset) +{ + return (u32)offset & ASC_CSC_OFFSET_MASK; +} +EXPORT_SYMBOL_IF_KUNIT(asc_csc_offset_reg); + +/** + * asc_scale_ratio - one scaler ratio register value + * @in: input extent in pixels + * @out: output extent in pixels + * + * Return: in/out in Q22 with the two lowest bits clear, as the engine + * takes it; 0 if @out is 0. + */ +u32 asc_scale_ratio(u32 in, u32 out) +{ + if (!out) + return 0; + return (u32)div_u64((u64)in << ASC_RATIO_SHIFT, out) & ~3u; +} +EXPORT_SYMBOL_IF_KUNIT(asc_scale_ratio); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.h b/drivers/media/platform/apple/scaler/apple-scaler-hw.h new file mode 100644 index 0000000000000..3093aebc3d975 --- /dev/null +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple silicon scaler/CSC (MSR): pipeline field encodings. + * + * These helpers compute register values from V4L2 state. They touch no + * hardware, so KUnit checks them against the values the engine is known + * to accept. + */ +#ifndef APPLE_SCALER_HW_H +#define APPLE_SCALER_HW_H + +#include +#include + +/* Colour-space converter block. */ +#define ASC_CSC_CTRL 0x3800 +#define ASC_CSC_CTRL_BYPASS 0x000e0000 +#define ASC_CSC_CTRL_ENABLE 0x000e0003 +#define ASC_CSC_MODE 0x3804 +#define ASC_CSC_MODE_DEFAULT 0xc +#define ASC_CSC_PRE_OFFSET(i) (0x3808 + 4 * (i)) +#define ASC_CSC_COEF(row, col) (0x3814 + 4 * (3 * (row) + (col))) +#define ASC_CSC_POST_OFFSET(i) (0x3838 + 4 * (i)) + +/* Matrix coefficients are signed Q3.12. */ +#define ASC_CSC_COEF_SHIFT 12 +/* Offsets are signed fractions of full scale: 1.0 = 1 << 24, 26-bit field. */ +#define ASC_CSC_OFFSET_SHIFT 24 +#define ASC_CSC_OFFSET_MASK GENMASK(25, 0) + +/* Scale ratios are unsigned Q22. */ +#define ASC_RATIO_SHIFT 22 + +/** + * struct asc_csc - one colour conversion, in register encoding + * @coef: 3x3 matrix, row = output channel, column = input channel + * @pre: offsets added to the input channels before the matrix + * @post: offsets added to the output channels after the matrix + */ +struct asc_csc { + s16 coef[3][3]; + s32 pre[3]; + s32 post[3]; +}; + +/** + * struct asc_format - a pixel format the engine reads and writes + * @fourcc: V4L2 pixel format + * @layout: layout class (source 0x180 / destination 0x280) + * @code: format code (source 0x184; destination 0x284 without bit 15) + * @sampling: subsampling descriptor (source 0x188 / destination 0x288 without bit 15) + * @swizzle: component order (source 0x1f8 / destination 0x2d8) + * @planes: colour planes (each needs its own DMA address) + * @bpp: bytes per sample position of each plane, at that plane's resolution + * @hsub: horizontal chroma subsampling of plane 1 + * @vsub: vertical chroma subsampling of plane 1 + * @yuv: Y'CbCr format (needs the CSC to pair with RGB) + * @alpha: carries an alpha channel + */ +struct asc_format { + u32 fourcc; + u32 layout; + u32 code; + u32 sampling; + u32 swizzle; + u8 planes; + u8 bpp[2]; + u8 hsub; + u8 vsub; + bool yuv; + bool alpha; +}; + +const struct asc_format *asc_find_format(u32 fourcc); +const struct asc_format *asc_format_by_index(unsigned int index); + +int asc_csc_compute(enum v4l2_ycbcr_encoding enc, + enum v4l2_quantization quant, bool yuv_to_rgb, + struct asc_csc *csc); +u32 asc_csc_offset_reg(s32 offset); + +u32 asc_scale_ratio(u32 in, u32 out); + +#endif /* APPLE_SCALER_HW_H */ diff --git a/drivers/media/platform/apple/scaler/apple-scaler-test.c b/drivers/media/platform/apple/scaler/apple-scaler-test.c new file mode 100644 index 0000000000000..58f3b2f6f3469 --- /dev/null +++ b/drivers/media/platform/apple/scaler/apple-scaler-test.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the Apple scaler/CSC register encodings. + * + * The expected words are what the engine was driven with for the same + * conversions in our macOS capture (traces/display/20261108-msr-trace), + * and they equal the textbook BT.601/BT.709/BT.2020 matrices in Q3.12. + */ +#include +#include +#include +#include + +#include "apple-scaler-hw.h" + +struct asc_csc_case { + const char *name; + enum v4l2_ycbcr_encoding enc; + enum v4l2_quantization quant; + bool yuv_to_rgb; + u16 coef[9]; + u32 pre[3]; + u32 post[3]; +}; + +static const struct asc_csc_case asc_csc_cases[] = { + { "BT.709 limited to RGB", V4L2_YCBCR_ENC_709, V4L2_QUANTIZATION_LIM_RANGE, true, + { 0x12a1, 0x0000, 0x1caf, 0x12a1, 0xfc97, 0xf779, 0x12a1, 0x21cc, 0x0000 }, + { 0x03eff000, 0x037fe000, 0x037fe000 }, { 0, 0, 0 } }, + { "BT.601 limited to RGB", V4L2_YCBCR_ENC_601, V4L2_QUANTIZATION_LIM_RANGE, true, + { 0x12a1, 0x0000, 0x1989, 0x12a1, 0xf9bb, 0xf2fe, 0x12a1, 0x2047, 0x0000 }, + { 0x03eff000, 0x037fe000, 0x037fe000 }, { 0, 0, 0 } }, + { "BT.709 full to RGB", V4L2_YCBCR_ENC_709, V4L2_QUANTIZATION_FULL_RANGE, true, + { 0x1000, 0x0000, 0x1932, 0x1000, 0xfd01, 0xf883, 0x1000, 0x1db1, 0x0000 }, + { 0x00000000, 0x037fe000, 0x037fe000 }, { 0, 0, 0 } }, + { "BT.2020 limited to RGB", V4L2_YCBCR_ENC_BT2020, V4L2_QUANTIZATION_LIM_RANGE, true, + { 0x12a1, 0x0000, 0x1adc, 0x12a1, 0xfd01, 0xf598, 0x12a1, 0x2245, 0x0000 }, + { 0x03eff000, 0x037fe000, 0x037fe000 }, { 0, 0, 0 } }, + { "RGB to BT.709 limited", V4L2_YCBCR_ENC_709, V4L2_QUANTIZATION_LIM_RANGE, false, + { 0x02ec, 0x09d4, 0x00fe, 0xfe64, 0xfa95, 0x0707, 0x0707, 0xf99e, 0xff5b }, + { 0, 0, 0 }, { 0x00101000, 0x00802000, 0x00802000 } }, + { "RGB to BT.601 limited", V4L2_YCBCR_ENC_601, V4L2_QUANTIZATION_LIM_RANGE, false, + { 0x041c, 0x0811, 0x0191, 0xfda1, 0xfb58, 0x0707, 0x0707, 0xfa1e, 0xfedb }, + { 0, 0, 0 }, { 0x00101000, 0x00802000, 0x00802000 } }, +}; + +static void asc_csc_case_desc(const struct asc_csc_case *c, char *desc) +{ + strscpy(desc, c->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(asc_csc, asc_csc_cases, asc_csc_case_desc); + +static void asc_test_csc(struct kunit *test) +{ + const struct asc_csc_case *c = test->param_value; + struct asc_csc csc; + int i; + + KUNIT_ASSERT_EQ(test, asc_csc_compute(c->enc, c->quant, c->yuv_to_rgb, &csc), 0); + for (i = 0; i < 9; i++) + KUNIT_EXPECT_EQ_MSG(test, (u16)csc.coef[i / 3][i % 3], c->coef[i], + "coefficient %d", i); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ_MSG(test, asc_csc_offset_reg(csc.pre[i]), c->pre[i], + "pre-offset %d", i); + KUNIT_EXPECT_EQ_MSG(test, asc_csc_offset_reg(csc.post[i]), c->post[i], + "post-offset %d", i); + } +} + +static void asc_test_csc_rejects(struct kunit *test) +{ + struct asc_csc csc; + + KUNIT_EXPECT_EQ(test, asc_csc_compute(V4L2_YCBCR_ENC_SMPTE240M, + V4L2_QUANTIZATION_LIM_RANGE, true, &csc), + -EINVAL); + KUNIT_EXPECT_EQ(test, asc_csc_compute(V4L2_YCBCR_ENC_BT2020_CONST_LUM, + V4L2_QUANTIZATION_LIM_RANGE, true, &csc), + -EINVAL); +} + +/* Every matrix row must reproduce white and black exactly (to rounding). */ +static void asc_test_csc_white(struct kunit *test) +{ + static const enum v4l2_ycbcr_encoding encs[] = { + V4L2_YCBCR_ENC_601, V4L2_YCBCR_ENC_709, V4L2_YCBCR_ENC_BT2020, + }; + struct asc_csc csc; + int e, row; + + for (e = 0; e < ARRAY_SIZE(encs); e++) { + KUNIT_ASSERT_EQ(test, asc_csc_compute(encs[e], V4L2_QUANTIZATION_FULL_RANGE, + false, &csc), 0); + /* RGB -> YCbCr: Y row sums to 1.0, chroma rows to 0. */ + KUNIT_EXPECT_LE(test, abs(csc.coef[0][0] + csc.coef[0][1] + csc.coef[0][2] - 4096), 1); + for (row = 1; row < 3; row++) + KUNIT_EXPECT_LE(test, abs(csc.coef[row][0] + csc.coef[row][1] + + csc.coef[row][2]), 1); + } +} + +static void asc_test_ratio(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, asc_scale_ratio(1920, 1920), 0x400000); + KUNIT_EXPECT_EQ(test, asc_scale_ratio(1920, 1280), 0x600000); + KUNIT_EXPECT_EQ(test, asc_scale_ratio(1280, 1920), 0x2aaaa8); + KUNIT_EXPECT_EQ(test, asc_scale_ratio(640, 1920), 0x155554); + KUNIT_EXPECT_EQ(test, asc_scale_ratio(1920, 640), 0xc00000); + KUNIT_EXPECT_EQ(test, asc_scale_ratio(1, 0), 0); +} + +static void asc_test_formats(struct kunit *test) +{ + const struct asc_format *f; + unsigned int i; + + f = asc_find_format(V4L2_PIX_FMT_NV12M); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_EXPECT_EQ(test, f->layout, 0xc); + KUNIT_EXPECT_EQ(test, f->code, 0x1); + KUNIT_EXPECT_EQ(test, f->sampling, 0x6); + KUNIT_EXPECT_EQ(test, f->swizzle, 0x03000102); + + f = asc_find_format(V4L2_PIX_FMT_ARGB32); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_EXPECT_EQ(test, f->code, 0x1e); + KUNIT_EXPECT_EQ(test, f->swizzle, 0x03020100); + + KUNIT_EXPECT_NULL(test, asc_find_format(V4L2_PIX_FMT_YUYV)); + + for (i = 0; (f = asc_format_by_index(i)); i++) { + KUNIT_EXPECT_GE(test, f->planes, 1); + KUNIT_EXPECT_LE(test, f->planes, 2); + KUNIT_EXPECT_EQ(test, f->yuv, f->layout != 0x1); + KUNIT_EXPECT_EQ(test, f->planes == 2, f->yuv); + } +} + +static struct kunit_case asc_test_cases[] = { + KUNIT_CASE_PARAM(asc_test_csc, asc_csc_gen_params), + KUNIT_CASE(asc_test_csc_rejects), + KUNIT_CASE(asc_test_csc_white), + KUNIT_CASE(asc_test_ratio), + KUNIT_CASE(asc_test_formats), + {} +}; + +static struct kunit_suite asc_test_suite = { + .name = "apple-scaler", + .test_cases = asc_test_cases, +}; +kunit_test_suite(asc_test_suite); + +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); +MODULE_DESCRIPTION("KUnit tests for the Apple scaler/CSC encodings"); +MODULE_LICENSE("GPL"); From 937b53d686221230b089bedeebff30be01b11811 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:54:46 +1100 Subject: [PATCH 169/392] media: apple: scaler: build the register list for same-size jobs Add the job builder the submission path will use: it turns a source image, a destination image and an optional colour matrix into the ordered register writes for one job, ending with the pre-start tag and START. It covers conversions without scaling, which are the common case (decoder output to RGB, RGB to encoder input): - read and write DMA: layout, format code, subsampling, plane addresses, strides, component order, picture and per-plane sizes; - alpha fill when the source has no alpha and the destination does; - chroma upsampling for 4:2:x sources and downsampling for 4:2:x destinations, ordered dither when going from 10 to 8 bits; - the scaler at 1:1 (filters off, unit Q22 ratios); - the CSC matrix, offsets and clamps; - the per-plane memory request words and control words the engine is driven with on every job. KUnit builds three 1080p jobs (NV12 BT.709 to ARGB32, ARGB32 to NV12, P010 BT.2020 to ARGB32) and checks that every register ends with the value the engine was given for the same job on macOS, and that nothing else is written. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .../platform/apple/scaler/apple-scaler-hw.c | 233 ++++++++++++- .../platform/apple/scaler/apple-scaler-hw.h | 43 +++ .../platform/apple/scaler/apple-scaler-test.c | 324 ++++++++++++++++++ 3 files changed, 599 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.c b/drivers/media/platform/apple/scaler/apple-scaler-hw.c index 2cbff1aee9e3d..5eb645a776470 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-hw.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.c @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -45,7 +46,7 @@ static const struct asc_format asc_formats[] = { /* 10 bits in the top of each 16-bit sample. */ .fourcc = V4L2_PIX_FMT_P010, .layout = 0xc, .code = 0x2, .sampling = 0x12, .swizzle = ASC_SWIZZLE_CBCR, .planes = 2, - .bpp = { 2, 4 }, .hsub = 2, .vsub = 2, .yuv = true, + .bpp = { 2, 4 }, .hsub = 2, .vsub = 2, .yuv = true, .wide = true, }, }; @@ -190,3 +191,233 @@ u32 asc_scale_ratio(u32 in, u32 out) return (u32)div_u64((u64)in << ASC_RATIO_SHIFT, out) & ~3u; } EXPORT_SYMBOL_IF_KUNIT(asc_scale_ratio); + +/* + * Job programming. The engine takes the whole pipeline state for every + * job; nothing carries over from the previous one. The order below follows + * the pipeline (source, destination, chroma, scaler, CSC, memory system) + * and ends with the pre-start tag and START. + */ +struct asc_emit { + struct asc_reg *w; + unsigned int n, max; +}; + +static void asc_emit(struct asc_emit *e, u32 off, u32 val) +{ + if (e->n < e->max) + e->w[e->n] = (struct asc_reg){ off, val }; + e->n++; +} + +/* Source (read DMA) and destination (write DMA) register banks. */ +struct asc_dma_regs { + u32 layout, code, sampling, plane, stride, offset, swizzle, size; + u32 plane_width, plane_height; + u32 flag; /* or-ed into layout-adjacent words on the write side */ +}; + +static const struct asc_dma_regs asc_rdma = { + .layout = 0x180, .code = 0x184, .sampling = 0x188, .plane = 0x198, + .stride = 0x1d8, .offset = 0x1e8, .swizzle = 0x1f8, .size = 0x1fc, + .plane_width = 0x234, .plane_height = 0x244, .flag = 0, +}; + +static const struct asc_dma_regs asc_wdma = { + .layout = 0x280, .code = 0x284, .sampling = 0x288, .plane = 0x290, + .stride = 0x2c0, .offset = 0x2cc, .swizzle = 0x2d8, .size = 0x2dc, + .plane_width = 0x308, .plane_height = 0x314, .flag = BIT(15), +}; + +static void asc_emit_image(struct asc_emit *e, const struct asc_dma_regs *r, + const struct asc_image *img) +{ + const struct asc_format *f = img->fmt; + /* Plane 2 is never used; the engine still takes it at luma size. */ + u32 pw[3] = { img->width, img->width / f->hsub, img->width }; + u32 ph[3] = { img->height, img->height / f->vsub, img->height }; + unsigned int i, nplanes = f->planes == 1 ? 3 : f->planes; + + asc_emit(e, r->layout, f->layout); + asc_emit(e, r->code, r->flag | f->code); + asc_emit(e, r->sampling, r->flag | f->sampling); + if (r->flag) + asc_emit(e, r->sampling + 4, r->flag); + else { + asc_emit(e, r->sampling + 4, 0); + asc_emit(e, r->sampling + 8, 0); + } + for (i = 0; i < nplanes; i++) { + u64 dva = i < f->planes ? img->dva[i] : 0; + + asc_emit(e, r->plane + 8 * i, lower_32_bits(dva)); + asc_emit(e, r->plane + 8 * i + 4, upper_32_bits(dva)); + asc_emit(e, r->stride + 4 * i, i < f->planes ? img->stride[i] : 0); + } + for (i = 0; i < 3; i++) + asc_emit(e, r->offset + 4 * i, 0); + asc_emit(e, r->swizzle, f->swizzle); + asc_emit(e, r->size, img->width); + asc_emit(e, r->size + 4, img->height); + for (i = 0; i < 3; i++) { + asc_emit(e, r->plane_width + 4 * i, f->planes == 1 ? img->width : pw[i]); + asc_emit(e, r->plane_height + 4 * i, f->planes == 1 ? img->height : ph[i]); + } +} + +/* Per-plane memory request words, as the engine is driven on macOS. */ +static void asc_emit_memory(struct asc_emit *e, const struct asc_job *job) +{ + const struct asc_format *s = job->src.fmt, *d = job->dst.fmt; + u32 wide = s->wide ? BIT(1) : 0; + unsigned int i; + + for (i = 0; i < 4; i++) { + asc_emit(e, 0x204 + 4 * i, 0x0007d311); + asc_emit(e, 0x214 + 4 * i, 0x00000707); + asc_emit(e, 0x224 + 4 * i, 0x00000020); + } + for (i = 0; i < 3; i++) { + asc_emit(e, 0x2e4 + 4 * i, 0x000ac211); + asc_emit(e, 0x2f0 + 4 * i, 0x0c0ac207); + asc_emit(e, 0x2fc + 4 * i, 0x04000620); + } + asc_emit(e, 0x254, 1); + + for (i = 0; i < 4; i++) + asc_emit(e, 0x13800 + 4 * i, 0); + asc_emit(e, 0x13810, 0x8 | wide); + asc_emit(e, 0x13814, (s->yuv ? 0x28 : 0x8) | wide); + asc_emit(e, 0x13818, (s->yuv ? 0x28 : 0x8) | wide); + asc_emit(e, 0x1381c, s->yuv ? 0x20 : 0x8); + asc_emit(e, 0x13c04, 0); + asc_emit(e, 0x13c10, 0x8); + asc_emit(e, 0x13c14, d->yuv ? 0x28 : 0x8); + asc_emit(e, 0x13c18, d->yuv ? 0x28 : 0x8); + asc_emit(e, 0x13c1c, d->yuv ? 0x20 : 0x8); + asc_emit(e, 0x13c00, 1); +} + +static void asc_emit_chroma(struct asc_emit *e, const struct asc_job *job) +{ + /* Upsampler interpolation weights, 0x800 = one half. */ + static const u32 up[8] = { 0, 0x800, 0x800, 0, 0, 0x800, 0x800, 0 }; + /* 2x2 ordered-dither index pattern, repeated over 4x4. */ + static const u32 dither[16] = { 0, 2, 0, 2, 3, 1, 3, 1, 0, 2, 0, 2, 3, 1, 3, 1 }; + const struct asc_format *s = job->src.fmt, *d = job->dst.fmt; + unsigned int i; + + asc_emit(e, 0x800, s->yuv && s->hsub > 1 ? 0xd : 0xc); + if (s->yuv && s->hsub > 1) + for (i = 0; i < ARRAY_SIZE(up); i++) + asc_emit(e, 0x804 + 4 * i, up[i]); + asc_emit(e, 0x824, 0xc); + asc_emit(e, 0x900, d->yuv && d->hsub > 1 ? 0x00010103 : 0); + if (s->wide && !d->wide) { + asc_emit(e, 0xa00, 0x02020200); + for (i = 0; i < ARRAY_SIZE(dither); i++) + asc_emit(e, 0xa80 + 4 * i, dither[i]); + } else { + asc_emit(e, 0xa00, 0); + } +} + +/* Scaler at 1:1: polyphase filters off, unit ratios, zero initial phase. */ +static void asc_emit_unscaled(struct asc_emit *e, const struct asc_job *job) +{ + static const u32 base[2] = { 0x1000, 0x2000 }; + u32 flags = job->src.fmt->yuv && job->src.fmt->hsub > 1 ? BIT(1) : 0; + u32 one = 1u << ASC_RATIO_SHIFT; + unsigned int i; + + for (i = 0; i < 2; i++) { + asc_emit(e, base[i], flags); + asc_emit(e, base[i] + 0x04, 0); + asc_emit(e, base[i] + 0x08, 0); + asc_emit(e, base[i] + 0x0c, one); + asc_emit(e, base[i] + 0x10, one); + asc_emit(e, base[i] + 0x14, one); + asc_emit(e, base[i] + 0x18, one); + asc_emit(e, base[i] + 0x1c, 0); + asc_emit(e, base[i] + 0x20, one); + asc_emit(e, base[i] + 0x24, one); + } + asc_emit(e, 0x480, 0); +} + +static void asc_emit_csc(struct asc_emit *e, const struct asc_csc *csc) +{ + unsigned int r, c; + + asc_emit(e, ASC_CSC_MODE, ASC_CSC_MODE_DEFAULT); + for (c = 0; c < 3; c++) + asc_emit(e, ASC_CSC_PRE_OFFSET(c), asc_csc_offset_reg(csc->pre[c])); + for (r = 0; r < 3; r++) + for (c = 0; c < 3; c++) + asc_emit(e, ASC_CSC_COEF(r, c), (u16)csc->coef[r][c]); + for (c = 0; c < 3; c++) + asc_emit(e, ASC_CSC_POST_OFFSET(c), asc_csc_offset_reg(csc->post[c])); + /* Output clamps: full signed range on every channel. */ + for (c = 0; c < 3; c++) { + asc_emit(e, 0xa98c + 4 * c, 0x80000001); + asc_emit(e, 0xa998 + 4 * c, 0x7fffffff); + asc_emit(e, 0xa9a4 + 4 * c, 0x80000001); + asc_emit(e, 0xa9b0 + 4 * c, 0x7fffffff); + } + asc_emit(e, ASC_CSC_CTRL, ASC_CSC_CTRL_ENABLE); +} + +/* Control words the engine is driven with for every job on macOS. */ +static const struct asc_reg asc_job_fixed[] = { + { 0x000e4, 0x00000000 }, { 0x000e8, 0x00000038 }, { 0x00124, 0x00000000 }, + { 0x00380, 0x00000000 }, { 0x03000, 0x00000000 }, { 0x07418, 0xffffffff }, + { 0x16000, 0x00000000 }, { 0x16004, 0x244024c0 }, { 0x16080, 0x00000000 }, + { 0x16084, 0x0b710367 }, { 0x16088, 0x00000128 }, { 0x16504, 0x02000500 }, + { 0x16508, 0x00003200 }, { 0x1651c, 0x00851400 }, { 0x16534, 0x00000008 }, + { 0x16568, 0x00250500 }, { 0x16588, 0x00496513 }, { 0x16600, 0x00000015 }, + { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, { 0x2e300, 0x03804038 }, + { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, { 0x2e30c, 0x02550255 }, + { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, { 0x2e318, 0x02550000 }, +}; + +/** + * asc_job_build - register writes for one job, in order + * @job: the conversion; source and destination must be the same size + * @w: output array + * @max: capacity of @w + * + * Return: number of writes, -EINVAL for a job this path cannot run, or + * -ENOSPC if @w is too small. + */ +int asc_job_build(const struct asc_job *job, struct asc_reg *w, unsigned int max) +{ + const struct asc_format *s = job->src.fmt, *d = job->dst.fmt; + struct asc_emit e = { .w = w, .max = max }; + unsigned int i; + + if (!s || !d || job->src.width != job->dst.width || + job->src.height != job->dst.height) + return -EINVAL; + /* Y'CbCr <-> R'G'B' needs a matrix; same-family jobs must not have one. */ + if ((s->yuv != d->yuv) != !!job->csc) + return -EINVAL; + + asc_emit(&e, ASC_CSC_CTRL, ASC_CSC_CTRL_BYPASS); + asc_emit_image(&e, &asc_rdma, &job->src); + asc_emit_image(&e, &asc_wdma, &job->dst); + asc_emit_memory(&e, job); + /* Alpha fill: constant opaque alpha when the source has none. */ + asc_emit(&e, 0x8c, (d->alpha && !s->alpha ? BIT(31) : 0) | 0xffff); + asc_emit_chroma(&e, job); + asc_emit_unscaled(&e, job); + if (job->csc) + asc_emit_csc(&e, job->csc); + for (i = 0; i < ARRAY_SIZE(asc_job_fixed); i++) + asc_emit(&e, asc_job_fixed[i].off, asc_job_fixed[i].val); + asc_emit(&e, 0x110, job->transform_id); + asc_emit(&e, 0x098, 0xfffffffe); + asc_emit(&e, 0x080, 1); + + return e.n <= max ? e.n : -ENOSPC; +} +EXPORT_SYMBOL_IF_KUNIT(asc_job_build); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.h b/drivers/media/platform/apple/scaler/apple-scaler-hw.h index 3093aebc3d975..cf371272dce66 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-hw.h +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.h @@ -56,6 +56,7 @@ struct asc_csc { * @vsub: vertical chroma subsampling of plane 1 * @yuv: Y'CbCr format (needs the CSC to pair with RGB) * @alpha: carries an alpha channel + * @wide: samples are 16 bits in memory (more than 8 significant bits) */ struct asc_format { u32 fourcc; @@ -69,8 +70,50 @@ struct asc_format { u8 vsub; bool yuv; bool alpha; + bool wide; }; +/* One register write: byte offset in the MSR aperture and value. */ +struct asc_reg { + u32 off; + u32 val; +}; + +#define ASC_MAX_PLANES 2 +#define ASC_JOB_MAX_WRITES 224 + +/** + * struct asc_image - one side of a job + * @fmt: pixel format + * @width: width in pixels + * @height: height in pixels + * @dva: DMA address of each colour plane + * @stride: bytes per line of each colour plane + */ +struct asc_image { + const struct asc_format *fmt; + u32 width; + u32 height; + u64 dva[ASC_MAX_PLANES]; + u32 stride[ASC_MAX_PLANES]; +}; + +/** + * struct asc_job - a complete conversion + * @src: image read + * @dst: image written + * @csc: colour conversion, or NULL to pass components through + * @transform_id: tag the engine carries for this job + */ +struct asc_job { + struct asc_image src; + struct asc_image dst; + const struct asc_csc *csc; + u32 transform_id; +}; + +int asc_job_build(const struct asc_job *job, struct asc_reg *w, unsigned int max); + const struct asc_format *asc_find_format(u32 fourcc); const struct asc_format *asc_format_by_index(unsigned int index); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-test.c b/drivers/media/platform/apple/scaler/apple-scaler-test.c index 58f3b2f6f3469..4e61ad54e59a9 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-test.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-test.c @@ -12,6 +12,7 @@ #include #include "apple-scaler-hw.h" +#include "apple-scaler-regs.h" struct asc_csc_case { const char *name; @@ -138,12 +139,335 @@ static void asc_test_formats(struct kunit *test) } } + +/* + * Final register state of three jobs as the engine was driven on macOS + * (1920x1080, same size): NV12 BT.709 limited -> ARGB32, ARGB32 -> NV12 + * BT.709 limited, P010 BT.2020 limited -> ARGB32. The CSC matrix words + * (0x3814..0x3834) are covered by asc_test_csc. + */ +static const struct asc_reg asc_expect_nv12_709_argb[] = { + { 0x00080, 0x00000001 }, { 0x0008c, 0x8000ffff }, { 0x00098, 0xfffffffe }, + { 0x000e4, 0x00000000 }, { 0x000e8, 0x00000038 }, { 0x00110, 0x00000094 }, + { 0x00124, 0x00000000 }, { 0x00180, 0x0000000c }, { 0x00184, 0x00000001 }, + { 0x00188, 0x00000006 }, { 0x0018c, 0x00000000 }, { 0x00190, 0x00000000 }, + { 0x00198, 0x007f0200 }, { 0x0019c, 0x00000100 }, { 0x001a0, 0x009ea600 }, + { 0x001a4, 0x00000100 }, { 0x001d8, 0x00000780 }, { 0x001dc, 0x00000780 }, + { 0x001e8, 0x00000000 }, { 0x001ec, 0x00000000 }, { 0x001f0, 0x00000000 }, + { 0x001f8, 0x03000102 }, { 0x001fc, 0x00000780 }, { 0x00200, 0x00000438 }, + { 0x00204, 0x0007d311 }, { 0x00208, 0x0007d311 }, { 0x0020c, 0x0007d311 }, + { 0x00210, 0x0007d311 }, { 0x00214, 0x00000707 }, { 0x00218, 0x00000707 }, + { 0x0021c, 0x00000707 }, { 0x00220, 0x00000707 }, { 0x00224, 0x00000020 }, + { 0x00228, 0x00000020 }, { 0x0022c, 0x00000020 }, { 0x00230, 0x00000020 }, + { 0x00234, 0x00000780 }, { 0x00238, 0x000003c0 }, { 0x0023c, 0x00000780 }, + { 0x00244, 0x00000438 }, { 0x00248, 0x0000021c }, { 0x0024c, 0x00000438 }, + { 0x00254, 0x00000001 }, { 0x00280, 0x00000001 }, { 0x00284, 0x0000801e }, + { 0x00288, 0x00008000 }, { 0x0028c, 0x00008000 }, { 0x00290, 0x00000000 }, + { 0x00294, 0x00000100 }, { 0x00298, 0x00000000 }, { 0x0029c, 0x00000000 }, + { 0x002a0, 0x00000000 }, { 0x002a4, 0x00000000 }, { 0x002c0, 0x00001e00 }, + { 0x002c4, 0x00000000 }, { 0x002c8, 0x00000000 }, { 0x002cc, 0x00000000 }, + { 0x002d0, 0x00000000 }, { 0x002d4, 0x00000000 }, { 0x002d8, 0x03020100 }, + { 0x002dc, 0x00000780 }, { 0x002e0, 0x00000438 }, { 0x002e4, 0x000ac211 }, + { 0x002e8, 0x000ac211 }, { 0x002ec, 0x000ac211 }, { 0x002f0, 0x0c0ac207 }, + { 0x002f4, 0x0c0ac207 }, { 0x002f8, 0x0c0ac207 }, { 0x002fc, 0x04000620 }, + { 0x00300, 0x04000620 }, { 0x00304, 0x04000620 }, { 0x00308, 0x00000780 }, + { 0x0030c, 0x00000780 }, { 0x00310, 0x00000780 }, { 0x00314, 0x00000438 }, + { 0x00318, 0x00000438 }, { 0x0031c, 0x00000438 }, { 0x00380, 0x00000000 }, + { 0x00480, 0x00000000 }, { 0x00800, 0x0000000d }, { 0x00804, 0x00000000 }, + { 0x00808, 0x00000800 }, { 0x0080c, 0x00000800 }, { 0x00810, 0x00000000 }, + { 0x00814, 0x00000000 }, { 0x00818, 0x00000800 }, { 0x0081c, 0x00000800 }, + { 0x00820, 0x00000000 }, { 0x00824, 0x0000000c }, { 0x00900, 0x00000000 }, + { 0x00a00, 0x00000000 }, { 0x01000, 0x00000002 }, { 0x01004, 0x00000000 }, + { 0x01008, 0x00000000 }, { 0x0100c, 0x00400000 }, { 0x01010, 0x00400000 }, + { 0x01014, 0x00400000 }, { 0x01018, 0x00400000 }, { 0x0101c, 0x00000000 }, + { 0x01020, 0x00400000 }, { 0x01024, 0x00400000 }, { 0x02000, 0x00000002 }, + { 0x02004, 0x00000000 }, { 0x02008, 0x00000000 }, { 0x0200c, 0x00400000 }, + { 0x02010, 0x00400000 }, { 0x02014, 0x00400000 }, { 0x02018, 0x00400000 }, + { 0x0201c, 0x00000000 }, { 0x02020, 0x00400000 }, { 0x02024, 0x00400000 }, + { 0x03000, 0x00000000 }, { 0x03800, 0x000e0003 }, { 0x03804, 0x0000000c }, + { 0x03808, 0x03eff000 }, { 0x0380c, 0x037fe000 }, { 0x03810, 0x037fe000 }, + { 0x03838, 0x00000000 }, { 0x0383c, 0x00000000 }, { 0x03840, 0x00000000 }, + { 0x07418, 0xffffffff }, { 0x0a98c, 0x80000001 }, { 0x0a990, 0x80000001 }, + { 0x0a994, 0x80000001 }, { 0x0a998, 0x7fffffff }, { 0x0a99c, 0x7fffffff }, + { 0x0a9a0, 0x7fffffff }, { 0x0a9a4, 0x80000001 }, { 0x0a9a8, 0x80000001 }, + { 0x0a9ac, 0x80000001 }, { 0x0a9b0, 0x7fffffff }, { 0x0a9b4, 0x7fffffff }, + { 0x0a9b8, 0x7fffffff }, { 0x13800, 0x00000000 }, { 0x13804, 0x00000000 }, + { 0x13808, 0x00000000 }, { 0x1380c, 0x00000000 }, { 0x13810, 0x00000008 }, + { 0x13814, 0x00000028 }, { 0x13818, 0x00000028 }, { 0x1381c, 0x00000020 }, + { 0x13c00, 0x00000001 }, { 0x13c04, 0x00000000 }, { 0x13c10, 0x00000008 }, + { 0x13c14, 0x00000008 }, { 0x13c18, 0x00000008 }, { 0x13c1c, 0x00000008 }, + { 0x16000, 0x00000000 }, { 0x16004, 0x244024c0 }, { 0x16080, 0x00000000 }, + { 0x16084, 0x0b710367 }, { 0x16088, 0x00000128 }, { 0x16504, 0x02000500 }, + { 0x16508, 0x00003200 }, { 0x1651c, 0x00851400 }, { 0x16534, 0x00000008 }, + { 0x16568, 0x00250500 }, { 0x16588, 0x00496513 }, { 0x16600, 0x00000015 }, + { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, { 0x2e300, 0x03804038 }, + { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, { 0x2e30c, 0x02550255 }, + { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, { 0x2e318, 0x02550000 }, +}; + +static const struct asc_reg asc_expect_argb_nv12_709[] = { + { 0x00080, 0x00000001 }, { 0x0008c, 0x0000ffff }, { 0x00098, 0xfffffffe }, + { 0x000e4, 0x00000000 }, { 0x000e8, 0x00000038 }, { 0x00110, 0x000000b4 }, + { 0x00124, 0x00000000 }, { 0x00180, 0x00000001 }, { 0x00184, 0x0000001e }, + { 0x00188, 0x00000000 }, { 0x0018c, 0x00000000 }, { 0x00190, 0x00000000 }, + { 0x00198, 0x00300000 }, { 0x0019c, 0x00000100 }, { 0x001a0, 0x00000000 }, + { 0x001a4, 0x00000000 }, { 0x001a8, 0x00000000 }, { 0x001ac, 0x00000000 }, + { 0x001d8, 0x00001e00 }, { 0x001dc, 0x00000000 }, { 0x001e0, 0x00000000 }, + { 0x001e8, 0x00000000 }, { 0x001ec, 0x00000000 }, { 0x001f0, 0x00000000 }, + { 0x001f8, 0x03020100 }, { 0x001fc, 0x00000780 }, { 0x00200, 0x00000438 }, + { 0x00204, 0x0007d311 }, { 0x00208, 0x0007d311 }, { 0x0020c, 0x0007d311 }, + { 0x00210, 0x0007d311 }, { 0x00214, 0x00000707 }, { 0x00218, 0x00000707 }, + { 0x0021c, 0x00000707 }, { 0x00220, 0x00000707 }, { 0x00224, 0x00000020 }, + { 0x00228, 0x00000020 }, { 0x0022c, 0x00000020 }, { 0x00230, 0x00000020 }, + { 0x00234, 0x00000780 }, { 0x00238, 0x00000780 }, { 0x0023c, 0x00000780 }, + { 0x00244, 0x00000438 }, { 0x00248, 0x00000438 }, { 0x0024c, 0x00000438 }, + { 0x00254, 0x00000001 }, { 0x00280, 0x0000000c }, { 0x00284, 0x00008001 }, + { 0x00288, 0x00008006 }, { 0x0028c, 0x00008000 }, { 0x00290, 0x00000200 }, + { 0x00294, 0x00000100 }, { 0x00298, 0x001fa600 }, { 0x0029c, 0x00000100 }, + { 0x002c0, 0x00000780 }, { 0x002c4, 0x00000780 }, { 0x002cc, 0x00000000 }, + { 0x002d0, 0x00000000 }, { 0x002d4, 0x00000000 }, { 0x002d8, 0x03000102 }, + { 0x002dc, 0x00000780 }, { 0x002e0, 0x00000438 }, { 0x002e4, 0x000ac211 }, + { 0x002e8, 0x000ac211 }, { 0x002ec, 0x000ac211 }, { 0x002f0, 0x0c0ac207 }, + { 0x002f4, 0x0c0ac207 }, { 0x002f8, 0x0c0ac207 }, { 0x002fc, 0x04000620 }, + { 0x00300, 0x04000620 }, { 0x00304, 0x04000620 }, { 0x00308, 0x00000780 }, + { 0x0030c, 0x000003c0 }, { 0x00310, 0x00000780 }, { 0x00314, 0x00000438 }, + { 0x00318, 0x0000021c }, { 0x0031c, 0x00000438 }, { 0x00380, 0x00000000 }, + { 0x00480, 0x00000000 }, { 0x00800, 0x0000000c }, { 0x00824, 0x0000000c }, + { 0x00900, 0x00010103 }, { 0x00a00, 0x00000000 }, { 0x01000, 0x00000000 }, + { 0x01004, 0x00000000 }, { 0x01008, 0x00000000 }, { 0x0100c, 0x00400000 }, + { 0x01010, 0x00400000 }, { 0x01014, 0x00400000 }, { 0x01018, 0x00400000 }, + { 0x0101c, 0x00000000 }, { 0x01020, 0x00400000 }, { 0x01024, 0x00400000 }, + { 0x02000, 0x00000000 }, { 0x02004, 0x00000000 }, { 0x02008, 0x00000000 }, + { 0x0200c, 0x00400000 }, { 0x02010, 0x00400000 }, { 0x02014, 0x00400000 }, + { 0x02018, 0x00400000 }, { 0x0201c, 0x00000000 }, { 0x02020, 0x00400000 }, + { 0x02024, 0x00400000 }, { 0x03000, 0x00000000 }, { 0x03800, 0x000e0003 }, + { 0x03804, 0x0000000c }, { 0x03808, 0x00000000 }, { 0x0380c, 0x00000000 }, + { 0x03810, 0x00000000 }, { 0x03838, 0x00101000 }, { 0x0383c, 0x00802000 }, + { 0x03840, 0x00802000 }, { 0x07418, 0xffffffff }, { 0x0a98c, 0x80000001 }, + { 0x0a990, 0x80000001 }, { 0x0a994, 0x80000001 }, { 0x0a998, 0x7fffffff }, + { 0x0a99c, 0x7fffffff }, { 0x0a9a0, 0x7fffffff }, { 0x0a9a4, 0x80000001 }, + { 0x0a9a8, 0x80000001 }, { 0x0a9ac, 0x80000001 }, { 0x0a9b0, 0x7fffffff }, + { 0x0a9b4, 0x7fffffff }, { 0x0a9b8, 0x7fffffff }, { 0x13800, 0x00000000 }, + { 0x13804, 0x00000000 }, { 0x13808, 0x00000000 }, { 0x1380c, 0x00000000 }, + { 0x13810, 0x00000008 }, { 0x13814, 0x00000008 }, { 0x13818, 0x00000008 }, + { 0x1381c, 0x00000008 }, { 0x13c00, 0x00000001 }, { 0x13c04, 0x00000000 }, + { 0x13c10, 0x00000008 }, { 0x13c14, 0x00000028 }, { 0x13c18, 0x00000028 }, + { 0x13c1c, 0x00000020 }, { 0x16000, 0x00000000 }, { 0x16004, 0x244024c0 }, + { 0x16080, 0x00000000 }, { 0x16084, 0x0b710367 }, { 0x16088, 0x00000128 }, + { 0x16504, 0x02000500 }, { 0x16508, 0x00003200 }, { 0x1651c, 0x00851400 }, + { 0x16534, 0x00000008 }, { 0x16568, 0x00250500 }, { 0x16588, 0x00496513 }, + { 0x16600, 0x00000015 }, { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, + { 0x2e300, 0x03804038 }, { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, + { 0x2e30c, 0x02550255 }, { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, + { 0x2e318, 0x02550000 }, +}; + +static const struct asc_reg asc_expect_p010_2020_argb[] = { + { 0x00080, 0x00000001 }, { 0x0008c, 0x8000ffff }, { 0x00098, 0xfffffffe }, + { 0x000e4, 0x00000000 }, { 0x000e8, 0x00000038 }, { 0x00110, 0x000000bc }, + { 0x00124, 0x00000000 }, { 0x00180, 0x0000000c }, { 0x00184, 0x00000002 }, + { 0x00188, 0x00000012 }, { 0x0018c, 0x00000000 }, { 0x00190, 0x00000000 }, + { 0x00198, 0x007f0200 }, { 0x0019c, 0x00000100 }, { 0x001a0, 0x00be4a00 }, + { 0x001a4, 0x00000100 }, { 0x001d8, 0x00000f00 }, { 0x001dc, 0x00000f00 }, + { 0x001e8, 0x00000000 }, { 0x001ec, 0x00000000 }, { 0x001f0, 0x00000000 }, + { 0x001f8, 0x03000102 }, { 0x001fc, 0x00000780 }, { 0x00200, 0x00000438 }, + { 0x00204, 0x0007d311 }, { 0x00208, 0x0007d311 }, { 0x0020c, 0x0007d311 }, + { 0x00210, 0x0007d311 }, { 0x00214, 0x00000707 }, { 0x00218, 0x00000707 }, + { 0x0021c, 0x00000707 }, { 0x00220, 0x00000707 }, { 0x00224, 0x00000020 }, + { 0x00228, 0x00000020 }, { 0x0022c, 0x00000020 }, { 0x00230, 0x00000020 }, + { 0x00234, 0x00000780 }, { 0x00238, 0x000003c0 }, { 0x0023c, 0x00000780 }, + { 0x00244, 0x00000438 }, { 0x00248, 0x0000021c }, { 0x0024c, 0x00000438 }, + { 0x00254, 0x00000001 }, { 0x00280, 0x00000001 }, { 0x00284, 0x0000801e }, + { 0x00288, 0x00008000 }, { 0x0028c, 0x00008000 }, { 0x00290, 0x00000000 }, + { 0x00294, 0x00000100 }, { 0x00298, 0x00000000 }, { 0x0029c, 0x00000000 }, + { 0x002a0, 0x00000000 }, { 0x002a4, 0x00000000 }, { 0x002c0, 0x00001e00 }, + { 0x002c4, 0x00000000 }, { 0x002c8, 0x00000000 }, { 0x002cc, 0x00000000 }, + { 0x002d0, 0x00000000 }, { 0x002d4, 0x00000000 }, { 0x002d8, 0x03020100 }, + { 0x002dc, 0x00000780 }, { 0x002e0, 0x00000438 }, { 0x002e4, 0x000ac211 }, + { 0x002e8, 0x000ac211 }, { 0x002ec, 0x000ac211 }, { 0x002f0, 0x0c0ac207 }, + { 0x002f4, 0x0c0ac207 }, { 0x002f8, 0x0c0ac207 }, { 0x002fc, 0x04000620 }, + { 0x00300, 0x04000620 }, { 0x00304, 0x04000620 }, { 0x00308, 0x00000780 }, + { 0x0030c, 0x00000780 }, { 0x00310, 0x00000780 }, { 0x00314, 0x00000438 }, + { 0x00318, 0x00000438 }, { 0x0031c, 0x00000438 }, { 0x00380, 0x00000000 }, + { 0x00480, 0x00000000 }, { 0x00800, 0x0000000d }, { 0x00804, 0x00000000 }, + { 0x00808, 0x00000800 }, { 0x0080c, 0x00000800 }, { 0x00810, 0x00000000 }, + { 0x00814, 0x00000000 }, { 0x00818, 0x00000800 }, { 0x0081c, 0x00000800 }, + { 0x00820, 0x00000000 }, { 0x00824, 0x0000000c }, { 0x00900, 0x00000000 }, + { 0x00a00, 0x02020200 }, { 0x00a80, 0x00000000 }, { 0x00a84, 0x00000002 }, + { 0x00a88, 0x00000000 }, { 0x00a8c, 0x00000002 }, { 0x00a90, 0x00000003 }, + { 0x00a94, 0x00000001 }, { 0x00a98, 0x00000003 }, { 0x00a9c, 0x00000001 }, + { 0x00aa0, 0x00000000 }, { 0x00aa4, 0x00000002 }, { 0x00aa8, 0x00000000 }, + { 0x00aac, 0x00000002 }, { 0x00ab0, 0x00000003 }, { 0x00ab4, 0x00000001 }, + { 0x00ab8, 0x00000003 }, { 0x00abc, 0x00000001 }, { 0x01000, 0x00000002 }, + { 0x01004, 0x00000000 }, { 0x01008, 0x00000000 }, { 0x0100c, 0x00400000 }, + { 0x01010, 0x00400000 }, { 0x01014, 0x00400000 }, { 0x01018, 0x00400000 }, + { 0x0101c, 0x00000000 }, { 0x01020, 0x00400000 }, { 0x01024, 0x00400000 }, + { 0x02000, 0x00000002 }, { 0x02004, 0x00000000 }, { 0x02008, 0x00000000 }, + { 0x0200c, 0x00400000 }, { 0x02010, 0x00400000 }, { 0x02014, 0x00400000 }, + { 0x02018, 0x00400000 }, { 0x0201c, 0x00000000 }, { 0x02020, 0x00400000 }, + { 0x02024, 0x00400000 }, { 0x03000, 0x00000000 }, { 0x03800, 0x000e0003 }, + { 0x03804, 0x0000000c }, { 0x03808, 0x03eff000 }, { 0x0380c, 0x037fe000 }, + { 0x03810, 0x037fe000 }, { 0x03838, 0x00000000 }, { 0x0383c, 0x00000000 }, + { 0x03840, 0x00000000 }, { 0x07418, 0xffffffff }, { 0x0a98c, 0x80000001 }, + { 0x0a990, 0x80000001 }, { 0x0a994, 0x80000001 }, { 0x0a998, 0x7fffffff }, + { 0x0a99c, 0x7fffffff }, { 0x0a9a0, 0x7fffffff }, { 0x0a9a4, 0x80000001 }, + { 0x0a9a8, 0x80000001 }, { 0x0a9ac, 0x80000001 }, { 0x0a9b0, 0x7fffffff }, + { 0x0a9b4, 0x7fffffff }, { 0x0a9b8, 0x7fffffff }, { 0x13800, 0x00000000 }, + { 0x13804, 0x00000000 }, { 0x13808, 0x00000000 }, { 0x1380c, 0x00000000 }, + { 0x13810, 0x0000000a }, { 0x13814, 0x0000002a }, { 0x13818, 0x0000002a }, + { 0x1381c, 0x00000020 }, { 0x13c00, 0x00000001 }, { 0x13c04, 0x00000000 }, + { 0x13c10, 0x00000008 }, { 0x13c14, 0x00000008 }, { 0x13c18, 0x00000008 }, + { 0x13c1c, 0x00000008 }, { 0x16000, 0x00000000 }, { 0x16004, 0x244024c0 }, + { 0x16080, 0x00000000 }, { 0x16084, 0x0b710367 }, { 0x16088, 0x00000128 }, + { 0x16504, 0x02000500 }, { 0x16508, 0x00003200 }, { 0x1651c, 0x00851400 }, + { 0x16534, 0x00000008 }, { 0x16568, 0x00250500 }, { 0x16588, 0x00496513 }, + { 0x16600, 0x00000015 }, { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, + { 0x2e300, 0x03804038 }, { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, + { 0x2e30c, 0x02550255 }, { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, + { 0x2e318, 0x02550000 }, +}; + + +struct asc_job_case { + const char *name; + u32 src, dst; + enum v4l2_ycbcr_encoding enc; + const struct asc_reg *expect; + unsigned int n; +}; + +#define ASC_JOB_CASE(_n, _s, _d, _e, _x) \ + { _n, _s, _d, _e, _x, ARRAY_SIZE(_x) } + +static const struct asc_job_case asc_job_cases[] = { + ASC_JOB_CASE("NV12 BT.709 to ARGB32", V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_ARGB32, + V4L2_YCBCR_ENC_709, asc_expect_nv12_709_argb), + ASC_JOB_CASE("ARGB32 to NV12 BT.709", V4L2_PIX_FMT_ARGB32, V4L2_PIX_FMT_NV12M, + V4L2_YCBCR_ENC_709, asc_expect_argb_nv12_709), + ASC_JOB_CASE("P010 BT.2020 to ARGB32", V4L2_PIX_FMT_P010, V4L2_PIX_FMT_ARGB32, + V4L2_YCBCR_ENC_BT2020, asc_expect_p010_2020_argb), +}; + +static void asc_job_case_desc(const struct asc_job_case *c, char *desc) +{ + strscpy(desc, c->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(asc_job, asc_job_cases, asc_job_case_desc); + +static u32 asc_expect_get(const struct asc_job_case *c, u32 off) +{ + unsigned int i; + + for (i = 0; i < c->n; i++) + if (c->expect[i].off == off) + return c->expect[i].val; + return 0; +} + +static void asc_image_from_expect(const struct asc_job_case *c, struct asc_image *img, + u32 fourcc, u32 plane, u32 stride) +{ + unsigned int i; + + img->fmt = asc_find_format(fourcc); + img->width = 1920; + img->height = 1080; + for (i = 0; i < ASC_MAX_PLANES; i++) { + img->dva[i] = asc_expect_get(c, plane + 8 * i) | + (u64)asc_expect_get(c, plane + 8 * i + 4) << 32; + img->stride[i] = asc_expect_get(c, stride + 4 * i); + } +} + +static bool asc_is_matrix(u32 off) +{ + return off >= ASC_CSC_COEF(0, 0) && off <= ASC_CSC_COEF(2, 2); +} + +static void asc_test_job(struct kunit *test) +{ + const struct asc_job_case *c = test->param_value; + struct asc_reg *w; + struct asc_csc csc; + struct asc_job job = {}; + unsigned int i, j; + int n; + + w = kunit_kcalloc(test, ASC_JOB_MAX_WRITES, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + + asc_image_from_expect(c, &job.src, c->src, 0x198, 0x1d8); + asc_image_from_expect(c, &job.dst, c->dst, 0x290, 0x2c0); + KUNIT_ASSERT_NOT_NULL(test, job.src.fmt); + KUNIT_ASSERT_NOT_NULL(test, job.dst.fmt); + KUNIT_ASSERT_EQ(test, asc_csc_compute(c->enc, V4L2_QUANTIZATION_LIM_RANGE, + job.src.fmt->yuv, &csc), 0); + job.csc = &csc; + job.transform_id = asc_expect_get(c, ASC_TRANSFORM_ID); + + n = asc_job_build(&job, w, ASC_JOB_MAX_WRITES); + KUNIT_ASSERT_GT(test, n, 0); + + /* START last, the pre-start tag right before it. */ + KUNIT_EXPECT_EQ(test, w[n - 1].off, ASC_START); + KUNIT_EXPECT_EQ(test, w[n - 1].val, 1); + KUNIT_EXPECT_EQ(test, w[n - 2].off, ASC_PRE_START); + + /* Every register macOS set ends with the same value... */ + for (i = 0; i < c->n; i++) { + u32 val = 0; + bool found = false; + + for (j = 0; j < n; j++) + if (w[j].off == c->expect[i].off) { + val = w[j].val; + found = true; + } + KUNIT_EXPECT_TRUE_MSG(test, found, "%#x not written", c->expect[i].off); + KUNIT_EXPECT_EQ_MSG(test, val, c->expect[i].val, "register %#x", + c->expect[i].off); + } + /* ...and nothing else is written. */ + for (j = 0; j < n; j++) { + bool known = asc_is_matrix(w[j].off); + + for (i = 0; i < c->n && !known; i++) + known = c->expect[i].off == w[j].off; + KUNIT_EXPECT_TRUE_MSG(test, known, "unexpected write %#x", w[j].off); + } +} + +static void asc_test_job_rejects(struct kunit *test) +{ + struct asc_reg *w = kunit_kcalloc(test, ASC_JOB_MAX_WRITES, sizeof(*w), GFP_KERNEL); + struct asc_csc csc; + struct asc_job job = { + .src = { .fmt = asc_find_format(V4L2_PIX_FMT_NV12M), .width = 64, .height = 64 }, + .dst = { .fmt = asc_find_format(V4L2_PIX_FMT_ARGB32), .width = 64, .height = 64 }, + }; + + KUNIT_ASSERT_NOT_NULL(test, w); + /* Y'CbCr to RGB without a matrix */ + KUNIT_EXPECT_EQ(test, asc_job_build(&job, w, ASC_JOB_MAX_WRITES), -EINVAL); + KUNIT_ASSERT_EQ(test, asc_csc_compute(V4L2_YCBCR_ENC_709, + V4L2_QUANTIZATION_LIM_RANGE, true, &csc), 0); + job.csc = &csc; + KUNIT_EXPECT_GT(test, asc_job_build(&job, w, ASC_JOB_MAX_WRITES), 0); + KUNIT_EXPECT_EQ(test, asc_job_build(&job, w, 8), -ENOSPC); + /* scaling is not handled by this path yet */ + job.dst.width = 32; + KUNIT_EXPECT_EQ(test, asc_job_build(&job, w, ASC_JOB_MAX_WRITES), -EINVAL); +} + static struct kunit_case asc_test_cases[] = { KUNIT_CASE_PARAM(asc_test_csc, asc_csc_gen_params), KUNIT_CASE(asc_test_csc_rejects), KUNIT_CASE(asc_test_csc_white), KUNIT_CASE(asc_test_ratio), KUNIT_CASE(asc_test_formats), + KUNIT_CASE_PARAM(asc_test_job, asc_job_gen_params), + KUNIT_CASE(asc_test_job_rejects), {} }; From 1bb49d722fbe79c1a276669fc3a0e3d8c5338e3c Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:58:01 +1100 Subject: [PATCH 170/392] media: apple: scaler: add the mem2mem device for same-size conversion Expose the engine as a V4L2 memory-to-memory device. The OUTPUT queue takes NV12, NV12M, P010, ARGB32 or XRGB32, the CAPTURE queue any of them at the same size. Y'CbCr to R'G'B' and back uses the colorimetry of the Y'CbCr side (BT.601, BT.709 or BT.2020, limited or full range); the R'G'B' side is full range and keeps the primaries and transfer function, since the engine only changes the encoding. Each job is written with MMIO in one go from the register list the job builder produces, START last. Completion and errors arrive on the MSR interrupt; an error or a 500 ms timeout fails the job and resets the engine through its power-domain reset before the next one. Interrupts are enabled on the first job after every power-up. Not yet tested on hardware: the DT nodes stay disabled until the qualification run. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/media/platform/apple/scaler/Kconfig | 11 +- .../platform/apple/scaler/apple-scaler-drv.c | 661 +++++++++++++++++- 2 files changed, 635 insertions(+), 37 deletions(-) diff --git a/drivers/media/platform/apple/scaler/Kconfig b/drivers/media/platform/apple/scaler/Kconfig index 799707d9938d1..dcfb18859d689 100644 --- a/drivers/media/platform/apple/scaler/Kconfig +++ b/drivers/media/platform/apple/scaler/Kconfig @@ -5,12 +5,13 @@ config VIDEO_APPLE_SCALER depends on VIDEO_DEV depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) depends on OF_ADDRESS && IOMMU_API && PM + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG help - Driver for the memory-to-memory image scaler and colour-space - converter (MSR) in Apple silicon SoCs. This first version - identifies the engine, attaches its power domains, IOMMU and - interrupt, and keeps it idle; it does not yet expose a video - device. + V4L2 memory-to-memory driver for the image scaler and + colour-space converter (MSR) in Apple silicon SoCs (M1 to M4 + generations and A18 Pro). It converts between Y'CbCr (NV12, + P010) and 32-bit RGB at the same size; scaling follows. To compile this driver as a module, choose M here: the module will be called apple-scaler. diff --git a/drivers/media/platform/apple/scaler/apple-scaler-drv.c b/drivers/media/platform/apple/scaler/apple-scaler-drv.c index 37fd72719bac1..17e901c7e686a 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-drv.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-drv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Apple silicon scaler and colour-space converter (MSR). + * Apple silicon scaler and colour-space converter (MSR), V4L2 mem2mem. * * Every Apple silicon SoC has one or more MSR engines: memory-to-memory * scalers with a programmable colour-space converter, chroma resampling, @@ -9,26 +9,44 @@ * interrupt; later generations front the engine with firmware and are not * handled here. * - * This version only brings the engine up: it attaches the power domains, - * checks the IOMMU, identifies the hardware, takes the completion - * interrupt and keeps the engine idle. Jobs and the V4L2 mem2mem device - * come in later patches. + * The OUTPUT queue takes the source image, the CAPTURE queue the result. + * This version converts between formats at the same size: Y'CbCr <-> + * R'G'B' with the colorimetry of the Y'CbCr side, chroma resampling and + * alpha fill. Scaling comes once the filter path is qualified. */ #include #include #include #include #include +#include #include #include #include #include #include - +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple-scaler-hw.h" #include "apple-scaler-regs.h" #define ASC_NAME "apple-scaler" #define ASC_MIN_REG_SIZE 0x28000 +#define ASC_MIN_SIZE 16 +#define ASC_MAX_SIZE 16384 +#define ASC_STRIDE_ALIGN 128 +#define ASC_JOB_TIMEOUT_MS 500 /** * struct apple_scaler_variant - per-SoC engine description @@ -46,10 +64,35 @@ struct apple_scaler { const struct apple_scaler_variant *variant; struct dev_pm_domain_list *pd_list; struct reset_control *rst; + struct mutex lock; /* serialises ioctls and queue state */ + struct v4l2_device v4l2_dev; + struct video_device vdev; + struct v4l2_m2m_dev *m2m_dev; int irq; u32 id; + + spinlock_t hw_lock; /* protects the fields below */ + struct apple_scaler_ctx *active; + bool need_init; + bool need_reset; + u32 transform_id; + struct delayed_work watchdog; + struct asc_reg writes[ASC_JOB_MAX_WRITES]; }; +struct apple_scaler_ctx { + struct v4l2_fh fh; + struct apple_scaler *asc; + struct v4l2_pix_format_mplane src; /* OUTPUT queue */ + struct v4l2_pix_format_mplane dst; /* CAPTURE queue */ + u32 sequence; +}; + +static struct apple_scaler_ctx *asc_file_to_ctx(struct file *file) +{ + return container_of(file_to_v4l2_fh(file), struct apple_scaler_ctx, fh); +} + static u32 asc_read(struct apple_scaler *asc, u32 reg) { return readl_relaxed(asc->regs + reg); @@ -60,29 +103,549 @@ static void asc_write(struct apple_scaler *asc, u32 reg, u32 val) writel_relaxed(val, asc->regs + reg); } -/* - * No job is ever started yet, so any interrupt is unexpected: acknowledge - * it and report it. The handler only touches the engine while it is - * runtime-active, because a powered-down MSR must not be read. - */ +/* ---- hardware --------------------------------------------------------- */ + +/* After power-up: clear stale status and enable completion and errors. */ +static void asc_hw_init(struct apple_scaler *asc) +{ + asc_write(asc, ASC_IRQ_STATUS, asc_read(asc, ASC_IRQ_STATUS)); + asc_write(asc, ASC_IRQ_ENABLE, ASC_IRQ_DONE | ASC_IRQ_ERRORS); + asc->need_init = false; +} + +static void asc_finish(struct apple_scaler *asc, struct apple_scaler_ctx *ctx, int err) +{ + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + enum vb2_buffer_state state = err ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + struct vb2_v4l2_buffer *src, *dst; + unsigned int i; + + src = v4l2_m2m_src_buf_remove(m2m); + dst = v4l2_m2m_dst_buf_remove(m2m); + if (src && dst) { + for (i = 0; i < dst->vb2_buf.num_planes; i++) + vb2_set_plane_payload(&dst->vb2_buf, i, + err ? 0 : ctx->dst.plane_fmt[i].sizeimage); + v4l2_m2m_buf_copy_metadata(src, dst); + dst->sequence = ctx->sequence++; + v4l2_m2m_buf_done(src, state); + v4l2_m2m_buf_done(dst, state); + } + pm_runtime_put_autosuspend(asc->dev); + v4l2_m2m_job_finish(asc->m2m_dev, m2m); +} + +/* A failed job leaves the engine in an unknown state: reset before the next. */ +static void asc_recover(struct apple_scaler *asc) +{ + asc->need_reset = true; + asc->need_init = true; +} + static irqreturn_t asc_irq(int irq, void *data) { struct apple_scaler *asc = data; + struct apple_scaler_ctx *ctx; u32 status; + int err = 0; if (pm_runtime_get_if_active(asc->dev) <= 0) return IRQ_NONE; + spin_lock(&asc->hw_lock); status = asc_read(asc, ASC_IRQ_STATUS); - if (status) { - asc_write(asc, ASC_IRQ_STATUS, status); - dev_warn_ratelimited(asc->dev, "unexpected interrupt, status %#x\n", - status); + if (!status) { + spin_unlock(&asc->hw_lock); + pm_runtime_put_noidle(asc->dev); + return IRQ_NONE; } - pm_runtime_put_autosuspend(asc->dev); - return status ? IRQ_HANDLED : IRQ_NONE; + asc_write(asc, ASC_IRQ_STATUS, status); + ctx = asc->active; + if (!ctx || !(status & (ASC_IRQ_DONE | ASC_IRQ_ERRORS))) { + spin_unlock(&asc->hw_lock); + dev_warn_ratelimited(asc->dev, "unexpected interrupt, status %#x\n", status); + pm_runtime_put_noidle(asc->dev); + return IRQ_HANDLED; + } + asc->active = NULL; + if (status & ASC_IRQ_ERRORS) { + dev_err_ratelimited(asc->dev, "job failed: status %#x debug %#x\n", + status, asc_read(asc, ASC_DEBUG_STATUS)); + err = -EIO; + } + spin_unlock(&asc->hw_lock); + pm_runtime_put_noidle(asc->dev); + + cancel_delayed_work(&asc->watchdog); + if (err) + asc_recover(asc); + asc_finish(asc, ctx, err); + return IRQ_HANDLED; +} + +static void asc_watchdog(struct work_struct *work) +{ + struct apple_scaler *asc = container_of(work, struct apple_scaler, watchdog.work); + struct apple_scaler_ctx *ctx; + unsigned long flags; + + spin_lock_irqsave(&asc->hw_lock, flags); + ctx = asc->active; + asc->active = NULL; + if (ctx) + asc_write(asc, ASC_IRQ_ENABLE, 0); + spin_unlock_irqrestore(&asc->hw_lock, flags); + if (ctx) { + dev_err(asc->dev, "job timed out (running %#x)\n", + asc_read(asc, ASC_IS_RUNNING)); + asc_recover(asc); + asc_finish(asc, ctx, -ETIMEDOUT); + } +} + +static void asc_image(struct asc_image *img, const struct v4l2_pix_format_mplane *pix, + struct vb2_buffer *vb) +{ + unsigned int i; + + img->fmt = asc_find_format(pix->pixelformat); + img->width = pix->width; + img->height = pix->height; + for (i = 0; i < img->fmt->planes; i++) { + if (pix->num_planes > 1) { + img->dva[i] = vb2_dma_contig_plane_dma_addr(vb, i); + img->stride[i] = pix->plane_fmt[i].bytesperline; + } else { + /* Chroma follows luma in the same buffer, at the same stride. */ + img->dva[i] = vb2_dma_contig_plane_dma_addr(vb, 0) + + (u64)i * pix->plane_fmt[0].bytesperline * pix->height; + img->stride[i] = pix->plane_fmt[0].bytesperline; + } + } +} + +static void asc_device_run(void *priv) +{ + struct apple_scaler_ctx *ctx = priv; + struct apple_scaler *asc = ctx->asc; + struct vb2_v4l2_buffer *src, *dst; + struct asc_job job = {}; + struct asc_csc csc; + unsigned long flags; + int n, ret; + + src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + ret = pm_runtime_resume_and_get(asc->dev); + if (ret < 0) { + pm_runtime_get_noresume(asc->dev); + asc_finish(asc, ctx, ret); + return; + } + + if (asc->need_reset) { + ret = asc->rst ? reset_control_reset(asc->rst) : 0; + if (ret) { + dev_err_ratelimited(asc->dev, "engine reset failed: %d\n", ret); + asc_finish(asc, ctx, ret); + return; + } + asc->need_reset = false; + } + + asc_image(&job.src, &ctx->src, &src->vb2_buf); + asc_image(&job.dst, &ctx->dst, &dst->vb2_buf); + if (job.src.fmt->yuv != job.dst.fmt->yuv) { + const struct v4l2_pix_format_mplane *y = job.src.fmt->yuv ? &ctx->src : &ctx->dst; + + ret = asc_csc_compute(y->ycbcr_enc, y->quantization, job.src.fmt->yuv, &csc); + if (ret) { + asc_finish(asc, ctx, ret); + return; + } + job.csc = &csc; + } + + spin_lock_irqsave(&asc->hw_lock, flags); + /* Job tags as the engine is driven on macOS: a counter above bit 3, bit 2 set. */ + job.transform_id = (++asc->transform_id << 3) | BIT(2); + n = asc_job_build(&job, asc->writes, ARRAY_SIZE(asc->writes)); + if (n < 0) { + spin_unlock_irqrestore(&asc->hw_lock, flags); + asc_finish(asc, ctx, n); + return; + } + if (asc->need_init) + asc_hw_init(asc); + asc->active = ctx; + /* Buffers must be visible to the engine before START, the last write. */ + dma_wmb(); + for (int i = 0; i < n; i++) + writel(asc->writes[i].val, asc->regs + asc->writes[i].off); + spin_unlock_irqrestore(&asc->hw_lock, flags); + schedule_delayed_work(&asc->watchdog, msecs_to_jiffies(ASC_JOB_TIMEOUT_MS)); +} + +static const struct v4l2_m2m_ops asc_m2m_ops = { + .device_run = asc_device_run, +}; + +/* ---- formats ---------------------------------------------------------- */ + +/* + * Fill in a format the engine can take. The CAPTURE side has the OUTPUT + * size (no scaling yet). The Y'CbCr side of a conversion carries the + * colorimetry; the R'G'B' side is full range with the same primaries and + * transfer function, since the engine converts only the encoding. + */ +static void asc_fill(struct v4l2_pix_format_mplane *pix, const struct v4l2_pix_format_mplane *ref, + u32 default_fourcc) +{ + const struct asc_format *f = asc_find_format(pix->pixelformat); + u32 w = ref ? ref->width : pix->width, h = ref ? ref->height : pix->height; + struct v4l2_pix_format_mplane in = *pix; + + if (!f) + f = asc_find_format(default_fourcc); + w = clamp_t(u32, ALIGN(w, f->hsub), ASC_MIN_SIZE, ASC_MAX_SIZE); + h = clamp_t(u32, ALIGN(h, f->vsub), ASC_MIN_SIZE, ASC_MAX_SIZE); + + memset(pix, 0, sizeof(*pix)); + v4l2_fill_pixfmt_mp_aligned(pix, f->fourcc, w, h, ASC_STRIDE_ALIGN); + pix->field = V4L2_FIELD_NONE; + + if (ref && ref->colorspace != V4L2_COLORSPACE_DEFAULT) { + pix->colorspace = ref->colorspace; + pix->xfer_func = ref->xfer_func; + } else { + pix->colorspace = in.colorspace ?: V4L2_COLORSPACE_REC709; + pix->xfer_func = in.xfer_func; + } + if (f->yuv) { + pix->ycbcr_enc = in.ycbcr_enc; + if (ref && v4l2_is_format_yuv(v4l2_format_info(ref->pixelformat))) + pix->ycbcr_enc = ref->ycbcr_enc; + if (pix->ycbcr_enc == V4L2_YCBCR_ENC_DEFAULT) + pix->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix->colorspace); + /* Only the encodings the matrix can express. */ + if (pix->ycbcr_enc != V4L2_YCBCR_ENC_601 && pix->ycbcr_enc != V4L2_YCBCR_ENC_709 && + pix->ycbcr_enc != V4L2_YCBCR_ENC_BT2020) + pix->ycbcr_enc = V4L2_YCBCR_ENC_709; + pix->quantization = in.quantization; + if (ref && v4l2_is_format_yuv(v4l2_format_info(ref->pixelformat))) + pix->quantization = ref->quantization; + if (pix->quantization == V4L2_QUANTIZATION_DEFAULT) + pix->quantization = V4L2_QUANTIZATION_LIM_RANGE; + } else { + pix->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + pix->quantization = V4L2_QUANTIZATION_FULL_RANGE; + } + if (pix->xfer_func == V4L2_XFER_FUNC_DEFAULT) + pix->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix->colorspace); +} + +static int asc_querycap(struct file *file, void *priv, struct v4l2_capability *cap) +{ + strscpy(cap->driver, ASC_NAME, sizeof(cap->driver)); + strscpy(cap->card, "Apple scaler/CSC", sizeof(cap->card)); + return 0; +} + +static int asc_enum_fmt(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + const struct asc_format *fmt = asc_format_by_index(f->index); + + if (!fmt) + return -EINVAL; + f->pixelformat = fmt->fourcc; + return 0; +} + +static int asc_enum_framesizes(struct file *file, void *priv, struct v4l2_frmsizeenum *fs) +{ + const struct asc_format *f = asc_find_format(fs->pixel_format); + + if (fs->index || !f) + return -EINVAL; + fs->type = V4L2_FRMSIZE_TYPE_STEPWISE; + fs->stepwise = (struct v4l2_frmsize_stepwise){ + ASC_MIN_SIZE, ASC_MAX_SIZE, f->hsub, ASC_MIN_SIZE, ASC_MAX_SIZE, f->vsub, + }; + return 0; +} + +static int asc_g_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = asc_file_to_ctx(file)->src; + return 0; +} + +static int asc_g_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = asc_file_to_ctx(file)->dst; + return 0; +} + +static int asc_try_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + asc_fill(&f->fmt.pix_mp, NULL, V4L2_PIX_FMT_NV12M); + return 0; +} + +static int asc_try_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + asc_fill(&f->fmt.pix_mp, &asc_file_to_ctx(file)->src, V4L2_PIX_FMT_ARGB32); + return 0; +} + +static int asc_s_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + struct apple_scaler_ctx *ctx = asc_file_to_ctx(file); + + if (vb2_is_busy(v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx)) || + vb2_is_busy(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx))) + return -EBUSY; + asc_fill(&f->fmt.pix_mp, NULL, V4L2_PIX_FMT_NV12M); + ctx->src = f->fmt.pix_mp; + asc_fill(&ctx->dst, &ctx->src, V4L2_PIX_FMT_ARGB32); + return 0; +} + +static int asc_s_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + struct apple_scaler_ctx *ctx = asc_file_to_ctx(file); + + if (vb2_is_busy(v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx))) + return -EBUSY; + asc_fill(&f->fmt.pix_mp, &ctx->src, V4L2_PIX_FMT_ARGB32); + ctx->dst = f->fmt.pix_mp; + return 0; +} + +static const struct v4l2_ioctl_ops asc_ioctl_ops = { + .vidioc_querycap = asc_querycap, + .vidioc_enum_fmt_vid_out = asc_enum_fmt, + .vidioc_enum_fmt_vid_cap = asc_enum_fmt, + .vidioc_enum_framesizes = asc_enum_framesizes, + .vidioc_g_fmt_vid_out_mplane = asc_g_fmt_out, + .vidioc_g_fmt_vid_cap_mplane = asc_g_fmt_cap, + .vidioc_try_fmt_vid_out_mplane = asc_try_fmt_out, + .vidioc_try_fmt_vid_cap_mplane = asc_try_fmt_cap, + .vidioc_s_fmt_vid_out_mplane = asc_s_fmt_out, + .vidioc_s_fmt_vid_cap_mplane = asc_s_fmt_cap, + + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +/* ---- queues ----------------------------------------------------------- */ + +static struct v4l2_pix_format_mplane *asc_q_fmt(struct apple_scaler_ctx *ctx, u32 type) +{ + return V4L2_TYPE_IS_OUTPUT(type) ? &ctx->src : &ctx->dst; } +static int asc_queue_setup(struct vb2_queue *q, unsigned int *nbufs, unsigned int *nplanes, + unsigned int sizes[], struct device *alloc_devs[]) +{ + struct v4l2_pix_format_mplane *pix = asc_q_fmt(vb2_get_drv_priv(q), q->type); + unsigned int i; + + if (*nplanes) { + if (*nplanes != pix->num_planes) + return -EINVAL; + for (i = 0; i < *nplanes; i++) + if (sizes[i] < pix->plane_fmt[i].sizeimage) + return -EINVAL; + return 0; + } + *nplanes = pix->num_planes; + for (i = 0; i < *nplanes; i++) + sizes[i] = pix->plane_fmt[i].sizeimage; + return 0; +} + +static int asc_buf_prepare(struct vb2_buffer *vb) +{ + struct v4l2_pix_format_mplane *pix = asc_q_fmt(vb2_get_drv_priv(vb->vb2_queue), vb->type); + unsigned int i; + + for (i = 0; i < pix->num_planes; i++) { + if (vb2_plane_size(vb, i) < pix->plane_fmt[i].sizeimage) + return -EINVAL; + if (vb2_dma_contig_plane_dma_addr(vb, i) & (ASC_STRIDE_ALIGN - 1)) + return -EINVAL; + if (V4L2_TYPE_IS_OUTPUT(vb->type)) + vb2_set_plane_payload(vb, i, pix->plane_fmt[i].sizeimage); + else + vb2_set_plane_payload(vb, i, 0); + } + return 0; +} + +static void asc_buf_queue(struct vb2_buffer *vb) +{ + struct apple_scaler_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); +} + +static int asc_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct apple_scaler_ctx *ctx = vb2_get_drv_priv(q); + + if (V4L2_TYPE_IS_OUTPUT(q->type)) + ctx->sequence = 0; + return 0; +} + +static void asc_stop_streaming(struct vb2_queue *q) +{ + struct apple_scaler_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_v4l2_buffer *vbuf; + + for (;;) { + vbuf = V4L2_TYPE_IS_OUTPUT(q->type) ? v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx) : + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!vbuf) + break; + v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR); + } +} + +static const struct vb2_ops asc_qops = { + .queue_setup = asc_queue_setup, + .buf_prepare = asc_buf_prepare, + .buf_queue = asc_buf_queue, + .start_streaming = asc_start_streaming, + .stop_streaming = asc_stop_streaming, +}; + +static int asc_init_queue(struct apple_scaler_ctx *ctx, struct vb2_queue *q, u32 type) +{ + q->type = type; + q->io_modes = VB2_MMAP | VB2_DMABUF; + q->drv_priv = ctx; + q->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + q->ops = &asc_qops; + q->mem_ops = &vb2_dma_contig_memops; + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + q->lock = &ctx->asc->lock; + q->dev = ctx->asc->dev; + return vb2_queue_init(q); +} + +static int asc_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *dst) +{ + int ret = asc_init_queue(priv, src, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + + return ret ?: asc_init_queue(priv, dst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); +} + +/* ---- file operations -------------------------------------------------- */ + +static int asc_open(struct file *file) +{ + struct apple_scaler *asc = video_drvdata(file); + struct apple_scaler_ctx *ctx; + int ret; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + ctx->asc = asc; + ctx->src.width = 1920; + ctx->src.height = 1080; + ctx->src.pixelformat = V4L2_PIX_FMT_NV12M; + asc_fill(&ctx->src, NULL, V4L2_PIX_FMT_NV12M); + ctx->dst.pixelformat = V4L2_PIX_FMT_ARGB32; + asc_fill(&ctx->dst, &ctx->src, V4L2_PIX_FMT_ARGB32); + v4l2_fh_init(&ctx->fh, video_devdata(file)); + + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(asc->m2m_dev, ctx, asc_queue_init); + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return ret; + } + v4l2_fh_add(&ctx->fh, file); + return 0; +} + +static int asc_release(struct file *file) +{ + struct apple_scaler_ctx *ctx = asc_file_to_ctx(file); + struct apple_scaler *asc = ctx->asc; + + mutex_lock(&asc->lock); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + mutex_unlock(&asc->lock); + v4l2_fh_del(&ctx->fh, file); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return 0; +} + +static const struct v4l2_file_operations asc_fops = { + .owner = THIS_MODULE, + .open = asc_open, + .release = asc_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +/* ---- power management and probe --------------------------------------- */ + +static int asc_runtime_suspend(struct device *dev) +{ + return 0; +} + +static int asc_runtime_resume(struct device *dev) +{ + struct apple_scaler *asc = dev_get_drvdata(dev); + + /* The engine loses its state when powered down. */ + asc->need_init = true; + return 0; +} + +static int asc_suspend(struct device *dev) +{ + struct apple_scaler *asc = dev_get_drvdata(dev); + + v4l2_m2m_suspend(asc->m2m_dev); + return pm_runtime_force_suspend(dev); +} + +static int asc_resume(struct device *dev) +{ + struct apple_scaler *asc = dev_get_drvdata(dev); + int ret = pm_runtime_force_resume(dev); + + if (ret) + return ret; + v4l2_m2m_resume(asc->m2m_dev); + return 0; +} + +static const struct dev_pm_ops asc_pm_ops = { + RUNTIME_PM_OPS(asc_runtime_suspend, asc_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(asc_suspend, asc_resume) +}; + static int asc_identify(struct apple_scaler *asc) { u32 running, status; @@ -95,7 +658,7 @@ static int asc_identify(struct apple_scaler *asc) asc->id = asc_read(asc, ASC_HW_VERSION); running = asc_read(asc, ASC_IS_RUNNING); status = asc_read(asc, ASC_IRQ_STATUS); - /* Nothing may be enabled until a job path owns the engine. */ + /* Masked until the first job enables it. */ asc_write(asc, ASC_IRQ_ENABLE, 0); if (status) asc_write(asc, ASC_IRQ_STATUS, status); @@ -109,26 +672,21 @@ static int asc_identify(struct apple_scaler *asc) return 0; } -static int asc_runtime_suspend(struct device *dev) +static void asc_cleanup(void *data) { - return 0; -} + struct apple_scaler *asc = data; -static int asc_runtime_resume(struct device *dev) -{ - /* The engine loses its state when powered down; jobs program it fully. */ - return 0; + cancel_delayed_work_sync(&asc->watchdog); + if (asc->m2m_dev) + v4l2_m2m_release(asc->m2m_dev); + v4l2_device_unregister(&asc->v4l2_dev); } -static const struct dev_pm_ops asc_pm_ops = { - RUNTIME_PM_OPS(asc_runtime_suspend, asc_runtime_resume, NULL) - SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) -}; - static int asc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_scaler *asc; + struct video_device *vdev; struct resource *res; int ret; @@ -139,6 +697,10 @@ static int asc_probe(struct platform_device *pdev) asc->variant = of_device_get_match_data(dev); if (!asc->variant) return -ENODEV; + mutex_init(&asc->lock); + spin_lock_init(&asc->hw_lock); + INIT_DELAYED_WORK(&asc->watchdog, asc_watchdog); + asc->need_init = true; platform_set_drvdata(pdev, asc); if (!device_iommu_mapped(dev)) @@ -161,7 +723,7 @@ static int asc_probe(struct platform_device *pdev) return dev_err_probe(dev, -ENODEV, "requires a power domain\n"); } - if (platform_get_resource(pdev, IORESOURCE_MEM, asc->variant->num_reg - 1) == NULL) + if (!platform_get_resource(pdev, IORESOURCE_MEM, asc->variant->num_reg - 1)) return dev_err_probe(dev, -EINVAL, "expected %u register ranges\n", asc->variant->num_reg); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -193,7 +755,41 @@ static int asc_probe(struct platform_device *pdev) asc->irq = platform_get_irq(pdev, 0); if (asc->irq < 0) return asc->irq; - return devm_request_irq(dev, asc->irq, asc_irq, 0, dev_name(dev), asc); + ret = devm_request_irq(dev, asc->irq, asc_irq, 0, dev_name(dev), asc); + if (ret) + return ret; + + ret = v4l2_device_register(dev, &asc->v4l2_dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, asc_cleanup, asc); + if (ret) + return ret; + asc->m2m_dev = v4l2_m2m_init(&asc_m2m_ops); + if (IS_ERR(asc->m2m_dev)) { + ret = PTR_ERR(asc->m2m_dev); + asc->m2m_dev = NULL; + return ret; + } + + vdev = &asc->vdev; + strscpy(vdev->name, ASC_NAME, sizeof(vdev->name)); + vdev->fops = &asc_fops; + vdev->ioctl_ops = &asc_ioctl_ops; + vdev->release = video_device_release_empty; + vdev->lock = &asc->lock; + vdev->v4l2_dev = &asc->v4l2_dev; + vdev->vfl_dir = VFL_DIR_M2M; + vdev->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + video_set_drvdata(vdev, asc); + return video_register_device(vdev, VFL_TYPE_VIDEO, -1); +} + +static void asc_remove(struct platform_device *pdev) +{ + struct apple_scaler *asc = platform_get_drvdata(pdev); + + video_unregister_device(&asc->vdev); } static const struct apple_scaler_variant apple_scaler_t6041 = { @@ -215,6 +811,7 @@ MODULE_DEVICE_TABLE(of, asc_of_match); static struct platform_driver asc_driver = { .probe = asc_probe, + .remove = asc_remove, .driver = { .name = ASC_NAME, .of_match_table = asc_of_match, From 8e57d48456ec0e11ab5f10d56149c3e1c9d62918 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:23:14 +1100 Subject: [PATCH 171/392] media: apple: scaler: pulse the engine reset before the first job After every power-up and after a failed job, wait for the engine to go idle, write RESET 1 then 0 without reading anything in between (the register file does not answer while reset is held), wait for idle again, then clear stale status and enable the done and error interrupts. Only if the engine never goes idle fall back to the PMGR reset of its power domain and retry once; otherwise fail the job. This is the start-up order the engine expects on every generation from M1 Max onwards. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .../platform/apple/scaler/apple-scaler-drv.c | 44 ++++++++++++++----- .../platform/apple/scaler/apple-scaler-regs.h | 1 + 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/apple/scaler/apple-scaler-drv.c b/drivers/media/platform/apple/scaler/apple-scaler-drv.c index 17e901c7e686a..6a8d5f8438ce8 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-drv.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-drv.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #define ASC_MAX_SIZE 16384 #define ASC_STRIDE_ALIGN 128 #define ASC_JOB_TIMEOUT_MS 500 +#define ASC_IDLE_TIMEOUT_US 10000 /** * struct apple_scaler_variant - per-SoC engine description @@ -74,7 +76,6 @@ struct apple_scaler { spinlock_t hw_lock; /* protects the fields below */ struct apple_scaler_ctx *active; bool need_init; - bool need_reset; u32 transform_id; struct delayed_work watchdog; struct asc_reg writes[ASC_JOB_MAX_WRITES]; @@ -105,12 +106,30 @@ static void asc_write(struct apple_scaler *asc, u32 reg, u32 val) /* ---- hardware --------------------------------------------------------- */ -/* After power-up: clear stale status and enable completion and errors. */ -static void asc_hw_init(struct apple_scaler *asc) +/* + * After power-up or a failed job: wait for idle, pulse the engine reset + * (no register may be read while it is held), clear stale status and + * enable completion and errors. + */ +static int asc_hw_init(struct apple_scaler *asc) { + u32 v; + int ret; + + ret = readl_relaxed_poll_timeout(asc->regs + ASC_IS_RUNNING, v, !(v & BIT(0)), + 10, ASC_IDLE_TIMEOUT_US); + if (ret) + return ret; + asc_write(asc, ASC_RESET, ASC_RESET_ENGINE); + asc_write(asc, ASC_RESET, 0); + ret = readl_relaxed_poll_timeout(asc->regs + ASC_IS_RUNNING, v, !(v & BIT(0)), + 10, ASC_IDLE_TIMEOUT_US); + if (ret) + return ret; asc_write(asc, ASC_IRQ_STATUS, asc_read(asc, ASC_IRQ_STATUS)); asc_write(asc, ASC_IRQ_ENABLE, ASC_IRQ_DONE | ASC_IRQ_ERRORS); asc->need_init = false; + return 0; } static void asc_finish(struct apple_scaler *asc, struct apple_scaler_ctx *ctx, int err) @@ -138,7 +157,6 @@ static void asc_finish(struct apple_scaler *asc, struct apple_scaler_ctx *ctx, i /* A failed job leaves the engine in an unknown state: reset before the next. */ static void asc_recover(struct apple_scaler *asc) { - asc->need_reset = true; asc->need_init = true; } @@ -243,14 +261,20 @@ static void asc_device_run(void *priv) return; } - if (asc->need_reset) { - ret = asc->rst ? reset_control_reset(asc->rst) : 0; + /* + * Engine reset and interrupt setup, once per power-up and after any + * failure. If the engine does not go idle, fall back to the PMGR reset + * of its power domain and try once more. + */ + if (asc->need_init) { + ret = asc_hw_init(asc); + if (ret && asc->rst && !reset_control_reset(asc->rst)) + ret = asc_hw_init(asc); if (ret) { - dev_err_ratelimited(asc->dev, "engine reset failed: %d\n", ret); - asc_finish(asc, ctx, ret); + dev_err_ratelimited(asc->dev, "engine does not go idle: %d\n", ret); + asc_finish(asc, ctx, -EIO); return; } - asc->need_reset = false; } asc_image(&job.src, &ctx->src, &src->vb2_buf); @@ -275,8 +299,6 @@ static void asc_device_run(void *priv) asc_finish(asc, ctx, n); return; } - if (asc->need_init) - asc_hw_init(asc); asc->active = ctx; /* Buffers must be visible to the engine before START, the last write. */ dma_wmb(); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-regs.h b/drivers/media/platform/apple/scaler/apple-scaler-regs.h index 7402b9a97752c..6abbcb7759394 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-regs.h +++ b/drivers/media/platform/apple/scaler/apple-scaler-regs.h @@ -13,6 +13,7 @@ #define ASC_HW_VERSION 0x0000 #define ASC_RESET 0x0004 +#define ASC_RESET_ENGINE BIT(0) #define ASC_RESET_APIODMA 0x0008 #define ASC_IS_RUNNING 0x000c #define ASC_REG_FIFO_AVAIL 0x0014 From 1e46c8f78251bc267552a2f5004e6dacf1001cee Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:14:09 +1100 Subject: [PATCH 172/392] arm64: dts: apple: t6040: debounce the GPU zones critical trips The seven GPU zones (Tg05, Tg0S, Tg0Y, Tg0k, Tg0z, Tg1V, Tg1l) shut the Mac down on the first 1 s SMC sample at or above 118 C. The CPU and die zones in the same file already need three samples (critical-samples), because single SMC samples on these Macs jump by several degrees during load transients. Give the GPU zones the same rule. On a J616s the GPU keys read real values only while the GPU is powered (glmark2: Tg05 49 to 53 C, about 8 C above Tp0G) and fall to -4 to 4 C on the first sample after the firmware powers the GPU off. The transition samples measured at 1 Hz stay in range (no spikes), so the debounce only guards against noise; it does not hide a real excursion, which still shuts down after three seconds. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi b/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi index 74af4a032b1d8..1dab69b0f9a0e 100644 --- a/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi @@ -172,8 +172,9 @@ * throttle only the cluster their sensor sits on. The 1 s samples * jump by about 10 C, so the zones are integral-led. * - GPU zones: the GPU firmware limits itself from 101 C; these shed - * CPU power from the same point (on at 90 C) and shut down at 118 C. - * A powered-off GPU reads -4 to 4 C. + * CPU power from the same point (on at 90 C) and shut down at 118 C + * after three samples, as the CPU zones do. A powered-off GPU reads + * -4 to 4 C. * - Hot at 120 C only reports; critical is the ADT's 125 C sochot * level, after three samples (critical-samples). * - The fans stay with the SMC, as on macOS: no active trips. @@ -1000,6 +1001,7 @@ gpu0-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 17>; /* Tg05 */ @@ -1058,6 +1060,7 @@ gpu1-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 18>; /* Tg0S */ @@ -1116,6 +1119,7 @@ gpu2-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 19>; /* Tg0Y */ @@ -1174,6 +1178,7 @@ gpu3-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 20>; /* Tg0k */ @@ -1232,6 +1237,7 @@ gpu4-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 21>; /* Tg0z */ @@ -1290,6 +1296,7 @@ gpu5-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 22>; /* Tg1V */ @@ -1348,6 +1355,7 @@ gpu6-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + critical-samples = <3>; sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 23>; /* Tg1l */ From 97bc174a0e86cf91f7fd718482524ca48a950031 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:39:24 +1100 Subject: [PATCH 173/392] hwmon: macsmc: report the M4 Pro GPU sensors' powered-down placeholders as no data On the M4 Pro (T6040) the SMC publishes every GPU temperature key (Tg*) only while the GPU is powered. When the GPU firmware powers the GPU off at idle, the next 1 Hz SMC update replaces each key with a placeholder: -4.0 C on the base keys, 0, 2.2 or 3.9 C on the others. Linux reported those as temperatures: `sensors` showed a 3.9 C GPU, and the seven GPU thermal zones fed it to the power allocator as a reading. Describe the placeholder range in the device tree: apple,no-data-below-millicelsius on a sensor node makes macsmc-hwmon return -ENODATA for a reading below it, so userspace sees no reading instead of a false one. The hwmon thermal bridge turns -ENODATA into THERMAL_TEMP_INVALID, which the thermal core already treats as "no valid temperature this time": the zone keeps polling and its last real temperature, and the update is skipped. Without that, the core would back off on the error and eventually disable the zone as broken, so it could not trip when the GPU powered up again. The 22 Tg* sensors of the T604x MacBook Pros get a 10 C threshold. The real readings measured on a J616s are 36-75 C, and a powered GPU does not run below 10 C. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../bindings/hwmon/apple,smc-hwmon.yaml | 9 +++++++++ .../boot/dts/apple/t6040-gpu-thermal.dtsi | 7 +++++++ arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi | 15 +++++++++++++++ drivers/hwmon/hwmon.c | 9 +++++++++ drivers/hwmon/macsmc-hwmon.c | 18 +++++++++++++++--- 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index fcfe475d4dcf2..e98bdcc7b7897 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -33,6 +33,15 @@ $defs: label: description: Human-readable name for the sensor + apple,no-data-below-millicelsius: + $ref: /schemas/types.yaml#/definitions/int32 + description: + The SMC publishes a placeholder below this temperature, in + millidegrees Celsius, while the sensor is powered down (the GPU + temperature sensors of the M4 Pro read -4 to 4 C while the GPU is + off). Such readings are reported as no data instead of as a + temperature, and a thermal zone on the sensor skips them. + required: - apple,key-id diff --git a/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi b/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi index 1dab69b0f9a0e..6d719c7532867 100644 --- a/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-gpu-thermal.dtsi @@ -1507,35 +1507,42 @@ temperature-Tg05 { apple,key-id = "Tg05"; label = "GPU Tg05"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0S { apple,key-id = "Tg0S"; label = "GPU Tg0S"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0Y { apple,key-id = "Tg0Y"; label = "GPU Tg0Y"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0k { apple,key-id = "Tg0k"; label = "GPU Tg0k"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0z { apple,key-id = "Tg0z"; label = "GPU Tg0z"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1V { apple,key-id = "Tg1V"; label = "GPU Tg1V"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1l { apple,key-id = "Tg1l"; label = "GPU Tg1l"; + apple,no-data-below-millicelsius = <10000>; }; }; diff --git a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi index 864e87400bb87..c431dec0f3c4f 100644 --- a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi +++ b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi @@ -1011,76 +1011,91 @@ temperature-Tg04 { apple,key-id = "Tg04"; label = "GPU Tg04"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0K { apple,key-id = "Tg0K"; label = "GPU Tg0K"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0L { apple,key-id = "Tg0L"; label = "GPU Tg0L"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0R { apple,key-id = "Tg0R"; label = "GPU Tg0R"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0X { apple,key-id = "Tg0X"; label = "GPU Tg0X"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0d { apple,key-id = "Tg0d"; label = "GPU Tg0d"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0e { apple,key-id = "Tg0e"; label = "GPU Tg0e"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0j { apple,key-id = "Tg0j"; label = "GPU Tg0j"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg0y { apple,key-id = "Tg0y"; label = "GPU Tg0y"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1E { apple,key-id = "Tg1E"; label = "GPU Tg1E"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1F { apple,key-id = "Tg1F"; label = "GPU Tg1F"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1U { apple,key-id = "Tg1U"; label = "GPU Tg1U"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1c { apple,key-id = "Tg1c"; label = "GPU Tg1c"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1d { apple,key-id = "Tg1d"; label = "GPU Tg1d"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tg1k { apple,key-id = "Tg1k"; label = "GPU Tg1k"; + apple,no-data-below-millicelsius = <10000>; }; temperature-Tp00 { diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 10d2df3efdfae..96381f929842b 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -171,6 +171,15 @@ static int hwmon_thermal_get_temp(struct thermal_zone_device *tz, int *temp) ret = hwdev->chip->ops->read(tdata->dev, hwmon_temp, hwmon_temp_input, tdata->index, &t); + /* + * -ENODATA: the sensor has no reading right now (for example it is + * powered down). Let the zone skip this update and keep polling, + * instead of backing off and eventually disabling it as broken. + */ + if (ret == -ENODATA) { + *temp = THERMAL_TEMP_INVALID; + return 0; + } if (ret < 0) return ret; diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 0e146c0fa68fc..d0206489fa372 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -74,6 +74,9 @@ struct macsmc_hwmon_sensor { u32 attrs; /* Part of the SoC die temperature zone. */ bool die; + /* Readings below no_data_below are powered-down placeholders, not temperatures. */ + bool has_no_data_below; + s32 no_data_below; }; struct macsmc_hwmon_fan { @@ -616,10 +619,15 @@ static int macsmc_hwmon_read(struct device *dev, enum hwmon_sensor_types type, int ret = 0; switch (type) { - case hwmon_temp: - ret = macsmc_hwmon_read_key(hwmon->smc, - &hwmon->temp.sensors[channel], 1000, val); + case hwmon_temp: { + struct macsmc_hwmon_sensor *sensor = &hwmon->temp.sensors[channel]; + + ret = macsmc_hwmon_read_key(hwmon->smc, sensor, 1000, val); + /* A powered-down sensor's placeholder is not a temperature. */ + if (!ret && sensor->has_no_data_below && *val < sensor->no_data_below) + ret = -ENODATA; break; + } case hwmon_in: ret = macsmc_hwmon_read_key(hwmon->smc, &hwmon->volt.sensors[channel], 1000, val); @@ -778,6 +786,10 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, else strscpy_pad(sensor->label, label, sizeof(sensor->label)); + sensor->has_no_data_below = + !of_property_read_s32(sensor_node, "apple,no-data-below-millicelsius", + &sensor->no_data_below); + return 0; } From b724b53d26b90e520d23b3a865c732df1dbd10d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 21:14:17 +1100 Subject: [PATCH 174/392] PCI: apple: attach and retain all host power domains in s2idle Devices with multiple power-domains entries are not attached by the platform bus. Attach each domain with an active device link before touching the controller, while leaving the single-domain attachment to the bus. Preallocate one notifier per attached domain and extend the PCIe-C sleep hold to cover all of them. Internal hosts leave their links running in suspend-to-idle and cannot recover from a power cycle, so veto power-off on every domain until resume. Unwind partial holds on error and abort suspend rather than lose an internal link. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 136 +++++++++++++++++++++------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 6ad3376dcf75e..409b95e68e1a9 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -214,7 +214,7 @@ struct hw_info { bool msimap_by_vector; /* * In suspend-to-idle a tunnel host whose power domain is not always-on - * keeps its link while the domain stays on, see apple_pcie_hold_domain(). + * keeps its link while the domain stays on, see apple_pcie_hold_domains(). */ bool s2idle_hold_domain; /* AppleT8152PCIePort::_initializeRefclkBuffer clears REFBUF_OFF. */ @@ -391,6 +391,12 @@ bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); +struct apple_pcie_pd { + struct device *dev; + struct notifier_block nb; + unsigned int vetoed; +}; + struct apple_pcie { struct mutex lock; struct device *dev; @@ -413,10 +419,10 @@ struct apple_pcie { bool bus_stopped; bool reset_on_resume; bool link_kept; - /* a power domain that is not always-on, held on for this sleep */ - struct notifier_block pd_nb; - bool pd_held; - unsigned int pd_vetoed; + struct dev_pm_domain_list *pd_list; + struct apple_pcie_pd *pds; + unsigned int num_pds; + unsigned int pd_held; bool resume_failed; /* cleared only by teardown and reprobe */ struct reset_control *reset; u16 managed_sids; @@ -3101,6 +3107,42 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) return ret; } +static int apple_pcie_attach_domains(struct apple_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct dev_pm_domain_attach_data data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + int count, ret, i; + + if (!of_property_present(dev->of_node, "power-domains")) + return 0; + + count = of_count_phandle_with_args(dev->of_node, "power-domains", + "#power-domain-cells"); + if (count < 0) + return count; + if (count > 1) { + ret = devm_pm_domain_attach_list(dev, &data, &pcie->pd_list); + if (ret < 0) + return ret; + if (ret != count) + return -ENODEV; + } else if (!dev->pm_domain) { + return 0; + } + + /* A single domain is already attached by the platform bus. */ + pcie->pds = devm_kcalloc(dev, count, sizeof(*pcie->pds), GFP_KERNEL); + if (!pcie->pds) + return -ENOMEM; + pcie->num_pds = count; + for (i = 0; i < count; i++) + pcie->pds[i].dev = pcie->pd_list ? pcie->pd_list->pd_devs[i] : dev; + + return 0; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -3148,6 +3190,10 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie = pci_host_bridge_priv(bridge); pcie->dev = dev; pcie->hw = hw; + ret = apple_pcie_attach_domains(pcie); + if (ret) + return dev_err_probe(dev, ret, "Failed to attach power domains\n"); + if (of_property_present(dev->of_node, "apple,managed-sids")) { u32 ids[15]; int count = of_property_count_u32_elems(dev->of_node, "apple,managed-sids"); @@ -3828,46 +3874,63 @@ static bool apple_pcie_tunnel_link_healthy(struct apple_pcie *pcie) */ static int apple_pcie_pd_notify(struct notifier_block *nb, unsigned long action, void *data) { - struct apple_pcie *pcie = container_of(nb, struct apple_pcie, pd_nb); + struct apple_pcie_pd *pd = container_of(nb, struct apple_pcie_pd, nb); if (action != GENPD_NOTIFY_PRE_OFF) return NOTIFY_OK; - pcie->pd_vetoed++; + pd->vetoed++; return notifier_from_errno(-EBUSY); } +static void apple_pcie_release_domains(struct apple_pcie *pcie) +{ + unsigned int vetoed = 0; + struct apple_pcie_pd *pd; + + if (!pcie->pd_held) + return; + while (pcie->pd_held) { + pd = &pcie->pds[--pcie->pd_held]; + dev_pm_genpd_remove_notifier(pd->dev); + vetoed += pd->vetoed; + } + dev_dbg(pcie->dev, "PCIe power domains released after suspend-to-idle (%u power-offs refused)\n", + vetoed); +} + /* - * Hold a power domain that is not always-on through this sleep. ACIO keeps + * Hold every attached domain through this sleep. Internal links are left + * running and cannot be retrained after a power cycle. ACIO keeps * the domain of its PCIe-C host on for as long as a cable is connected (its * power-domain devices are syscore devices, which system sleep leaves * alone), so a host with a link up finds its domain on here, and the veto * makes sure it stays on. A domain that is already off has no link to keep. */ -static int apple_pcie_hold_domain(struct apple_pcie *pcie) +static int apple_pcie_hold_domains(struct apple_pcie *pcie) { + struct apple_pcie_pd *pd; int ret; - pcie->pd_vetoed = 0; - pcie->pd_nb.notifier_call = apple_pcie_pd_notify; - ret = dev_pm_genpd_add_notifier(pcie->dev, &pcie->pd_nb); - if (ret) - return ret; - if (!dev_pm_genpd_is_on(pcie->dev)) { - dev_pm_genpd_remove_notifier(pcie->dev); + if (!pcie->num_pds) return -ENODEV; + while (pcie->pd_held < pcie->num_pds) { + pd = &pcie->pds[pcie->pd_held]; + pd->vetoed = 0; + pd->nb.notifier_call = apple_pcie_pd_notify; + ret = dev_pm_genpd_add_notifier(pd->dev, &pd->nb); + if (ret) + goto err_release; + pcie->pd_held++; + if (!dev_pm_genpd_is_on(pd->dev)) { + ret = -ENODEV; + goto err_release; + } } - pcie->pd_held = true; return 0; -} -static void apple_pcie_release_domain(struct apple_pcie *pcie) -{ - if (!pcie->pd_held) - return; - dev_pm_genpd_remove_notifier(pcie->dev); - pcie->pd_held = false; - dev_dbg(pcie->dev, "PCIe-C power domain released after suspend-to-idle (%u power-offs refused)\n", - pcie->pd_vetoed); +err_release: + apple_pcie_release_domains(pcie); + return ret; } /* @@ -3876,7 +3939,7 @@ static void apple_pcie_release_domain(struct apple_pcie *pcie) * and resume like any other PCIe device; nothing has to be re-trained. * The host's power domain must stay on for that: an always-on one does; * on a host that opts in (s2idle_hold_domain), any other is held on for the - * sleep (apple_pcie_hold_domain()). + * sleep (apple_pcie_hold_domains()). */ static bool apple_pcie_keep_link(struct apple_pcie *pcie) { @@ -3891,7 +3954,7 @@ static bool apple_pcie_keep_link(struct apple_pcie *pcie) if (!pcie->power_retained) { if (!pcie->hw->s2idle_hold_domain) return false; - ret = apple_pcie_hold_domain(pcie); + ret = apple_pcie_hold_domains(pcie); if (ret) { dev_dbg(pcie->dev, "PCIe-C power domain not held (%d): link not kept\n", ret); @@ -3900,7 +3963,7 @@ static bool apple_pcie_keep_link(struct apple_pcie *pcie) } if (apple_pcie_tunnel_link_healthy(pcie)) return true; - apple_pcie_release_domain(pcie); + apple_pcie_release_domains(pcie); return false; } @@ -3948,12 +4011,15 @@ static int apple_pcie_suspend_noirq(struct device *dev) /* * The managed radio has no suspend boundary of its own; suspend-to-idle - * leaves it in place like any other internal port (no power domain). + * leaves it in place like any other internal port. */ - if (pcie->managed_sids) - return pm_suspend_target_state == PM_SUSPEND_TO_IDLE ? 0 : -EBUSY; - if (!pcie->hw->tunneled) + if (pcie->managed_sids && pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EBUSY; + if (!pcie->hw->tunneled) { + if (pm_suspend_no_platform() && pcie->num_pds) + return apple_pcie_hold_domains(pcie); return 0; + } pcie->reset_on_resume = false; pcie->link_kept = apple_pcie_keep_link(pcie); if (!pcie->link_kept) @@ -4008,10 +4074,10 @@ static int apple_pcie_resume_noirq(struct device *dev) struct apple_pcie_port *port; int ret; + /* The domain's hold for the sleep ends first, whatever follows. */ + apple_pcie_release_domains(pcie); if (!pcie->hw->tunneled) return 0; - /* The domain's hold for the sleep ends first, whatever follows. */ - apple_pcie_release_domain(pcie); if (pcie->resume_failed) return -EIO; /* From 930c1893972475e19b6a7e98411ff5872f4b46ef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 21:15:15 +1100 Subject: [PATCH 175/392] arm64: dts: apple: complete the J616s and M2 internal PCIe power chains The internal host needs both the general-purpose and storage/PHY chains listed by the ADT clock gates. On J616s APCIE_SYS_GP reaches APCIE_GP, while APCIE_PHY_SW reaches both storage chains and ANS. On T8112 APCIE_ST adds the ANS/storage chain that NVMe happens to keep powered today. Keep the J616s interim always-on states until the host can cold-initialize its internal links after resume. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040-j616s.dts | 2 +- arch/arm64/boot/dts/apple/t8112.dtsi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index c263d1da558cc..4380094b8e071 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -305,5 +305,5 @@ }; &pcie0 { - power-domains = <&ps_apcie_sys_gp>; + power-domains = <&ps_apcie_sys_gp>, <&ps_apcie_phy_sw>; }; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index fbfe2a632e92a..e6e7b726e53c6 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -2560,7 +2560,7 @@ ranges = <0x43000000 0x6 0xa0000000 0x6 0xa0000000 0x0 0x20000000>, <0x02000000 0x0 0xc0000000 0x6 0xc0000000 0x0 0x40000000>; - power-domains = <&ps_apcie_gp>; + power-domains = <&ps_apcie_gp>, <&ps_apcie_st>; pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; From 5c96ab0a5f97ab0b1338f03a16d4df33e710c0c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 21:15:29 +1100 Subject: [PATCH 176/392] arm64: dts: apple: attach the Neo internal PCIe host to its full power chain The T8140 ADT gates APCIE_GP, APCIE_SYS_GP, APCIE_ST, APCIE_SYS_ST, ANS and APCIE_PHY_SW for apcie0. APCIE_PHY_SW already has all of the other domains as ancestors, so attach the host to that leaf instead of APCIE_GP alone. This removes its dependency on NVMe keeping the storage and PHY chain on. The host retains this chain during suspend-to-idle while the radio wake path leaves the internal link running. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b781dd756a2d2..4242e655a3af7 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1206,7 +1206,7 @@ #size-cells = <2>; ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; - power-domains = <&ps_apcie_gp>; + power-domains = <&ps_apcie_phy_sw>; status = "disabled"; pcie0_port0: pci@0,0 { From 9e86c161a811d528bac6a47c43b255ceeab4a502 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:00:56 +1100 Subject: [PATCH 177/392] rtc: apple-pmu: clear Maverick control event latch Maverick keeps a second fired-event latch in control bit 1, next to the event register. The arm path preserved that bit, so after one alarm had fired (or firmware had left the latch set) the alarm was armed as 0x43. The comparator still matched and cleared ARM, but no new event edge latched and the SPMI interrupt never fired. Clear the control event bit on every control write, so the alarm is armed as 0x41 and each match raises interrupt 2. On a J316s (M1 Pro) with this series, two awake alarms each raised the RTC interrupt once and cleared wakealarm, and six 15 s s2idle suspends woke on the RTC interrupt after 15.5-16.0 s. Extend the host model with Maverick's write-zero-to-clear control latch and its stale-edge suppression. Stowe keeps its exact control value checks, so the T8112 path is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/rtc/rtc-apple-pmu.c | 4 +++ tools/asahi/pmu-rtc-tests.py | 49 +++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index d324234cf807b..6043d7ee87a84 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -30,6 +30,7 @@ struct apple_pmu_rtc_hw { u16 mask; u16 event; u8 event_bit; + u8 control_event_bit; u8 monitor_bit; bool offset_in_seconds; }; @@ -73,6 +74,7 @@ static const struct apple_pmu_rtc_hw apple_pmu_rtc_maverick = { .mask = 0x14c2, .event = 0x14c0, .event_bit = BIT(0), + .control_event_bit = BIT(1), .monitor_bit = BIT(0), }; @@ -177,6 +179,8 @@ static int apple_pmu_rtc_arm(struct apple_pmu_rtc *rtc, bool enable) /* Do not acknowledge a shared W1C event as a side effect of RMW. */ if (rtc->hw->event == rtc->hw->control) control &= ~rtc->hw->event_bit; + /* Maverick clears its additional fired latch when this bit is zero. */ + control &= ~rtc->hw->control_event_bit; control &= ~APPLE_PMU_RTC_ARM; if (enable) control |= APPLE_PMU_RTC_ARM; diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 3c85238cc6343..469cacc4b3082 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -21,6 +21,8 @@ 'event-bit': ('.event_bit = BIT(1)', '.event_bit = BIT(0)'), 'wrong-control': ('.control = 0x2000', '.control = 0xf800'), 'maverick-event': ('.event = 0x14c0', '.event = 0x140c'), + 'control-event-clear': ('control &= ~rtc->hw->control_event_bit;', + 'control |= rtc->hw->control_event_bit;'), 'stowe-offset': ('static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {', 'static const struct apple_pmu_rtc_hw apple_pmu_rtc_stowe = {\n\t.offset_in_seconds = true,'), } @@ -62,7 +64,11 @@ struct rtc_device { int events; }; struct device { void *data; bool wake; int wake_events; }; struct nvmem_cell { u8 data[8]; size_t bytes; int error; }; -struct regmap { u8 bytes[65536]; unsigned ctrl, counter, compare, mask, event, bit; int ops, fail; }; +struct regmap { + u8 bytes[65536]; + unsigned ctrl, counter, compare, mask, event, bit, control_event; + int ops, fail; +}; static int disabled, pm_suspend_target_state = PM_SUSPEND_TO_IDLE; static inline __attribute__((unused)) s64 sign_extend64(u64 v, int bit) { return (s64)(v << (63 - bit)) >> (63 - bit); } static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v, p, 8); return v; } @@ -94,6 +100,10 @@ if (reg == m->event) { /* W1C event, normal R/W bits in the same byte. */ m->bytes[reg] = (v & ~m->bit) | (m->bytes[reg] & m->bit & ~v); + } else if (reg == m->ctrl && m->control_event) { + /* Maverick's control-side event latch is cleared by writing zero. */ + m->bytes[reg] = (v & ~m->control_event) | + (m->bytes[reg] & m->control_event & v); } else m->bytes[reg] = v; return 0; } @@ -113,6 +123,17 @@ assert(!(m->bytes[m->ctrl] & 0x40) && (m->bytes[m->mask] & 1)); memcpy(m->bytes + reg, v, n); return 0; } +static bool fire_alarm(struct regmap *m) { + assert(m->bytes[m->ctrl] & 0x40); + m->bytes[m->ctrl] &= ~0x40; + if (m->control_event) { + /* A stale control latch suppresses the next separate-event edge. */ + if (m->bytes[m->ctrl] & m->control_event) return false; + m->bytes[m->ctrl] |= m->control_event; + } + m->bytes[m->event] |= m->bit; + return true; +} ''' CASES = r''' @@ -137,7 +158,7 @@ [STOWE] = {.ctrl = 0xf800, .counter = 0xf802, .compare = 0xf808, .mask = 0xf80e, .event = 0xf80c, .bit = 1}, [MAVERICK] = {.ctrl = 0x1401, .counter = 0x1406, .compare = 0x140c, .mask = 0x14c2, - .event = 0x14c0, .bit = 1}, + .event = 0x14c0, .bit = 1, .control_event = 2}, }; static const struct apple_pmu_rtc_hw *hw[] = { [BAKU] = &apple_pmu_rtc_baku, [ABBEY] = &apple_pmu_rtc_abbey, @@ -149,8 +170,9 @@ rtc = (struct apple_pmu_rtc){.hw = hw[v], .map = &map, .offset = &offset, .clock = &clock_model}; dev = (struct device){.data = &rtc, .wake = true}; disabled = 0; - /* Keep an unrelated writable bit, and Abbey's monitor bit. */ + /* Keep an unrelated writable bit, and each variant's monitor bit. */ map.bytes[map.ctrl] = a ? 0x81 : 0x80; + if (v == MAVERICK) map.bytes[map.ctrl] = 0x03; set_counter(100ULL << 16); set_offset(a ? 1000LL * 32768 + 16384 : 1000); } @@ -161,20 +183,29 @@ struct rtc_wkalrm alarm = {.time.seconds = 1130, .enabled = true}, readback; s64 ticks; reset(v); + if (v == MAVERICK) { + /* A stale 0x03 latch suppresses a new edge until control bit 1 clears. */ + assert(map.bytes[map.ctrl] == 0x03 && !(map.bytes[map.event] & map.bit)); + map.bytes[map.ctrl] |= 0x40; + assert(!fire_alarm(&map)); + assert(map.bytes[map.ctrl] == 0x03 && !(map.bytes[map.event] & map.bit)); + } assert(!apple_pmu_rtc_read_time(&dev, &time) && time.seconds == 1100); set_counter((100ULL << 16) + 49152); assert(!apple_pmu_rtc_read_time(&dev, &time) && time.seconds == (a ? 1101 : 1100)); assert(!apple_pmu_rtc_set_alarm(&dev, &alarm)); assert(comparator() == 130 && (map.bytes[map.ctrl] & 0x40)); - assert(map.bytes[map.ctrl] == (a ? 0xc1 : 0xc0)); + assert(map.bytes[map.ctrl] == (v == MAVERICK ? 0x41 : (a ? 0xc1 : 0xc0))); assert(!(map.bytes[map.mask] & 1)); assert(!apple_pmu_rtc_read_alarm(&dev, &readback)); assert(readback.time.seconds == 1130 && readback.enabled && !readback.pending); /* Acknowledge without changing ARM, monitor or the unrelated bit. */ - map.bytes[map.event] |= map.bit; + assert(fire_alarm(&map)); + if (v == MAVERICK) assert(map.bytes[map.ctrl] == 0x03); assert(!apple_pmu_rtc_ack_alarm(&rtc)); - assert(map.bytes[map.ctrl] == (a ? 0xc1 : 0xc0)); + assert(map.bytes[map.ctrl] == (a ? (v == MAVERICK ? 0x03 : 0x81) : 0x80)); assert(!(map.bytes[map.event] & map.bit)); + assert(!apple_pmu_rtc_arm(&rtc, true)); /* An ordinary control write must not consume a latched event. */ map.bytes[map.event] |= map.bit; assert(!apple_pmu_rtc_arm(&rtc, false)); @@ -182,12 +213,12 @@ assert(!apple_pmu_rtc_read_alarm(&dev, &readback) && readback.pending); assert(apple_pmu_rtc_interrupt(7, &dev) == IRQ_HANDLED); assert(clock_model.events == 1 && dev.wake_events == 1); - assert(map.bytes[map.ctrl] == (a ? 0x81 : 0x80)); + assert(map.bytes[map.ctrl] == (v == MAVERICK ? 0x01 : (a ? 0x81 : 0x80))); assert(!(map.bytes[map.event] & map.bit) && (map.bytes[map.mask] & 1)); assert(apple_pmu_rtc_interrupt(7, &dev) == IRQ_NONE && clock_model.events == 1); /* Repeat arm/fire; no stuck status or stale IRQ after the first alarm. */ assert(!apple_pmu_rtc_alarm_enable(&dev, 1)); - map.bytes[map.event] |= map.bit; + assert(fire_alarm(&map)); dev.wake = false; assert(apple_pmu_rtc_interrupt(7, &dev) == IRQ_HANDLED); assert(clock_model.events == 2 && dev.wake_events == 1); @@ -218,7 +249,7 @@ assert(!apple_pmu_rtc_set_alarm(&dev, &alarm)); map.ops = 0; assert(!apple_pmu_rtc_init_alarm(&rtc) && (map.bytes[map.ctrl] & 0x40)); - set_counter(131ULL << 16); map.bytes[map.event] |= map.bit; + set_counter(131ULL << 16); assert(fire_alarm(&map)); assert(!apple_pmu_rtc_init_alarm(&rtc)); assert(!(map.bytes[map.ctrl] & 0x40) && !(map.bytes[map.event] & map.bit)); assert(map.bytes[map.mask] & 1); From f31c0a338df5bf085d4e9d6b1d8cdbc540a6c987 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 21:19:01 +1100 Subject: [PATCH 178/392] PCI: apple: preserve probe without power-management support The domain-attachment helper is a no-op without CONFIG_PM. Keep the host usable in that configuration instead of treating its zero return as a missing domain. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 409b95e68e1a9..7857378340110 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3115,6 +3115,8 @@ static int apple_pcie_attach_domains(struct apple_pcie *pcie) }; int count, ret, i; + if (!IS_ENABLED(CONFIG_PM)) + return 0; if (!of_property_present(dev->of_node, "power-domains")) return 0; From bf15de581c59ec264c8291183cfee28c279bea02 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:41:42 +1100 Subject: [PATCH 179/392] media: apple: scaler: support the T6000 and T8112 engines The fixed job words at 0x2e300-0x2e318 belong to the MSR20 generation (M4, M4 Pro/Max, A18 Pro). Older engines have a 0x28000 aperture, so writing them faults. Move those words to a table that only MSR20 variants emit, and refuse a job whose register list does not fit the aperture instead of writing past it. Add the T8112 (hardware-version 13) and T6000 (hardware-version 12) engines. Both have three register ranges and run the same jobs as the MSR20 engines without that control block. Tested with msrv4l2, which converts NV12, NV12M and P010 to ARGB32 and ARGB32 to NV12M at 1080p, 20 jobs each, against a software reference: M2 (T8112) PASS, max error 1, about 1.0 ms per job M1 Pro (T6000) PASS, max error 1, about 2.15 ms per job No DART faults; the engine runtime-suspends when idle. KUnit apple-scaler 14/14 passes. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../platform/apple/scaler/apple-scaler-drv.c | 30 +++++++++++++++++++ .../platform/apple/scaler/apple-scaler-hw.c | 18 +++++++++-- .../platform/apple/scaler/apple-scaler-hw.h | 1 + .../platform/apple/scaler/apple-scaler-test.c | 1 + 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/scaler/apple-scaler-drv.c b/drivers/media/platform/apple/scaler/apple-scaler-drv.c index 6a8d5f8438ce8..e65a7385407d0 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-drv.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-drv.c @@ -54,15 +54,18 @@ * struct apple_scaler_variant - per-SoC engine description * @hw_version: MSR hardware-version of this SoC (from the hardware description) * @num_reg: number of register apertures the binding describes + * @msr20: the engine has the MSR20-generation control block */ struct apple_scaler_variant { u32 hw_version; unsigned int num_reg; + bool msr20; }; struct apple_scaler { struct device *dev; void __iomem *regs; + resource_size_t regs_size; const struct apple_scaler_variant *variant; struct dev_pm_domain_list *pd_list; struct reset_control *rst; @@ -106,6 +109,7 @@ static void asc_write(struct apple_scaler *asc, u32 reg, u32 val) /* ---- hardware --------------------------------------------------------- */ + /* * After power-up or a failed job: wait for idle, pulse the engine reset * (no register may be read while it is held), clear stale status and @@ -293,12 +297,23 @@ static void asc_device_run(void *priv) spin_lock_irqsave(&asc->hw_lock, flags); /* Job tags as the engine is driven on macOS: a counter above bit 3, bit 2 set. */ job.transform_id = (++asc->transform_id << 3) | BIT(2); + job.msr20 = asc->variant->msr20; n = asc_job_build(&job, asc->writes, ARRAY_SIZE(asc->writes)); if (n < 0) { spin_unlock_irqrestore(&asc->hw_lock, flags); asc_finish(asc, ctx, n); return; } + /* The job list must fit this engine's register aperture. */ + for (int i = 0; i < n; i++) { + if (asc->writes[i].off + sizeof(u32) > asc->regs_size) { + spin_unlock_irqrestore(&asc->hw_lock, flags); + dev_err_ratelimited(asc->dev, "job register %#x is outside the aperture\n", + asc->writes[i].off); + asc_finish(asc, ctx, -EINVAL); + return; + } + } asc->active = ctx; /* Buffers must be visible to the engine before START, the last write. */ dma_wmb(); @@ -754,6 +769,7 @@ static int asc_probe(struct platform_device *pdev) asc->regs = devm_ioremap_resource(dev, res); if (IS_ERR(asc->regs)) return PTR_ERR(asc->regs); + asc->regs_size = resource_size(res); asc->rst = devm_reset_control_get_optional_exclusive(dev, NULL); if (IS_ERR(asc->rst)) @@ -814,17 +830,31 @@ static void asc_remove(struct platform_device *pdev) video_unregister_device(&asc->vdev); } +static const struct apple_scaler_variant apple_scaler_t6000 = { + .hw_version = 12, + .num_reg = 3, +}; + +static const struct apple_scaler_variant apple_scaler_t8112 = { + .hw_version = 13, + .num_reg = 3, +}; + static const struct apple_scaler_variant apple_scaler_t6041 = { .hw_version = 21, .num_reg = 4, + .msr20 = true, }; static const struct apple_scaler_variant apple_scaler_t8140 = { .hw_version = 22, .num_reg = 4, + .msr20 = true, }; static const struct of_device_id asc_of_match[] = { + { .compatible = "apple,t6000-scaler", .data = &apple_scaler_t6000 }, + { .compatible = "apple,t8112-scaler", .data = &apple_scaler_t8112 }, { .compatible = "apple,t6041-scaler", .data = &apple_scaler_t6041 }, { .compatible = "apple,t8140-scaler", .data = &apple_scaler_t8140 }, { } diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.c b/drivers/media/platform/apple/scaler/apple-scaler-hw.c index 5eb645a776470..ff6d7751135cc 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-hw.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.c @@ -375,9 +375,18 @@ static const struct asc_reg asc_job_fixed[] = { { 0x16084, 0x0b710367 }, { 0x16088, 0x00000128 }, { 0x16504, 0x02000500 }, { 0x16508, 0x00003200 }, { 0x1651c, 0x00851400 }, { 0x16534, 0x00000008 }, { 0x16568, 0x00250500 }, { 0x16588, 0x00496513 }, { 0x16600, 0x00000015 }, - { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, { 0x2e300, 0x03804038 }, - { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, { 0x2e30c, 0x02550255 }, - { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, { 0x2e318, 0x02550000 }, + { 0x16800, 0x00000000 }, { 0x25080, 0x00000000 }, +}; + +/* + * The MSR20 generation (M4, M4 Pro/Max, A18 Pro) has a control block at + * 0x2e300 beyond the 0x28000 aperture of older engines. Engines without + * it run the same jobs with only the common words above. + */ +static const struct asc_reg asc_job_fixed_msr20[] = { + { 0x2e300, 0x03804038 }, { 0x2e304, 0xa0a0a0a0 }, { 0x2e308, 0x02550000 }, + { 0x2e30c, 0x02550255 }, { 0x2e310, 0x00000000 }, { 0x2e314, 0x00100008 }, + { 0x2e318, 0x02550000 }, }; /** @@ -414,6 +423,9 @@ int asc_job_build(const struct asc_job *job, struct asc_reg *w, unsigned int max asc_emit_csc(&e, job->csc); for (i = 0; i < ARRAY_SIZE(asc_job_fixed); i++) asc_emit(&e, asc_job_fixed[i].off, asc_job_fixed[i].val); + if (job->msr20) + for (i = 0; i < ARRAY_SIZE(asc_job_fixed_msr20); i++) + asc_emit(&e, asc_job_fixed_msr20[i].off, asc_job_fixed_msr20[i].val); asc_emit(&e, 0x110, job->transform_id); asc_emit(&e, 0x098, 0xfffffffe); asc_emit(&e, 0x080, 1); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-hw.h b/drivers/media/platform/apple/scaler/apple-scaler-hw.h index cf371272dce66..b598fb08cd397 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-hw.h +++ b/drivers/media/platform/apple/scaler/apple-scaler-hw.h @@ -110,6 +110,7 @@ struct asc_job { struct asc_image dst; const struct asc_csc *csc; u32 transform_id; + bool msr20; /* engine has the MSR20-generation control block */ }; int asc_job_build(const struct asc_job *job, struct asc_reg *w, unsigned int max); diff --git a/drivers/media/platform/apple/scaler/apple-scaler-test.c b/drivers/media/platform/apple/scaler/apple-scaler-test.c index 4e61ad54e59a9..ddf8f633a469b 100644 --- a/drivers/media/platform/apple/scaler/apple-scaler-test.c +++ b/drivers/media/platform/apple/scaler/apple-scaler-test.c @@ -405,6 +405,7 @@ static void asc_test_job(struct kunit *test) job.src.fmt->yuv, &csc), 0); job.csc = &csc; job.transform_id = asc_expect_get(c, ASC_TRANSFORM_ID); + job.msr20 = true; /* traced on the J616s (MSR21) */ n = asc_job_build(&job, w, ASC_JOB_MAX_WRITES); KUNIT_ASSERT_GT(test, n, 0); From 133d87b4cd690ebb1c3d70e63d132d7ad4b36596 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:41:42 +1100 Subject: [PATCH 180/392] arm64: dts: apple: t8112: enable the scaler The scaler (MSR, hardware-version 13) and its DART work on the M2 with apple-scaler. Enable both for all T8112 boards. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-peripherals.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi index 1869b990d571f..f9cda024c5621 100644 --- a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi @@ -180,7 +180,6 @@ ; iommus = <&m2_dart_scaler 0>, <&m2_dart_scaler 1>; power-domains = <&ps_msr>, <&ps_msr_ase_core>; - status = "disabled"; }; m2_dart_scaler: iommu@263200000 { @@ -190,7 +189,6 @@ interrupts = ; power-domains = <&ps_msr>; #iommu-cells = <1>; - status = "disabled"; }; m2_jpeg0: jpeg@265000000 { From 7fc06c420370769c7fb7fa67e99f7b31d63da205 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:42:35 +1100 Subject: [PATCH 181/392] HID: magicmouse: hold the multitouch enable work across system sleep reset_resume re-arms the work that puts the trackpad back into multitouch mode 500 ms after resume. When the next suspend starts within that window, the work is still pending: its timer fires during the following suspend or resume, before the SPI HID transport is powered again, and the feature report fails with -ENODEV ("unable to request touch data (-19)"). The touch mode is then not restored after that resume. Cancel the work in suspend. reset_resume schedules it again as before; a plain resume, where the device kept its state, re-queues it only if suspend stopped it while it was pending. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/hid/hid-magicmouse.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index e909a698ac106..e7373d36b141b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -284,6 +284,7 @@ struct effect_job { * @haptic_deep_down: The host-derived click is in its deep stage. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. + * @work_held: @work was pending at suspend and must run again on resume. * @battery_timer: Timer for obtaining battery level information. * @input_ops: Input ops based on device type. */ @@ -324,6 +325,7 @@ struct magicmouse_sc { struct hid_device *hdev; struct delayed_work work; + bool work_held; struct timer_list battery_timer; struct magicmouse_input_ops input_ops; }; @@ -2553,12 +2555,34 @@ static void magicmouse_remove(struct hid_device *hdev) } #ifdef CONFIG_PM -static int magicmouse_resume(struct hid_device *hdev) +static int magicmouse_suspend(struct hid_device *hdev, pm_message_t message) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + /* + * The multitouch enable work sends a feature report. The transport + * powers down right after its children suspend, and the work's timer + * can fire while the system is still suspending or before the + * transport has resumed, so stop it here and re-arm it on resume. + */ if (msc) - magicmouse_haptic_forget_mode(msc); + msc->work_held = cancel_delayed_work_sync(&msc->work); + return 0; +} + +static int magicmouse_resume(struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + if (!msc) + return 0; + + /* The device kept its state, so only finish what suspend stopped. */ + if (msc->work_held) { + msc->work_held = false; + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + } + magicmouse_haptic_forget_mode(msc); return 0; } @@ -2570,6 +2594,8 @@ static int magicmouse_reset_resume(struct hid_device *hdev) * enable report. Only the HID_TYPE_USBMOUSE interface accepts it, and * it must be deferred. Sending it inline here is too early. */ + if (msc) + msc->work_held = false; if (msc && (hdev->type == HID_TYPE_USBMOUSE || hdev->bus == BUS_SPI)) schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); @@ -2648,6 +2674,7 @@ static struct hid_driver magicmouse_driver = { .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, #ifdef CONFIG_PM + .suspend = magicmouse_suspend, .resume = magicmouse_resume, .reset_resume = magicmouse_reset_resume, #endif From add3b8dda9149bf610eef72f7c8dc2724f628b02 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:47:36 +1100 Subject: [PATCH 182/392] arm64: dts: apple: t6000: enable the scaler The scaler (MSR, hardware-version 12) and its DART work on the M1 Pro with apple-scaler. Enable both on the T6000 (M1 Pro) boards. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000-peripherals.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi index 354ec7f161a77..8dbf14d92da28 100644 --- a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi @@ -135,7 +135,6 @@ clocks = <&peripheral_clocks 356>; iommus = <&scaler0_dart 0>, <&scaler0_dart 1>; resets = <&ps_msr0>; - status = "disabled"; }; scaler0_dart: iommu@40b200000 { @@ -145,7 +144,6 @@ interrupts = ; power-domains = <&ps_msr0>; #iommu-cells = <1>; - status = "disabled"; }; ave0_dma: dma-controller@40d020000 { From ec4467e80b4d2f6a616edde1baf322f071f5fab5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:07:06 +1100 Subject: [PATCH 183/392] EDAC: apple_soc: add the T8112 error handler The T8112 error handler has 19 interrupts: one AMCC aggregate line, two SEP lines (memory protection and clock/process monitor) and sixteen DCS lines (eight channels and their calibration partners). Claim the SEP lines and leave AMCC and DCS unclaimed, as on T6000 and T6041. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/edac/apple_soc_edac.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/edac/apple_soc_edac.c b/drivers/edac/apple_soc_edac.c index 0733a2712d4a4..a72e7f4dc351a 100644 --- a/drivers/edac/apple_soc_edac.c +++ b/drivers/edac/apple_soc_edac.c @@ -104,6 +104,15 @@ static const struct apple_soc_edac_range t6000_error_ranges[] = { { 72, 4, "nsr" }, }; +/* + * T8112: one AMCC aggregate line (0), two SEP lines (1-2) and sixteen DCS + * lines (3-18, eight channels plus their calibration partners). AMCC and + * DCS stay unclaimed, as on T6000 and T6041. + */ +static const struct apple_soc_edac_range t8112_error_ranges[] = { + { 1, 2, "sep" }, +}; + static const struct apple_soc_edac_range t8140_error_ranges[] = { { 4, 1, "pmgr" }, { 5, 9, "sep" }, @@ -130,6 +139,12 @@ static const struct apple_soc_edac_hw t6041_error_hw = { .num_status_regs = T6041_NI_REGS, }; +static const struct apple_soc_edac_hw t8112_error_hw = { + .ranges = t8112_error_ranges, + .nranges = ARRAY_SIZE(t8112_error_ranges), + .num_irqs = 19, +}; + static const struct apple_soc_edac_hw t8140_error_hw = { .ranges = t8140_error_ranges, .nranges = ARRAY_SIZE(t8140_error_ranges), @@ -390,6 +405,7 @@ static void apple_soc_edac_remove(struct platform_device *pdev) static const struct of_device_id apple_soc_edac_of_match[] = { { .compatible = "apple,t6000-error-handler", .data = &t6000_error_hw }, { .compatible = "apple,t6041-error-handler", .data = &t6041_error_hw }, + { .compatible = "apple,t8112-error-handler", .data = &t8112_error_hw }, { .compatible = "apple,t8140-error-handler", .data = &t8140_error_hw }, { .compatible = "apple,t6000-event-log", .data = &t6000_event_hw }, { .compatible = "apple,t6041-event-log", .data = &t6041_event_hw }, From 3a3b903ef8fcc7660621a19f260c63c4f2025ace Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:07:06 +1100 Subject: [PATCH 184/392] arm64: dts: apple: t8112: enable the error handler apple_soc_edac now matches the T8112 error handler. Enable it. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-peripherals.dtsi | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi index 1869b990d571f..8c46aa005ddad 100644 --- a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi @@ -42,7 +42,6 @@ , , ; - status = "disabled"; }; m2_mcc: memory-controller@200100000 { From 12cf63fc46ce01cbe6b32db427c4a6a274c6a61a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:07:33 +1100 Subject: [PATCH 185/392] tools/asahi: model the magicmouse suspend callback in mtp-pm-tests The MTP PM model now runs the production magicmouse_suspend() from the transport's suspend notification. It checks that pending multitouch enable work is cancelled at suspend and queued once on resume, and that idle work stays idle. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- tools/asahi/mtp-pm-tests.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py index 7d5b84d662ef0..4f4d183021905 100644 --- a/tools/asahi/mtp-pm-tests.py +++ b/tools/asahi/mtp-pm-tests.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only -"""Run production MTP PM and magicmouse resume callbacks without a device. +"""Run production MTP PM and magicmouse suspend/resume callbacks without a device. -Checks notification order, suspend rollback, resume-error handling and the -actuator mode cache. The old transport must compile and fail at runtime. +Checks notification order, suspend rollback, resume-error handling, the +actuator mode cache and the multitouch enable work held across sleep. The old transport must compile and fail at runtime. """ import argparse import os @@ -37,7 +37,13 @@ def function(source, name): #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x) = (v)) struct haptics { int mode; }; -struct magicmouse_sc { struct haptics *haptics; bool haptic_button_down, haptic_deep_down; }; +struct delayed_work { bool pending; unsigned queued; }; +struct magicmouse_sc { struct haptics *haptics; bool haptic_button_down, haptic_deep_down; + struct delayed_work work; bool work_held; }; +typedef int pm_message_t; +static unsigned long msecs_to_jiffies(unsigned m) { return m; } +static bool cancel_delayed_work_sync(struct delayed_work *w) { bool p = w->pending; w->pending = false; return p; } +static bool schedule_delayed_work(struct delayed_work *w, unsigned long d) { w->queued++; w->pending = true; return true; } struct hid_device { struct magicmouse_sc *data; int id; }; struct dchid_iface { struct hid_device *hid; }; struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; @@ -53,6 +59,8 @@ def function(source, name): static int hid_driver_suspend(struct hid_device *h, int state) { assert(state == PMSG_SUSPEND && !transport->suspended); suspends[h->id]++; + if (h->id != suspend_error_at) + magicmouse_suspend(h, state); return h->id == suspend_error_at ? -EBUSY : 0; } ''' @@ -97,7 +105,14 @@ def function(source, name): assert(!dockchannel_hid_resume(&d)); assert(hap.mode == HID_HAPTIC_MODE_DEVICE); } - puts("MTP PM: HID notifications, rollback, all-interface resume and haptic cache passed"); + /* Pending multitouch enable work is held across sleep and re-queued once. */ + m.work.pending = true; m.work.queued = 0; + assert(!dockchannel_hid_suspend(&d) && !m.work.pending && m.work_held); + assert(!dockchannel_hid_resume(&d) && m.work.pending && !m.work_held && m.work.queued == 1); + m.work.pending = false; + assert(!dockchannel_hid_suspend(&d) && !m.work_held); + assert(!dockchannel_hid_resume(&d) && !m.work.pending && m.work.queued == 1); + puts("MTP PM: HID notifications, rollback, all-interface resume, haptic cache and held enable work passed"); } ''' @@ -115,7 +130,8 @@ def main(): out = Path(args.out or tmp) out.mkdir(parents=True, exist_ok=True) c, exe = out/'mtp-pm.c', out/'mtp-pm' - c.write_text(PREFIX + function(mouse, 'magicmouse_haptic_forget_mode') + '\n' + + c.write_text(PREFIX.replace('static int hid_driver_suspend', function(mouse, 'magicmouse_suspend') + '\nstatic int hid_driver_suspend', 1) + + function(mouse, 'magicmouse_haptic_forget_mode') + '\n' + function(mouse, 'magicmouse_resume') + MIDDLE + function(source, 'dockchannel_hid_suspend') + '\n' + function(source, 'dockchannel_hid_resume') + CASES) From 09458b01af1c31919327cad070f17810ade588c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:19:01 +1100 Subject: [PATCH 186/392] thunderbolt: Unregister XDomains when the domain is removed Since XDomains are removed from the bus without holding tb->lock, tb_xdomain_remove() only takes an XDomain out of the topology; the caller unregisters it later with tb_domain_unregister_unplugged_xdomains(). tb_domain_remove() never does that. Stopping the connection manager removes the root switch, which marks every XDomain unplugged and calls tb_xdomain_remove(), but the XDomain stays registered on the bus and its service drivers stay bound. When the domain is then freed, a bound thunderbolt-net keeps its protocol handler on the global list with xd->tb pointing at the freed domain. A host router driver that removes and re-adds its domain when the link restarts (the Apple ACIO driver does so when the partner reboots) then gets a new domain with the same index: adding the new XDomain fails with a duplicate "0-1" sysfs name, and the stale thunderbolt-net answers the partner's ThunderboltIP login, which now arrives on the new control channel, through the freed one: Unable to handle kernel paging request at virtual address ... pc : queued_spin_lock_slowpath+0x18c/0x5d0 Call trace: queued_spin_lock_slowpath+0x18c/0x5d0 (P) _raw_spin_lock_irqsave+0x9c/0xa8 __mutex_unlock_slowpath+0x90/0x338 mutex_unlock+0x34/0x70 tb_cfg_request+0x218/0x250 [thunderbolt] tb_xdomain_response+0x74/0xb0 [thunderbolt] tbnet_login_response+0x9c/0x238 [thunderbolt_net] tbnet_handle_packet+0xd0/0x358 [thunderbolt_net] tb_xdomain_handle_request+0xf0/0x1f0 [thunderbolt] tb_domain_event_cb+0x60/0xa8 [thunderbolt] tb_ctl_handle_event+0x38/0x170 [thunderbolt] tb_ctl_rx_callback+0x1b8/0x528 [thunderbolt] ring_work+0x138/0x280 [thunderbolt] Unregister the unplugged XDomains in tb_domain_remove() once tb->lock is dropped and the domain workqueue is drained, before the connection manager is deinitialized and the domain device goes away. The control channel is already stopped at that point, so requests the service drivers send while unbinding (thunderbolt-net's logout) fail with -ENOTCONN instead of waiting for a timeout. Fixes: a8937f35cf39 ("thunderbolt: Remove XDomain from the bus without holding tb->lock") Closes: https://github.com/aurora-silicon/linux/issues/1318 Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/thunderbolt/domain.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 56ffea80c5981..72b227997fab7 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -510,6 +510,17 @@ void tb_domain_remove(struct tb *tb) flush_workqueue(tb->wq); + /* + * Stopping the connection manager removed the XDomains from the + * topology and marked them unplugged, but they are still on the bus + * with their service drivers bound. Unregister them now: a service + * driver keeps its protocol handler registered until it is unbound + * and would otherwise keep answering XDomain requests through + * tb->ctl after the domain is gone. The control channel is stopped, + * so whatever the service drivers send while unbinding fails at once. + */ + tb_domain_unregister_unplugged_xdomains(tb); + if (tb->cm_ops->deinit) tb->cm_ops->deinit(tb); From 758521bd2d2a9b5e635e7d3ec2eee052a0934c72 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:19:23 +1100 Subject: [PATCH 187/392] net: thunderbolt: Unregister the protocol handler before the netdev tbnet_remove() unregisters the network device first and the ThunderboltIP protocol handler after it. In between, a login or logout packet from the remote host still reaches tbnet_handle_packet(), which can queue login, connect or disconnect work for a device that is being torn down. A logout queues the disconnect work without checking whether the interface is up, and nothing cancels it after tbnet_stop() has run, so it can run after free_netdev(). Take the handler off the list first, then unregister the netdev, and cancel any disconnect work that is still pending before freeing it. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/thunderbolt/main.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index de56fc2722994..8f3a05e07d717 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -1521,8 +1521,16 @@ static void tbnet_remove(struct tb_service *svc) { struct tbnet *net = tb_service_get_drvdata(svc); - unregister_netdev(net->dev); + /* + * Take the protocol handler off the list before the interface goes + * down, so that ThunderboltIP packets arriving during the teardown + * no longer reach this device and queue new login, connect or + * disconnect work for it. + */ tb_unregister_protocol_handler(&net->handler); + unregister_netdev(net->dev); + /* A logout received while the interface was down queued this. */ + cancel_work_sync(&net->disconnect_work); free_netdev(net->dev); } From c0da15fe572c1b442568dab3c2ed91069264bb89 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:28:10 +1100 Subject: [PATCH 188/392] arm64: dts: apple: t8140: enable the scaler The scaler (MSR, hardware-version 22) and its DART work on the J700 with apple-scaler. Enable both. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-peripherals.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index 3d5bfc2d9e2a5..f73a86c0f4326 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -161,7 +161,6 @@ <&scaler0_dart 1>; clocks = <&peripheral_clocks 374>; resets = <&ps_msr>; - status = "disabled"; }; /* ADT /arm-io/aop2. */ @@ -428,7 +427,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_msr>; - status = "disabled"; }; /* ADT /arm-io/dart-rsm. */ From 5095d596083e6539f2e957273d0738432fd6fef8 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:30:27 +1100 Subject: [PATCH 189/392] dt-bindings: nfc: apple,stockholm-spmi: name the known interrupts, add VEN Three of the seven endpoint interrupts have a known role: the transport error (3), host wake (5) and receive data ready (6). Name all seven so that a driver can request them by name, and describe the VEN enable line. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../bindings/net/nfc/apple,stockholm-spmi.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/net/nfc/apple,stockholm-spmi.yaml b/Documentation/devicetree/bindings/net/nfc/apple,stockholm-spmi.yaml index 8a60fd5c95946..9ef7dab4092bf 100644 --- a/Documentation/devicetree/bindings/net/nfc/apple,stockholm-spmi.yaml +++ b/Documentation/devicetree/bindings/net/nfc/apple,stockholm-spmi.yaml @@ -11,8 +11,8 @@ maintainers: description: | The endpoint occupies an SPMI follower address and exports seven event - interrupts. Event roles and the firmware-controlled enable sequence are - unqualified. This description does not select the later NCI transport. + interrupts, of which three have a known role: the transport error, host + wake and receive data ready. VEN enables the controller. properties: compatible: @@ -30,6 +30,20 @@ properties: minItems: 7 maxItems: 7 + interrupt-names: + items: + - const: event0 + - const: event1 + - const: event2 + - const: error + - const: event4 + - const: host-wake + - const: data-ready + + enable-gpios: + maxItems: 1 + description: VEN, the controller enable line. + required: - compatible - reg From dc53956611b6bd68571979f4c6525b709eff9c61 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:30:28 +1100 Subject: [PATCH 190/392] nfc: apple-stockholm: support the M1/M2-generation controller The T6000 and T8112 Stockholm follower uses the same register map as the model-96 part, with follower ID 0x1c and IRQ base 0xa0. Its commands are not acknowledged until about 10 ms after VEN rises. Add it as a second model for the apple,t6000- and apple,t8112-stockholm-spmi compatibles. The T8112 AOP SPMI controller has no AP interrupt, so neither do its followers. In that case check that the follower answers, then park VEN low. Tested on the M2: "follower 0x1c present, no interrupt path; parked off" on every load. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/nfc/apple-stockholm.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/drivers/nfc/apple-stockholm.c b/drivers/nfc/apple-stockholm.c index d4810655bfc81..65731cf9b289a 100644 --- a/drivers/nfc/apple-stockholm.c +++ b/drivers/nfc/apple-stockholm.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -64,6 +65,20 @@ static const struct stockholm_model stockholm_model96 = { .startup_delay_us = 10000, }; +/* M1/M2 generation (T6000, T8112): the legacy follower, same map. */ +static const struct stockholm_model stockholm_model_b2 = { + .tx_reg = 0x1000, + .rx_reg = 0x80, + .id_reg = 0x100, + .irq_reg = 0x105, + .standby_reg = 0x0, + .follower_id = 0x1c, + .irq_base = 0xa0, + .standby_off = 0x14, + /* Commands are not acknowledged until about 10 ms after VEN rises. */ + .startup_delay_us = 10000, +}; + struct stockholm { struct spmi_device *sdev; const struct stockholm_model *model; @@ -408,6 +423,24 @@ static int stockholm_probe(struct spmi_device *sdev) return dev_err_probe(dev, PTR_ERR(st->enable_gpio), "failed to get VEN GPIO\n"); + /* + * The T8112 AOP SPMI controller has no AP interrupt, so its followers + * have none either. Check that the controller answers and park it. + */ + if (!of_property_present(sdev->ctrl->dev.of_node, "interrupts")) { + ret = stockholm_power_on(st); + if (ret) + return dev_err_probe(dev, ret, "failed to raise VEN\n"); + ret = stockholm_check_follower(st); + err = stockholm_park(st); + if (err) + return dev_err_probe(dev, err, "failed to park VEN low\n"); + if (!ret) + dev_info(dev, "follower %#04x present, no interrupt path; parked off\n", + st->model->follower_id); + return ret; + } + st->data_irq = stockholm_request_irq(st, "data-ready", stockholm_data_irq); if (st->data_irq < 0) @@ -442,6 +475,8 @@ static int stockholm_probe(struct spmi_device *sdev) static const struct of_device_id stockholm_of_match[] = { { .compatible = "apple,stockholm-nci", .data = &stockholm_model96 }, + { .compatible = "apple,t6000-stockholm-spmi", .data = &stockholm_model_b2 }, + { .compatible = "apple,t8112-stockholm-spmi", .data = &stockholm_model_b2 }, {} }; MODULE_DEVICE_TABLE(of, stockholm_of_match); From f336903141a3ab3905ee2c4223f1d0b7f8e1cb7e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:30:28 +1100 Subject: [PATCH 191/392] arm64: dts: apple: t8112-j413: enable the NFC controller Enable the AOP SPMI bus and the Stockholm NFC controller on it, with VEN on SMC GPIO 15. apple-stockholm finds the controller and keeps it off. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-j413.dts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 9c9e7532ae048..1914dd2361bfd 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -357,9 +357,15 @@ }; }; +/* NFC controller (Stockholm): identified once, then kept off. */ +&smc_gpio { + apple,output-templates = <0xf 0x00800000>; +}; + &aop_spmi1 { interrupt-controller; #interrupt-cells = <2>; + status = "okay"; nfc@c { compatible = "apple,t8112-stockholm-spmi", "apple,stockholm-spmi"; @@ -372,7 +378,9 @@ <164 IRQ_TYPE_NONE>, <165 IRQ_TYPE_NONE>, <166 IRQ_TYPE_NONE>; - status = "disabled"; + interrupt-names = "event0", "event1", "event2", "error", + "event4", "host-wake", "data-ready"; + enable-gpios = <&smc_gpio 15 GPIO_ACTIVE_HIGH>; }; }; From 4a7c0cffef4578ae258b77f165adeb512a6bd365 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:35:19 +1100 Subject: [PATCH 192/392] thunderbolt: Refuse XDomain DMA paths once the XDomain is unplugged tb_approve_xdomain_paths() looks up the destination and NHI ports before it takes tb->lock. A service driver can still ask for paths while the domain is being stopped: thunderbolt-net enables them from its connected work, which a ThunderboltIP login queued just before the control channel was stopped. By then tb_stop() may have removed the routers and cleared tb->root_switch, so tb_switch_find_port() follows a NULL pointer, and the XDomain's parent router is a removed switch. Take tb->lock first and return -ENODEV when the XDomain is unplugged or the root switch is gone. tb_disconnect_xdomain_paths() already skips an unplugged XDomain the same way. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/thunderbolt/tb.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index a4d439f2394df..19c1389c63d87 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3161,12 +3161,22 @@ static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, struct tb_switch *sw; int ret; + mutex_lock(&tb->lock); + + /* + * A service driver can still ask for paths after the XDomain was + * unplugged or while the domain is being stopped, when the router + * may already be gone and tb->root_switch is NULL. + */ + if (xd->is_unplugged || !tb->root_switch) { + mutex_unlock(&tb->lock); + return -ENODEV; + } + sw = tb_to_switch(xd->dev.parent); dst_port = tb_port_at(xd->route, sw); nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); - mutex_lock(&tb->lock); - /* * When tunneling DMA paths the link should not enter CL states * so disable them now. From 653f5375f4d33aba76c3636775060e49e5c3816c Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:24:07 -0800 Subject: [PATCH 193/392] dt-bindings: pci: apple,pcie: Add T6050 hosts adopted from the boot loader Add apple,t6050-pcie, a T6050 (M5 Ultra) PCIe host whose root ports the boot loader powered, trained and left running. Each such port is marked apple,link-trained-by-bootloader and named by its port, phy and intr2axi register ranges and its port interrupt; the host's local MSI map starts at apple,msi-vector-base. The OS adopts the trained state and never resets, retrains or changes the port's power: reset-gpios, pwren-gpios and power-domains are not allowed on it. Also name nonposted-mmio, which these hosts take. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 55 +++++++++++++++++-- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 245231ecfc3a9..5823cc0ff837a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -98,11 +98,11 @@ properties: reg: minItems: 3 - maxItems: 11 + maxItems: 14 reg-names: minItems: 3 - maxItems: 11 + maxItems: 14 ranges: minItems: 2 @@ -128,8 +128,7 @@ properties: apple,msi-vector-base: $ref: /schemas/types.yaml#/definitions/uint32 description: - First vector of this PCIe-C controller's slice of the MSI window it - shares with the internal PCIe controller. + First global vector targeted by this controller's local MSI map. iommu-map: true iommu-map-mask: true @@ -234,6 +233,48 @@ allOf: - apple,msi-vector-base - $ref: /schemas/pci/pci-host-bridge.yaml# - $ref: /schemas/interrupt-controller/msi-controller.yaml# + - if: + properties: + compatible: + contains: + const: apple,t6050-pcie + then: + properties: + reg: + minItems: 5 + maxItems: 14 + reg-names: + description: + config and rc, then port, phy and intr2axi for each + bootloader-trained root port 00:.0. + minItems: 5 + maxItems: 14 + items: + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + interrupts: + maxItems: 4 + interrupt-names: + items: + pattern: "^port[0-3]$" + apple,msi-vector-base: + const: 64 + power-domains: false + required: + - interrupt-names + - apple,msi-vector-base + - iommu-map + patternProperties: + "^pci@": + properties: + reset-gpios: false + pwren-gpios: false + required: + - apple,link-trained-by-bootloader + else: + patternProperties: + "^pci@": + properties: + apple,link-trained-by-bootloader: false - if: properties: compatible: @@ -390,10 +431,10 @@ allOf: properties: reg: minItems: 5 - maxItems: 11 + maxItems: 14 reg-names: minItems: 5 - maxItems: 11 + maxItems: 14 items: enum: [config, rc, port0, phy0, intr2axi0, port1, phy1, intr2axi1, port2, phy2, intr2axi2, port3, phy3, @@ -420,6 +461,8 @@ allOf: maxItems: 3 else: properties: + reg: + maxItems: 10 reg-names: minItems: 3 items: From ca6cac821eb3605ae935f1277f66f4ce5d668d72 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:25:06 -0800 Subject: [PATCH 194/392] PCI: apple: Adopt T6050 root ports the boot loader trained On the M5 Ultra (T6050, J775D) running Linux on the hardware, the boot loader brings up the internal PCIe host's ports itself: it powers the shared PHY, trains each port's link and leaves it up, with the endpoint idle. Port 2 carries the onboard RTL8127 Ethernet controller and port 3 the ASM3142 xHCI behind the rear USB-A ports. Repeating the reset and training sequence from Linux is not needed and not known to be safe on this SoC. Add apple,t6050-pcie, a host whose root ports marked apple,link-trained-by-bootloader are adopted as they are: no reset, no tunable replay, no clock or power change and no retraining. Every resource comes from the DT by the port's index (port, phy, intr2axi, interrupt port, the iommu-map entry for the endpoint's RID). Before the first PCIe register write the driver checks that the link is up and not retraining at Gen3 x1, that the boot loader numbered the root port's buses (primary 0, one secondary), that the endpoint on it is device 0 function 0 with DMA and MSI off and a well-formed capability list, and that the port's RID table holds no valid entry. A port that fails a check is left alone and the others are still adopted. Then the endpoint's RID is routed to its SID, the host's MSI map is set up from apple,msi-vector-base, and only the adopted root ports and the endpoint behind each are enumerated. The links cannot be recovered by a user reset or removal, since the boot loader owns them. No other SoC sets adopt_link. Testing: on J775D, every boot adopts port 2 (RTL8127, DHCP and SSH over it on every boot) and port 3 (ASM3142, a USB keyboard and mouse driving the desktop session). Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 283 ++++++++++++++++++++++++++-- 1 file changed, 270 insertions(+), 13 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 6ad3376dcf75e..b0a4a549cdd9e 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -207,6 +207,7 @@ struct hw_info { u32 pwren_on_ms; bool sid_indexed_rid2sid; bool retain_clocks; + bool adopt_link; bool root_bus_only; bool tunneled; bool tunnel_intr2axi; @@ -281,6 +282,24 @@ static const struct hw_info t8152_hw = { .pipe_rst = true, }; +/* + * Bare-metal T6050 adoption only. Firmware owns power, PHY, reset and training. + * The 64-entry route table is never probed with test writes. + */ +static const struct hw_info t6050_hw = { + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 64, + .retain_clocks = true, + .adopt_link = true, +}; + +#define T6050_ROUTE(rid, sid) (PORT_RID2SID_VALID | \ + ((sid) << PORT_RID2SID_SID_SHIFT) | (rid)) + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -456,6 +475,10 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; + bool adopted_route; + void __iomem *intr2axi; /* adopt_link: this port's */ + u16 adopted_rid; /* adopt_link: secondary bus << 8 */ + u32 adopted_sid; }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -767,9 +790,20 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 val = 0; - /* FIXME: consider moving each interrupt under each port */ - port->irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), - port->idx); + if (pcie->hw->adopt_link) { + char name[16]; + int irq; + + snprintf(name, sizeof(name), "port%d", port->idx); + irq = of_irq_get_byname(pcie->dev->of_node, name); + + if (irq <= 0) + return irq ?: -ENXIO; + port->irq = irq; + } else { + /* FIXME: consider moving each interrupt under each port */ + port->irq = irq_of_parse_and_map(pcie->dev->of_node, port->idx); + } if (!port->irq) return -ENXIO; @@ -782,6 +816,9 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) return -ENOMEM; } + if (pcie->hw->adopt_link) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + /* Disable all interrupts */ apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); @@ -815,7 +852,22 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } + if (pcie->hw->adopt_link) { + if (apple_pcie_port_readl(port, pcie->hw->port_msiaddr) != DOORBELL_ADDR || + apple_pcie_port_readl(port, pcie->hw->port_msiaddr_hi)) + return -EIO; + for (int i = 0; i < pcie->nvecs; i++) + if (apple_pcie_port_readl(port, pcie->hw->port_msimap + 4 * i) != + (PORT_MSIMAP_ENABLE | (pcie->msi_vector_base + i))) + return -EIO; + /* Enable command is not readable; +0x84 is only a completion read. */ + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + } apple_pcie_port_writel(port, val | PORT_MSICFG_EN, PORT_MSICFG); + if (pcie->hw->adopt_link && + !(apple_pcie_port_readl(port, PORT_MSICFG) & PORT_MSICFG_EN)) + return -EIO; return 0; } @@ -831,6 +883,8 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + if (port->pcie->hw->adopt_link) + WRITE_ONCE(port->link_failed, true); /* A later link-up alone does not prove downstream state survived. */ if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { WRITE_ONCE(port->link_failed, true); @@ -1509,9 +1563,18 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) apple_pcie_port_disable_irq(port); /* Busy managed admission has not acquired or programmed this IRQ block. */ - if (port->base && (!port->pcie->managed_sids || port->domain)) + if (port->base && + (port->domain || + (!port->pcie->managed_sids && !port->pcie->hw->adopt_link))) apple_pcie_port_writel(port, ~0, PORT_INTMSK); + if (port->pcie->hw->adopt_link && port->domain) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + if (port->adopted_route) { + apple_pcie_rid2sid_write(port, port->adopted_sid, 0); + port->adopted_route = false; + } + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); @@ -1595,6 +1658,134 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * Qualified J775D handoff of one bootloader-trained port: its root port + * 00:.0 and the single endpoint on its secondary bus, with DMA and + * endpoint MSI disabled. Every resource comes from the DT by the port's index + * (port, phy, intr2axi, interrupt port, the iommu-map entry for the + * endpoint's RID). All checks precede the first PCIe register write. + */ +static int apple_pcie_adopt_link(struct apple_pcie_port *port) +{ + static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; + struct apple_pcie *pcie = port->pcie; + struct platform_device *pdev = to_platform_device(pcie->dev); + struct of_phandle_args iommu = {}; + void __iomem *rp, *ep; + char name[16]; + u32 value, bus; + u8 cap; + bool msi = false; + int ret, i; + + if (pcie->cfg->busr.start != 0 || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "config") || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc")) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + snprintf(name, sizeof(name), names[i], port->idx); + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) + return -EINVAL; + } + + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN) || + !(readl(pcie->base + CORE_RC_STAT) & CORE_RC_STAT_READY) || + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY) || + (apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP || + !(apple_pcie_port_readl(port, PORT_LTSSMCTL) & PORT_LTSSMCTL_START) || + (apple_pcie_port_readl(port, PORT_APPCLK) & 0x101) != 0x101 || + (apple_pcie_port_readl(port, pcie->hw->port_perst) & 0x10001) != 1 || + (readl(port->phy + PHY_LANE_CFG) & 0x1000061f) != 0x60f) + return -ENOLINK; + + /* The bootloader numbered the root port's buses: primary 0, one secondary. */ + rp = pcie->cfg->win + (PCI_DEVFN(port->idx, 0) << 12); + value = readl(rp + PCI_PRIMARY_BUS); + bus = (value >> 8) & 0xff; + if (readl(rp) != 0x100c106b || (value & 0xff) != 0 || !bus || + ((value >> 16) & 0xff) != bus || bus > pcie->cfg->busr.end || + (readw(rp + PCI_BRIDGE_CONTROL) & PCI_BRIDGE_CTL_BUS_RESET)) + return -ENODEV; + ep = pcie->cfg->win + (bus << 20); + value = readl(ep); + if (value == 0xffffffff || !(value & 0xffff) || + (readb(ep + PCI_HEADER_TYPE) & 0x7f) != PCI_HEADER_TYPE_NORMAL || + (readw(ep + PCI_COMMAND) & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | + PCI_COMMAND_MASTER))) + return -ENODEV; + + port->adopted_rid = bus << 8; + ret = of_map_iommu_id(pcie->dev->of_node, port->adopted_rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + port->adopted_sid = iommu.args[0]; + of_node_put(iommu.np); + if (ret) + return ret; + + snprintf(name, sizeof(name), "intr2axi%d", port->idx); + port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->intr2axi)) + return PTR_ERR(port->intr2axi); + + /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ + cap = readb(rp + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + if (readb(rp + cap + PCI_CAP_LIST_ID) == PCI_CAP_ID_EXP) + break; + cap = readb(rp + cap + PCI_CAP_LIST_NEXT); + } + if (!cap || i == 48) + return -EINVAL; + value = readw(rp + cap + PCI_EXP_LNKSTA); + if ((value & (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != + (PCI_EXP_LNKSTA_CLS_8_0GB | (1 << PCI_EXP_LNKSTA_NLW_SHIFT))) + return -ENOLINK; + + /* Refuse an active endpoint or malformed capability chain. */ + cap = readb(ep + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + value = readw(ep + cap + PCI_MSI_FLAGS); + switch (readb(ep + cap + PCI_CAP_LIST_ID)) { + case PCI_CAP_ID_MSI: + if (value & PCI_MSI_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + case PCI_CAP_ID_MSIX: + if (value & PCI_MSIX_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + } + cap = readb(ep + cap + PCI_CAP_LIST_NEXT); + } + if (cap || !msi) + return -EINVAL; + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + if (apple_pcie_port_readl(port, port_rid2sid_offset(port, i)) & + PORT_RID2SID_VALID) + return -EBUSY; + + /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ + value = T6050_ROUTE(port->adopted_rid, port->adopted_sid); + port->adopted_route = true; + if (apple_pcie_rid2sid_write(port, port->adopted_sid, value) != value) + return -EIO; + dev_info(pcie->dev, "adopted port %d, RID %04x -> SID %u, slot %u\n", + port->idx, port->adopted_rid, port->adopted_sid, port->adopted_sid); + return 0; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -1881,6 +2072,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } } + } else if (pcie->hw->adopt_link) { + ret = apple_pcie_adopt_link(port); + if (ret) + goto err_teardown; } else { if (pcie->hw->retain_clocks) rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); @@ -1922,9 +2117,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; - if (pcie->managed_sids) { + if (pcie->managed_sids || pcie->hw->adopt_link) goto rid2sid_ready; - } /* Reset all RID/SID mappings, and check for RAZ/WI registers */ for (i = 0; i < pcie->hw->max_rid2sid; i++) { @@ -1949,6 +2143,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, unsigned long timeout, left = 0; int attempt, attempts = 1; + if (pcie->hw->adopt_link) { + ret = -ENOLINK; + goto err_teardown; + } if (pcie->hw->tunneled) { if (pcie->kernel_init) { apple_pcie_tunnel_apply_tunable(port->base, @@ -2095,6 +2293,17 @@ static int apple_msi_init(struct apple_pcie *pcie) return ret; } + if (pcie->hw->adopt_link) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + if (ret || pcie->msi_vector_base != 64 || + (pcie->nvecs != 32 && pcie->nvecs != 64) || + args.args_count != 4 || args.args[0] != 0 || args.args[1] != 0 || + args.args[2] != 1888 || args.args[3] != IRQ_TYPE_EDGE_RISING || + DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + /* * The controller-wide ADT count (512) is not a per-host allocation. * Internal PCIe owns vectors 0..31. Each PCIe-C bridge owns a distinct @@ -2218,7 +2427,7 @@ static int apple_msi_init(struct apple_pcie *pcie) invalid_slice: of_node_put(args.np); return dev_err_probe(pcie->dev, -EINVAL, - "invalid or missing PCIe-C MSI slice\n"); + "invalid or missing MSI slice\n"); } static void apple_pcie_cleanup(void *data) @@ -2419,6 +2628,11 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; + if (pcie->hw->adopt_link) + return rid == port->adopted_rid && sid == port->adopted_sid && + !READ_ONCE(port->link_failed) && + apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)) == + T6050_ROUTE(rid, sid) ? 0 : -ENODEV; mutex_lock(&port->pcie->lock); if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { @@ -2485,7 +2699,7 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci int idx; port = apple_pcie_get_port(pdev); - if (!port) + if (!port || port->pcie->hw->adopt_link) return; mutex_lock(&port->pcie->lock); @@ -2526,9 +2740,13 @@ static int apple_pcie_init(struct pci_config_window *cfg) ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); - return ret; + /* Adopted ports are independent: one refused leaves the others. */ + if (!pcie->hw->adopt_link) + return ret; } } + if (pcie->hw->adopt_link && list_empty(&pcie->ports)) + return -ENODEV; /* The bus scan follows this callback. */ apple_pcie_tunnel_wait_ready(pcie); @@ -2548,6 +2766,27 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); struct apple_pcie_port *port; + if (pcie->hw->adopt_link) { + /* Only adopted ports' root ports, and the one endpoint behind each. */ + list_for_each_entry(port, &pcie->ports, entry) { + if (bus->number == 0) { + if (devfn == PCI_DEVFN(port->idx, 0)) + return pci_ecam_map_bus(bus, devfn, where); + continue; + } + if (bus->number != port->adopted_rid >> 8) + continue; + if (devfn || READ_ONCE(port->link_failed)) + return NULL; + if ((apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { + WRITE_ONCE(port->link_failed, true); + return NULL; + } + return pci_ecam_map_bus(bus, devfn, where); + } + return NULL; + } if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); @@ -3121,7 +3360,22 @@ static int apple_pcie_probe(struct platform_device *pdev) * reset-gpios provider here rejects the valid synthesized port before * the tunneled setup path can consume that handoff. */ - if (!hw->tunneled) { + if (hw->adopt_link) { + if (!of_get_available_child_count(dev->of_node) || + of_property_present(dev->of_node, "power-domains")) + return -EINVAL; + for_each_available_child_of_node_scoped(dev->of_node, child) { + u32 reg; + + /* A root port 00:.0 on bus 0, n a GP port */ + if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || + !(reg >> 11) || (reg >> 11) > 3 || + !of_property_read_bool(child, "apple,link-trained-by-bootloader") || + of_property_present(child, "reset-gpios") || + of_property_present(child, "pwren-gpios")) + return -EINVAL; + } + } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_probe_port(of_port); @@ -3164,15 +3418,15 @@ static int apple_pcie_probe(struct platform_device *pdev) return ret; } pcie->piodma_supplier = piodma_supplier; - /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ - if (piodma_supplier || pcie->managed_sids) { + /* Firmware-owned links cannot be recovered by user reset/removal. */ + if (piodma_supplier || pcie->managed_sids || hw->adopt_link) { bridge->no_user_remove = true; bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_dbg(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); + dev_dbg(dev, "firmware-owned host: automatic AER/DPC recovery disabled\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -3952,6 +4206,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) */ if (pcie->managed_sids) return pm_suspend_target_state == PM_SUSPEND_TO_IDLE ? 0 : -EBUSY; + if (pcie->hw->adopt_link) + return -EBUSY; if (!pcie->hw->tunneled) return 0; pcie->reset_on_resume = false; @@ -4095,6 +4351,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8152-pcie", .data = &t8152_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, + { .compatible = "apple,t6050-pcie", .data = &t6050_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } }; From 3bb54de2de84878e43f60f4192e25ae5a702700e Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:16 -0400 Subject: [PATCH 195/392] PCI: Add pci_trylock_rescan_remove() A driver's work item that rescans or resets part of a bus takes the rescan/remove lock. When the device is being removed, the remover holds that lock while it cancels the work, so a worker that blocks on the lock deadlocks against its own cancellation. Add pci_trylock_rescan_remove(), which takes the rescan/remove lock only if it is free. A worker that cannot get it returns and retries later, and the remover's cancellation completes. Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/probe.c | 7 +++++++ include/linux/pci.h | 1 + 2 files changed, 8 insertions(+) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 27008e2ea5afc..d625dd5e778a8 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3517,6 +3517,13 @@ void pci_lock_rescan_remove(void) } EXPORT_SYMBOL_GPL(pci_lock_rescan_remove); +/* Work cancelled by removal must not wait on the remover's own lock. */ +bool pci_trylock_rescan_remove(void) +{ + return mutex_trylock(&pci_rescan_remove_lock); +} +EXPORT_SYMBOL_GPL(pci_trylock_rescan_remove); + void pci_unlock_rescan_remove(void) { mutex_unlock(&pci_rescan_remove_lock); diff --git a/include/linux/pci.h b/include/linux/pci.h index 5cec6d838b3f7..c0d76d93a78f5 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1559,6 +1559,7 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev); /* Functions for PCI Hotplug drivers to use */ unsigned int pci_rescan_bus(struct pci_bus *bus); void pci_lock_rescan_remove(void); +bool pci_trylock_rescan_remove(void); void pci_unlock_rescan_remove(void); /* Vital Product Data routines */ From d3ea57796c57270272dd218f2e5ba182158d438f Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 20:45:53 -0500 Subject: [PATCH 196/392] PCI: apple: Add the T6050 port reset, enable, configure and train sequences An adopted T6050 port is left as the boot loader trained it, but an endpoint that has to come back on a retrained link (the N1 on port 0, after its ROM boot) needs the host to take the port down and bring it up again. Add the sequences for that, on the port's own register blocks (port, phy and intr2axi) and the root port's config space: - Disable: wait for the link to leave L1 and go idle, assert the port's PERST# through its handshake, then stop the lane clocks and the port's application clock. - Enable: the port reset table, which also clears the RID2SID and MSI map tables, the port's tunables, then the application clock and the PERST# release handshake, until the port is ready and its link idle. - Configure: with the root complex's read-only registers opened, its tunables and the per-rate shadow tunables, then a Gen3 x1 link (target speed, link width and capability) and a directed speed change; Intr2AXI enabled. - Train: the lane's reference clocks, the endpoint's PERST# released through its pad, the LTSSM started, then the 100 ms wait after link up. The tunables are the lists the boot loader copies onto the root port node, groups of (offset, 4, mask, value); a list it did not copy is skipped. The root-port reset that runs the sequences follows; until then they are marked __maybe_unused. Co-developed-by: Dj Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 340 ++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b0a4a549cdd9e..521bc1738ae39 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -186,6 +186,20 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_MSIMAP_ENABLE BIT(31) #define PORT_MSIMAP_TARGET GENMASK(7, 0) +/* T6050: the root port's port logic, laid out as DesignWare's */ +#define RC_T6050_PORT_LINK_CTRL 0x710 +#define RC_T6050_LINK_MODE GENMASK(21, 16) +#define RC_T6050_PORT_DEBUG0 0x728 +#define RC_T6050_LTSSM_STATE GENMASK(5, 0) +#define RC_T6050_LTSSM_L1_IDLE 0x14 +#define RC_T6050_LINK_WIDTH_SPEED_CTRL 0x80c +#define RC_T6050_LINK_WIDTH GENMASK(12, 8) +#define RC_T6050_SPEED_CHANGE BIT(17) +#define RC_T6050_GEN3_RELATED 0x890 +#define RC_T6050_RATE_SHADOW_SEL GENMASK(25, 24) +#define RC_T6050_MISC_CONTROL_1 0x8bc +#define RC_T6050_DBI_RO_WR_EN BIT(0) + /* * The doorbell address must be in the bottom 4GB because the base register * is only 32 bits wide. Exclude it from the IOVA range and share the same @@ -479,6 +493,7 @@ struct apple_pcie_port { void __iomem *intr2axi; /* adopt_link: this port's */ u16 adopted_rid; /* adopt_link: secondary bus << 8 */ u32 adopted_sid; + void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -1786,6 +1801,331 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) return 0; } +/* + * T6050 port bring-up, for an adopted port whose endpoint must come back on a + * retrained link: port disable, the port reset table, port enable, root complex + * configuration for Gen3 x1, then the lane PHY, PERST# and the LTSSM. The port's + * registers are its port, phy and intr2axi blocks; the root complex's + * are the root port's config space. The boot loader copies the port's tunables + * onto its node. + */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001001ff }, + { 0x82c, 0x00010000 }, +}; + +/* Written after the RID2SID and MSI map tables are cleared */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs_late[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static void apple_pcie_t6050_rmw(void __iomem *addr, u32 clear, u32 set) +{ + writel((readl(addr) & ~clear) | set, addr); +} + +static int apple_pcie_t6050_rc_rmw(struct pci_dev *rp, int where, u32 clear, u32 set) +{ + u32 value; + int ret; + + ret = pci_read_config_dword(rp, where, &value); + if (!ret) + ret = pci_write_config_dword(rp, where, (value & ~clear) | set); + return pcibios_err_to_errno(ret); +} + +static int apple_pcie_t6050_wait(struct apple_pcie_port *port, void __iomem *base, + u32 offset, u32 mask, u32 value, unsigned int timeout_us) +{ + u32 stat; + int ret; + + ret = readl_poll_timeout(base + offset, stat, (stat & mask) == value, 1, timeout_us); + if (ret) + dev_err(port->pcie->dev, "port %d: %#x = %#x, waiting for %#x/%#x\n", + port->idx, offset, stat, value, mask); + return ret; +} + +/* + * A tunable list the boot loader copies onto the root port node, groups of + * (offset, 4, mask, value): for the port's registers, or with @rp for the root + * port's config space. A list it did not copy is skipped. + */ +static int apple_pcie_t6050_tune(struct apple_pcie_port *port, struct pci_dev *rp, + const char *name) +{ + u32 cells[64]; + int count, ret, i; + + if (!of_property_present(port->np, name)) + return 0; + count = of_property_count_u32_elems(port->np, name); + if (count <= 0 || count > ARRAY_SIZE(cells) || count % 4) + return -EINVAL; + ret = of_property_read_u32_array(port->np, name, cells, count); + if (ret) + return ret; + for (i = 0; i < count; i += 4) + if (cells[i + 1] != 4 || (cells[i] & 3) || + cells[i] >= (rp ? PCI_CFG_SPACE_EXP_SIZE : SZ_16K)) + return -EINVAL; + + for (i = 0; i < count; i += 4) { + if (!rp) { + apple_pcie_t6050_rmw(port->base + cells[i], cells[i + 2], cells[i + 3]); + continue; + } + ret = apple_pcie_t6050_rc_rmw(rp, cells[i], cells[i + 2], cells[i + 3]); + if (ret) + return ret; + } + return 0; +} + +/* PERST# through its pad: a GPIO output (function bits cleared) at the level asked for */ +static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) +{ + u32 old, value; + + old = readl(port->perst); + if (old & BIT(21)) + return -EIO; + value = (old & ~0x27f) | 0x202 | !asserted; + /* The input enable (bit 9) goes on after the rest */ + writel((value & ~0x200) | (old & 0x200), port->perst); + if (!(old & 0x200)) + writel(value, port->perst); + return (readl(port->perst) ^ value) & 0x27f ? -EIO : 0; +} + +/* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ +static int apple_pcie_t6050_port_disable(struct apple_pcie_port *port, struct pci_dev *rp) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + void __iomem *lane = port->phy + PHY_LANE_CFG; + u32 stat; + int ret, err; + + if (!(readl(port->base + PORT_STATUS) & BIT(2))) { + ret = apple_pcie_t6050_wait(port, port->base, 0x820, BIT(2), BIT(2), 10000); + if (ret) + return ret; + } + ret = readl_poll_timeout_atomic(port->base + 0x820, stat, + !(stat & GENMASK(8, 0)) || !(stat & GENMASK(23, 16)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + 0x13c, 0, BIT(8)); + + /* The link out of L1 */ + ret = read_poll_timeout_atomic(pci_read_config_dword, err, + err || FIELD_GET(RC_T6050_LTSSM_STATE, stat) != + RC_T6050_LTSSM_L1_IDLE, + 5, 5000, false, rp, RC_T6050_PORT_DEBUG0, &stat); + if (ret || err) + return ret ?: pcibios_err_to_errno(err); + + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(24)); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(16), BIT(16), 2000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(16)); + udelay(1); + apple_pcie_t6050_rmw(perst, PORT_PERST_OFF, 0); + udelay(1); + apple_pcie_t6050_rmw(perst, BIT(24), 0); + apple_pcie_t6050_rmw(port->base + 0x13c, BIT(8), 0); + + apple_pcie_t6050_rmw(lane, BIT(31), 0); + apple_pcie_t6050_rmw(lane, BIT(10), 0); + apple_pcie_t6050_rmw(lane, BIT(9), 0); + apple_pcie_t6050_rmw(lane, 0, BIT(4)); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK0REQ, 0); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK1REQ, 0); + + writel(~0, port->base + PORT_INTSTAT); + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, PORT_APPCLK_EN, 0); + return apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, 0, + 250000); +} + +/* The port reset table, then the port clocked and out of reset, ready to train */ +static int apple_pcie_t6050_port_enable(struct apple_pcie_port *port) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + u32 stat; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs); i++) + writel(apple_pcie_t6050_reset_regs[i].value, + port->base + apple_pcie_t6050_reset_regs[i].offset); + for (i = 0; i < port->pcie->hw->max_rid2sid; i++) + writel(0, port->base + port_rid2sid_offset(port, i)); + for (i = 0; i < 256; i++) + writel(0, port->base + PORT_T602X_MSIMAP + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs_late); i++) + writel(apple_pcie_t6050_reset_regs_late[i].value, + port->base + apple_pcie_t6050_reset_regs_late[i].offset); + + if (!(readl(perst) & BIT(16))) + return -EIO; + ret = apple_pcie_t6050_tune(port, NULL, "apple,apcie-config-tunables"); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_EN); + + ret = readl_poll_timeout_atomic(port->base + 0xa8, stat, + (stat & (BIT(16) | BIT(0))) != (BIT(16) | BIT(0)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, PORT_PERST_OFF); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 20000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, BIT(16), 0); + ret = apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, + PORT_STATUS_READY, 250000); + if (ret) + return ret; + + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_CGDIS); + return apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_BUSY, 0, + 250000); +} + +/* Root complex tunables and a Gen3 x1 link, with the read-only registers opened for it */ +static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port, struct pci_dev *rp) +{ + static const char * const shadow[] = { + "apple,pcie-rc-gen3-shadow-tunables", + "apple,pcie-rc-gen4-shadow-tunables", + "apple,pcie-rc-gen5-shadow-tunables", + }; + static const u32 shadow_sel[] = { 0, BIT(24), 0 }; + int cap = pci_pcie_cap(rp); + u32 value; + u16 ctl2; + int ret, err, i; + + if (!cap) + return -EIO; + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, 0, RC_T6050_DBI_RO_WR_EN); + if (ret) + return ret; + ret = apple_pcie_t6050_tune(port, rp, "apple,pcie-rc-tunables"); + if (ret) + goto lock; + for (i = 0; i < ARRAY_SIZE(shadow); i++) { + ret = apple_pcie_t6050_rc_rmw(rp, RC_T6050_GEN3_RELATED, + RC_T6050_RATE_SHADOW_SEL, shadow_sel[i]) ?: + apple_pcie_t6050_tune(port, rp, shadow[i]); + if (ret) + goto lock; + } + ret = apple_pcie_t6050_rc_rmw(rp, 0x194, 0, BIT(0)); + if (ret) + goto lock; + + ret = pcibios_err_to_errno(pci_read_config_dword(rp, cap + PCI_EXP_LNKCAP2, &value)); + if (ret) + goto lock; + if (!(value & PCI_EXP_LNKCAP2_SLS_8_0GB)) { + ret = -EIO; + goto lock; + } + ret = pcibios_err_to_errno(pci_read_config_word(rp, cap + PCI_EXP_LNKCTL2, &ctl2)) ?: + pcibios_err_to_errno(pci_write_config_word(rp, cap + PCI_EXP_LNKCTL2, + (ctl2 & ~PCI_EXP_LNKCTL2_TLS) | + PCI_EXP_LNKCTL2_TLS_8_0GT)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_PORT_LINK_CTRL, RC_T6050_LINK_MODE, + FIELD_PREP(RC_T6050_LINK_MODE, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, RC_T6050_LINK_WIDTH, + FIELD_PREP(RC_T6050_LINK_WIDTH, 1)) ?: + apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW, + FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1)); +lock: + err = apple_pcie_t6050_rc_rmw(rp, RC_T6050_MISC_CONTROL_1, RC_T6050_DBI_RO_WR_EN, 0); + ret = ret ?: err; + if (ret) + return ret; + + ret = apple_pcie_t6050_rc_rmw(rp, cap + PCI_EXP_LNKCTL, 0, PCI_EXP_LNKCTL_LBMIE) ?: + apple_pcie_t6050_rc_rmw(rp, RC_T6050_LINK_WIDTH_SPEED_CTRL, 0, RC_T6050_SPEED_CHANGE); + if (ret) + return ret; + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + return 0; +} + +/* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ +static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port *port, + struct pci_dev *rp) +{ + return apple_pcie_t6050_port_disable(port, rp) ?: + apple_pcie_t6050_port_enable(port) ?: + apple_pcie_t6050_port_configure(port, rp); +} + +/* Lane clocks, PERST# released, then the LTSSM to link up */ +static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *port) +{ + void __iomem *lane = port->phy + PHY_LANE_CFG; + int ret; + + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK0REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK0ACK, + PHY_LANE_CFG_REFCLK0ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK1REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK1ACK, + PHY_LANE_CFG_REFCLK1ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, BIT(4), 0); + udelay(1); + apple_pcie_t6050_rmw(lane, 0, BIT(9)); + apple_pcie_t6050_rmw(lane, 0, BIT(10)); + + /* Tperst-clk is at least 100 us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 110); + ret = apple_pcie_t6050_perst(port, false); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_LTSSMCTL, 0, PORT_LTSSMCTL_START); + ret = apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_UP, + PORT_LINKSTS_UP, 500000); + if (ret) + return ret; + + /* Wait for 100ms after the link trains (PCIe r5.0, 6.6.1) */ + msleep(100); + return 0; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) From 554bacbb3baa639642aef86ae09c2adfe4e07f24 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 197/392] PCI: apple: Adopt T6050 port 0's multi-function N1 and reset its root port The N1 Wi-Fi/Bluetooth sits on apcie0 port 0, which the bootloader now trains like ports 2 and 3. Two things the adoption assumed do not hold for it: - It is multi-function. In its ROM personality only function 0 answers; after its firmware starts, Control, Alpha and Beta (0-2) do. Each function's RID is routed to its own DART stream from the iommu-map (slot == SID) when it is enabled, instead of the one route set at adoption; config accesses on the adopted bus reach every function of device 0; teardown clears every slot it routed. Port 0 is accepted. - Its ROM links at Gen1. Adoption takes an x1 link at Gen1 to Gen3 (the bootloader's target), not only Gen3; the endpoint picks the speed. - After the ROM boot it must come back on a retrained link. The host provides reset_root_port for an adopted port that names its PERST# pad (perst): PME_Turn_Off to L2, PERST#, then the T6050 port disable, reset, enable, configure and train sequences, with the routes and the MSI map rewritten in between. Tunables the boot loader copied onto the root port node are applied; absent ones are skipped. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 9 +- drivers/pci/controller/pcie-apple.c | 218 ++++++++++++++---- 2 files changed, 184 insertions(+), 43 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 5823cc0ff837a..2446facb91d3a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -242,15 +242,16 @@ allOf: properties: reg: minItems: 5 - maxItems: 14 + maxItems: 18 reg-names: description: config and rc, then port, phy and intr2axi for each - bootloader-trained root port 00:.0. + bootloader-trained root port 00:.0, and perst (its PERST# + pad) for a port whose endpoint needs a root-port reset. minItems: 5 - maxItems: 14 + maxItems: 18 items: - pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + pattern: "^(config|rc|(port|phy|intr2axi|perst)[0-3])$" interrupts: maxItems: 4 interrupt-names: diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 521bc1738ae39..9a99347476bc4 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -489,14 +489,14 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; - bool adopted_route; void __iomem *intr2axi; /* adopt_link: this port's */ - u16 adopted_rid; /* adopt_link: secondary bus << 8 */ - u32 adopted_sid; + u8 adopted_bus; /* adopt_link: the secondary bus */ + unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); +static struct apple_pcie *apple_pcie_lookup(struct device *dev); static int apple_pcie_radio_check_cold(struct apple_pcie_port *port); static void rmw_set(u32 set, void __iomem *addr) @@ -1585,9 +1585,12 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) if (port->pcie->hw->adopt_link && port->domain) apple_pcie_port_writel(port, 0, PORT_MSICFG); - if (port->adopted_route) { - apple_pcie_rid2sid_write(port, port->adopted_sid, 0); - port->adopted_route = false; + if (port->adopted_slots) { + int slot; + + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + apple_pcie_rid2sid_write(port, slot, 0); + port->adopted_slots = 0; } if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) @@ -1673,9 +1676,60 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * The DART stream an adopted port's function `rid` maps to: one T6050 DART stream, nonzero + * and below the RID2SID table size (slot == SID), from the host's iommu-map. + */ +static int apple_pcie_adopted_sid(struct apple_pcie_port *port, u32 rid, u32 *sid) +{ + struct apple_pcie *pcie = port->pcie; + struct of_phandle_args iommu = {}; + int ret; + + if (rid >> 8 != port->adopted_bus) { + /* The PCI core numbers root ports' buses in devfn order: the bootloader must too */ + dev_err(pcie->dev, "port %d: RID %04x is off bus %02x, which the bootloader numbered\n", + port->idx, rid, port->adopted_bus); + return -ENODEV; + } + ret = of_map_iommu_id(pcie->dev->of_node, rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + iommu.args[0] < BITS_PER_LONG && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + *sid = iommu.args[0]; + of_node_put(iommu.np); + return ret; +} + +/* Route an adopted port's function `rid` to its stream, or confirm the route is in place. */ +static int apple_pcie_adopted_route(struct apple_pcie_port *port, u32 rid) +{ + u32 sid, value, cur; + int ret = apple_pcie_adopted_sid(port, rid, &sid); + + if (ret) + return ret; + value = T6050_ROUTE(rid, sid); + cur = apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)); + if (cur == value && test_bit(sid, &port->adopted_slots)) + return 0; + if (cur & PORT_RID2SID_VALID) + return -EBUSY; + set_bit(sid, &port->adopted_slots); + if (apple_pcie_rid2sid_write(port, sid, value) != value) + return -EIO; + dev_info(port->pcie->dev, "port %d: RID %04x -> SID %u, slot %u\n", + port->idx, rid, sid, sid); + return 0; +} + /* * Qualified J775D handoff of one bootloader-trained port: its root port - * 00:.0 and the single endpoint on its secondary bus, with DMA and + * 00:.0 and the endpoint on its secondary bus (function 0 routed here; a + * multi-function endpoint's other functions when they are enabled), with DMA and * endpoint MSI disabled. Every resource comes from the DT by the port's index * (port, phy, intr2axi, interrupt port, the iommu-map entry for the * endpoint's RID). All checks precede the first PCIe register write. @@ -1685,7 +1739,6 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; struct apple_pcie *pcie = port->pcie; struct platform_device *pdev = to_platform_device(pcie->dev); - struct of_phandle_args iommu = {}; void __iomem *rp, *ep; char name[16]; u32 value, bus; @@ -1730,15 +1783,8 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) PCI_COMMAND_MASTER))) return -ENODEV; - port->adopted_rid = bus << 8; - ret = of_map_iommu_id(pcie->dev->of_node, port->adopted_rid, &iommu); - if (ret) - return ret; - ret = iommu.args_count == 1 && iommu.args[0] && - iommu.args[0] < pcie->hw->max_rid2sid && - of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; - port->adopted_sid = iommu.args[0]; - of_node_put(iommu.np); + port->adopted_bus = bus; + ret = apple_pcie_adopted_sid(port, bus << 8, &value); if (ret) return ret; @@ -1746,6 +1792,13 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); if (IS_ERR(port->intr2axi)) return PTR_ERR(port->intr2axi); + /* Only a port whose endpoint needs a root-port reset names its PERST# pad */ + snprintf(name, sizeof(name), "perst%d", port->idx); + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) { + port->perst = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->perst)) + return PTR_ERR(port->perst); + } /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ cap = readb(rp + PCI_CAPABILITY_LIST); @@ -1758,9 +1811,14 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) } if (!cap || i == 48) return -EINVAL; + /* + * x1 at up to the Gen3 target the bootloader set. The endpoint picks the speed: the N1's + * ROM personality links at Gen1 and comes back at Gen3 after its root-port reset. + */ value = readw(rp + cap + PCI_EXP_LNKSTA); - if ((value & (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != - (PCI_EXP_LNKSTA_CLS_8_0GB | (1 << PCI_EXP_LNKSTA_NLW_SHIFT))) + if ((value & (PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != (1 << PCI_EXP_LNKSTA_NLW_SHIFT) || + (value & PCI_EXP_LNKSTA_CLS) < PCI_EXP_LNKSTA_CLS_2_5GB || + (value & PCI_EXP_LNKSTA_CLS) > PCI_EXP_LNKSTA_CLS_8_0GB) return -ENOLINK; /* Refuse an active endpoint or malformed capability chain. */ @@ -1792,13 +1850,10 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) return -EBUSY; /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ - value = T6050_ROUTE(port->adopted_rid, port->adopted_sid); - port->adopted_route = true; - if (apple_pcie_rid2sid_write(port, port->adopted_sid, value) != value) - return -EIO; - dev_info(pcie->dev, "adopted port %d, RID %04x -> SID %u, slot %u\n", - port->idx, port->adopted_rid, port->adopted_sid, port->adopted_sid); - return 0; + ret = apple_pcie_adopted_route(port, bus << 8); + if (!ret) + dev_info(pcie->dev, "adopted port %d, endpoint %02x:00\n", port->idx, bus); + return ret; } /* @@ -2081,8 +2136,7 @@ static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port, struct } /* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ -static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port *port, - struct pci_dev *rp) +static int apple_pcie_t6050_port_restart(struct apple_pcie_port *port, struct pci_dev *rp) { return apple_pcie_t6050_port_disable(port, rp) ?: apple_pcie_t6050_port_enable(port) ?: @@ -2090,7 +2144,7 @@ static int __maybe_unused apple_pcie_t6050_port_restart(struct apple_pcie_port * } /* Lane clocks, PERST# released, then the LTSSM to link up */ -static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *port) +static int apple_pcie_t6050_port_train(struct apple_pcie_port *port) { void __iomem *lane = port->phy + PHY_LANE_CFG; int ret; @@ -2126,6 +2180,87 @@ static int __maybe_unused apple_pcie_t6050_port_train(struct apple_pcie_port *po return 0; } +/* + * Root-port reset of an adopted T6050 port, for an endpoint that must come back on a + * retrained link (the N1 after its ROM boot): PME_Turn_Off to L2, PERST# asserted, the + * port disabled, reset, enabled and configured, its routes and MSI map rewritten, then the + * lane PHY, PERST# release and LTSSM to a Gen3 x1 link. The caller has quiesced the + * endpoint and saved the root port's state; the PCI core restores it. + */ +static int apple_pcie_t6050_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *rp) +{ + struct pci_config_window *cfg = rp->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port = NULL, *p; + u32 routes[BITS_PER_LONG], value; + int ret, slot, i; + + if (!pcie || !pcie->hw->adopt_link || !pci_is_root_bus(rp->bus)) + return -EOPNOTSUPP; + list_for_each_entry(p, &pcie->ports, entry) + if (p->idx == PCI_SLOT(rp->devfn)) + port = p; + if (!port || !port->perst || !port->intr2axi) + return -EOPNOTSUPP; + + mutex_lock(&pcie->lock); + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + routes[slot] = apple_pcie_port_readl(port, port_rid2sid_offset(port, slot)); + + /* Bridge decoding off and the secondary bus unnumbered while the link is down */ + ret = pcibios_err_to_errno(pci_write_config_word(rp, PCI_COMMAND, 0)) ?: + pcibios_err_to_errno(pci_read_config_dword(rp, PCI_PRIMARY_BUS, &value)) ?: + pcibios_err_to_errno(pci_write_config_dword(rp, PCI_PRIMARY_BUS, + value & 0xff0000ff)); + if (ret) + goto out; + /* PME_Turn_Off, then L2 */ + apple_pcie_port_writel(port, apple_pcie_port_readl(port, PORT_APPCLK) | 0x100, PORT_APPCLK); + apple_pcie_port_writel(port, 0x11, 0x88); + ret = apple_pcie_t6050_wait(port, port->base, 0x88, 1, 0, 500000) ?: + apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, BIT(6), BIT(6), 500000); + if (ret) + goto out; + WRITE_ONCE(port->link_failed, true); + ret = apple_pcie_t6050_perst(port, true); + if (ret) + goto out; + writel(readl(port->phy) & ~BIT(31), port->phy); + ret = apple_pcie_t6050_port_restart(port, rp); + if (ret) + goto out; + + /* The reset table cleared the routes and the MSI setup: put back what Linux set */ + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + if (apple_pcie_rid2sid_write(port, slot, routes[slot]) != routes[slot]) + ret = -EIO; + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); + for (i = 0; i < pcie->nvecs; i++) + apple_pcie_port_writel(port, FIELD_PREP(PORT_MSIMAP_TARGET, pcie->msi_vector_base + i) | + PORT_MSIMAP_ENABLE, pcie->hw->port_msimap + 4 * i); + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + apple_pcie_port_writel(port, PORT_MSICFG_EN, PORT_MSICFG); + if (ret) + goto out; + + ret = apple_pcie_t6050_port_train(port); + if (ret) + goto out; + /* Link interrupts stay masked after the reset table: the link is checked on access */ + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + WRITE_ONCE(port->link_failed, false); + dev_info(pcie->dev, "port %d: link retrained after a root-port reset\n", port->idx); +out: + mutex_unlock(&pcie->lock); + if (ret) + dev_err(pcie->dev, "port %d: root-port reset stopped: %d\n", port->idx, ret); + return ret; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -2968,11 +3103,14 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; - if (pcie->hw->adopt_link) - return rid == port->adopted_rid && sid == port->adopted_sid && - !READ_ONCE(port->link_failed) && - apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)) == - T6050_ROUTE(rid, sid) ? 0 : -ENODEV; + if (pcie->hw->adopt_link) { + if (READ_ONCE(port->link_failed)) + return -ENODEV; + mutex_lock(&pcie->lock); + err = apple_pcie_adopted_route(port, rid); + mutex_unlock(&pcie->lock); + return err; + } mutex_lock(&port->pcie->lock); if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { @@ -3107,16 +3245,16 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie_port *port; if (pcie->hw->adopt_link) { - /* Only adopted ports' root ports, and the one endpoint behind each. */ + /* Only adopted ports' root ports, and the endpoint behind each. */ list_for_each_entry(port, &pcie->ports, entry) { if (bus->number == 0) { if (devfn == PCI_DEVFN(port->idx, 0)) return pci_ecam_map_bus(bus, devfn, where); continue; } - if (bus->number != port->adopted_rid >> 8) + if (bus->number != port->adopted_bus) continue; - if (devfn || READ_ONCE(port->link_failed)) + if (PCI_SLOT(devfn) || READ_ONCE(port->link_failed)) return NULL; if ((apple_pcie_port_readl(port, PORT_LINKSTS) & (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { @@ -3707,9 +3845,9 @@ static int apple_pcie_probe(struct platform_device *pdev) for_each_available_child_of_node_scoped(dev->of_node, child) { u32 reg; - /* A root port 00:.0 on bus 0, n a GP port */ + /* A root port 00:.0 on bus 0, n a GP port (0-3) */ if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || - !(reg >> 11) || (reg >> 11) > 3 || + (reg >> 11) > 3 || !of_property_read_bool(child, "apple,link-trained-by-bootloader") || of_property_present(child, "reset-gpios") || of_property_present(child, "pwren-gpios")) @@ -3832,6 +3970,8 @@ static int apple_pcie_probe(struct platform_device *pdev) } } + if (hw->adopt_link) + bridge->reset_root_port = apple_pcie_t6050_reset_root_port; ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); if (ret) { apple_pcie_cleanup(pcie); From 5843113c59c0cb0dad812414dedfaaa51888cae4 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 23:53:26 -0800 Subject: [PATCH 198/392] PCI: apple: Drive T6050 port 0's PERST# through the pin controller The adopted port 0 named its PERST# pad as a 4-byte reg of the host (perst0, one word inside the pin controller's block) and wrote it raw. With the pin controller in the device tree the two nodes claim the same MMIO word, and whichever probes second fails: when pinctrl wins, the host's port setup stops with -EBUSY and no port is adopted, so the Ethernet and USB behind ports 2 and 3 are gone with it. One owner for the pad: the pin controller. The port node carries reset-gpios, as every other Apple host's port nodes do, and the host takes the line as the bootloader left it (GPIOD_ASIS: probe never touches a trained link's reset) and drives it only in the root-port reset, as a GPIO output at the logical level the reset asks for. The raw pad write set the same things (function bits cleared, mode output, the data bit) plus the pad's input enable, which the pin controller leaves as the bootloader set it. Probe asks for the line before any port is set up, so a host whose pin controller is not up yet defers instead of adopting nothing; an adopted port without reset-gpios has no reset, as before (reset_root_port answers -EOPNOTSUPP). The perst reg, its binding entry and the raw pad write go; the binding allows reset-gpios on an adopted port again. Probe names the port whose line it cannot have, as the probe of the other hosts' ports does: a line the pin controller reserves (gpio-reserved-ranges) is refused with -EINVAL, which the host's probe otherwise returned with no word of which port or property. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/pci/apple,pcie.yaml | 15 +++--- drivers/pci/controller/pcie-apple.c | 49 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 2446facb91d3a..64f0a2d43e9aa 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -242,16 +242,15 @@ allOf: properties: reg: minItems: 5 - maxItems: 18 + maxItems: 14 reg-names: description: config and rc, then port, phy and intr2axi for each - bootloader-trained root port 00:.0, and perst (its PERST# - pad) for a port whose endpoint needs a root-port reset. + bootloader-trained root port 00:.0. minItems: 5 - maxItems: 18 + maxItems: 14 items: - pattern: "^(config|rc|(port|phy|intr2axi|perst)[0-3])$" + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" interrupts: maxItems: 4 interrupt-names: @@ -267,7 +266,11 @@ allOf: patternProperties: "^pci@": properties: - reset-gpios: false + reset-gpios: + description: + The PERST# line of a port whose endpoint needs a root-port + reset after the bootloader trained the link. Left as the + bootloader set it until that reset. pwren-gpios: false required: - apple,link-trained-by-bootloader diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 9a99347476bc4..a28781a22ed0d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -473,7 +473,7 @@ struct apple_pcie_port { struct device_node *np; void __iomem *base; void __iomem *phy; - /* PERST# of a conventional port with a single reset line. */ + /* PERST# of a conventional or adopted port with a single reset line. */ struct gpio_desc *perst; struct irq_domain *domain; struct list_head entry; @@ -492,7 +492,6 @@ struct apple_pcie_port { void __iomem *intr2axi; /* adopt_link: this port's */ u8 adopted_bus; /* adopt_link: the secondary bus */ unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ - void __iomem *perst; /* adopt_link: the PERST# pad, for a root-port reset */ }; static int apple_pcie_managed_admit(struct apple_pcie_port *port); @@ -1792,12 +1791,18 @@ static int apple_pcie_adopt_link(struct apple_pcie_port *port) port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); if (IS_ERR(port->intr2axi)) return PTR_ERR(port->intr2axi); - /* Only a port whose endpoint needs a root-port reset names its PERST# pad */ - snprintf(name, sizeof(name), "perst%d", port->idx); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) { - port->perst = devm_platform_ioremap_resource_byname(pdev, name); - if (IS_ERR(port->perst)) + /* + * Only a port whose endpoint needs a root-port reset names its PERST# line + * (reset-gpios, on the port node as on every other Apple host). The pin + * controller owns the pad; the line is taken as the bootloader left it, + * released, and driven only by the reset. + */ + port->perst = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), "reset", + GPIOD_ASIS, "PERST#"); + if (IS_ERR(port->perst)) { + if (PTR_ERR(port->perst) != -ENOENT) return PTR_ERR(port->perst); + port->perst = NULL; } /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ @@ -1955,20 +1960,14 @@ static int apple_pcie_t6050_tune(struct apple_pcie_port *port, struct pci_dev *r return 0; } -/* PERST# through its pad: a GPIO output (function bits cleared) at the level asked for */ +/* + * PERST# through the pin controller: the pad as a GPIO output (its function bits + * cleared, mode output) at the line's logical level, which the DT's + * GPIO_ACTIVE_LOW turns into the pad level. + */ static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) { - u32 old, value; - - old = readl(port->perst); - if (old & BIT(21)) - return -EIO; - value = (old & ~0x27f) | 0x202 | !asserted; - /* The input enable (bit 9) goes on after the rest */ - writel((value & ~0x200) | (old & 0x200), port->perst); - if (!(old & 0x200)) - writel(value, port->perst); - return (readl(port->perst) ^ value) & 0x27f ? -EIO : 0; + return gpiod_direction_output(port->perst, asserted); } /* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ @@ -3849,9 +3848,19 @@ static int apple_pcie_probe(struct platform_device *pdev) if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || (reg >> 11) > 3 || !of_property_read_bool(child, "apple,link-trained-by-bootloader") || - of_property_present(child, "reset-gpios") || of_property_present(child, "pwren-gpios")) return -EINVAL; + /* + * A port whose endpoint needs a root-port reset names its PERST# + * line: probe waits for the line's controller before any port is + * touched (a failed adoption would otherwise swallow the deferral). + */ + if (of_property_present(child, "reset-gpios")) { + ret = apple_pcie_probe_port(child); + if (ret) + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", + child); + } } } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ From 704b66f70f2ad14fd61f7ebbf4c3a7943f220fad Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 01:06:09 -0500 Subject: [PATCH 199/392] selftests: apple: Preserve multiline PCIe macros in policy tests The PCIe-C policy test extracts driver macro definitions to compile its production-code fixture. Taking only the first physical line truncates T6050_ROUTE and joins the following directive to its trailing backslash, so the fixture fails to compile after the T6050 host support is added. Copy continuation lines along with their define. The production fixture and both negative controls now pass. Developed with Claude Code. Signed-off-by: Erik Bourget --- tools/asahi/pciec-policy-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/pciec-policy-tests.py b/tools/asahi/pciec-policy-tests.py index a399c7ef6e023..a5a8fb96e143d 100644 --- a/tools/asahi/pciec-policy-tests.py +++ b/tools/asahi/pciec-policy-tests.py @@ -144,7 +144,7 @@ def main(): args = parser.parse_args() pci = (ROOT / 'drivers/pci/controller/pcie-apple.c').read_text() acio = (ROOT / 'drivers/thunderbolt/apple.c').read_text() - defines = '\n'.join(re.findall(r'^#define .*$', pci, re.M)) + defines = '\n'.join(re.findall(r'^#define (?:[^\n]*\\\n)*[^\n]*$', pci, re.M)) info = pci[pci.index('struct hw_info {'):pci.index('static const struct of_device_id apple_pcie_of_match[];')] matches = re.search(r'static const struct of_device_id apple_pcie_of_match\[\] = \{.*?\n\};', pci, re.S)[0] production = '\n'.join((defines, info, matches, From d1ccc586855d3a5bfc356ea20aa04c7402f73b7d Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:18 -0400 Subject: [PATCH 200/392] phy: apple: Add T6050 DisplayPort and USB combo PHY Add the T6050 DisplayPort/USB combo PHY and four-clock display crossbar profile. Carry the current Type-C, USB2, USB3 PIPE sequencing, and teardown behavior from the published T6050 source. Source snapshot: t6050-gpu ee68abb105200e99636a5edfc647a5cfc1d89954 Integration base: 9cbdd0734d9e186c5f5f5949a21b909d10c2571b Signed-off-by: Dj Signed-off-by: Erik Bourget --- .../bindings/mux/apple,display-crossbar.yaml | 6 +- .../bindings/phy/apple,t6050-dp-phy.yaml | 224 +++++ drivers/mux/apple-display-crossbar.c | 301 ++++++- drivers/phy/apple/Kconfig | 13 + drivers/phy/apple/Makefile | 2 + drivers/phy/apple/atc-t6050-combo.h | 394 +++++++++ drivers/phy/apple/atc-t6050-dp.c | 806 ++++++++++++++++++ drivers/phy/apple/atc-t6050-typec-data.h | 700 +++++++++++++++ .../apple/usb-t6050/apple_t6050_usb2_reset.h | 23 + .../apple/usb-t6050/apple_t6050_usb3_bist.h | 59 ++ .../apple/usb-t6050/apple_t6050_usb3_pipe.h | 136 +++ .../phy/apple/usb-t6050/apple_t6050_usb_phy.h | 286 +++++++ 12 files changed, 2942 insertions(+), 8 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml create mode 100644 drivers/phy/apple/atc-t6050-combo.h create mode 100644 drivers/phy/apple/atc-t6050-dp.c create mode 100644 drivers/phy/apple/atc-t6050-typec-data.h create mode 100644 drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h create mode 100644 drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h create mode 100644 drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h create mode 100644 drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h diff --git a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml index 245f520044e64..5597c9c1489ea 100644 --- a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml +++ b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) AND GPL-2.0-only %YAML 1.2 --- $id: http://devicetree.org/schemas/mux/apple,display-crossbar.yaml# @@ -11,8 +11,8 @@ maintainers: description: Routes display pipelines to the DisplayPort transmitters and inputs - of an Apple Type-C PHY. Each crossbar exposes three mux controls, - selected by the consumer's mux specifier. + of an Apple Type-C PHY. Older layouts expose three mux controls; the T8142/T6050 layout + exposes five. The consumer selects an output using the mux specifier. allOf: - $ref: mux-controller.yaml# diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml new file mode 100644 index 0000000000000..52cb96f4a965b --- /dev/null +++ b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/apple,t6050-dp-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T6050 DisplayPort and Type-C combo PHY + +maintainers: + - Dj + +description: | + Fixed four-lane DisplayPort PHY for HDMI bridges, or one shared owner of + Type-C USB2, USB3 and DisplayPort alternate mode. DisplayPort supports RBR + through HBR3 without spread-spectrum clocking. The combo provider serializes + controller shutdown, PIPE handoff, orientation and alternate-mode changes. + Register resources, power domains and board tunables come from the producer. + +properties: + compatible: + enum: + - apple,t6050-dp-phy + - apple,t6050-combo-phy + + reg: + minItems: 6 + maxItems: 11 + + reg-names: + minItems: 6 + maxItems: 11 + + '#phy-cells': + enum: [0, 1] + + '#reset-cells': + const: 1 + + apple,port-index: + enum: [1, 2] + description: Native Type-C port; the debug/proxy port is excluded + + apple,guest-port-reserved: + type: boolean + description: The boot producer has excluded this port from host ownership + + mode-switch: true + orientation-switch: true + + mboxes: + maxItems: 1 + + port: + $ref: /schemas/graph.yaml#/properties/port + + apple,tunable-aus40cmn-shm: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples for the common analog bank + + apple,tunable-cio4pll: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples for the CIO PLL bank + + apple,tunable-usb2-default: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples for the USB2 PHY bank + + apple,tunable-usb2-host: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples for the USB2 host bank + + apple,tunable-pipe-default: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples for the PIPE handler + + power-domains: + minItems: 3 + maxItems: 3 + + power-domain-names: + items: + - const: usb-aon + - const: common + - const: wrapper + + apple,tunable-axi2af: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-fabric: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-common-cfg: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-lioa: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-lane0-rx-clock: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-lane1-rx-clock: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-lane0-rx-eq: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-lane1-rx-eq: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + + apple,tunable-auspll: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Offset, mask and value tuples within the named bank + +patternProperties: + '^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$': + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 16 + maxItems: 16 + description: Drive presets indexed by voltage swing times four plus pre-emphasis + +required: + - compatible + - reg + - reg-names + - '#phy-cells' + - power-domains + - power-domain-names + - apple,tunable-axi2af + - apple,tunable-fabric + - apple,tunable-common-cfg + - apple,tunable-lioa + - apple,tunable-lane0-rx-clock + - apple,tunable-lane1-rx-clock + - apple,tunable-lane0-rx-eq + - apple,tunable-lane1-rx-eq + - apple,tunable-auspll + +allOf: + - if: + properties: + compatible: + const: apple,t6050-combo-phy + then: + properties: + '#phy-cells': + const: 1 + reg: + minItems: 11 + maxItems: 11 + reg-names: + items: + - const: core + - const: usb2phy + - const: usb2-digital + - const: reset + - const: fabric + - const: axi2af + - const: lioa + - const: common-cfg + - const: usb-control + - const: firmware-data + - const: pipehandler + required: + - '#reset-cells' + - apple,port-index + - apple,guest-port-reserved + - mode-switch + - orientation-switch + - mboxes + - apple,tunable-aus40cmn-shm + - apple,tunable-cio4pll + - apple,tunable-usb2-default + - apple,tunable-usb2-host + - apple,tunable-pipe-default + else: + properties: + '#phy-cells': + const: 0 + if: + required: + - mode-switch + then: + properties: + reg: + minItems: 8 + maxItems: 8 + reg-names: + items: + - const: core + - const: reset + - const: fabric + - const: axi2af + - const: lioa + - const: common-cfg + - const: usb-control + - const: firmware-data + required: + - orientation-switch + - mboxes + - apple,tunable-aus40cmn-shm + - apple,tunable-cio4pll + else: + properties: + reg: + minItems: 6 + maxItems: 6 + reg-names: + items: + - const: core + - const: reset + - const: fabric + - const: axi2af + - const: lioa + - const: common-cfg + +additionalProperties: false diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 65107c7de3800..223d7b813dc15 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple Silicon Display Crossbar multiplexer driver * @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -90,7 +93,10 @@ enum { MUX_DPPHY = 0, MUX_DPIN0 = 1, MUX_DPIN1 = 2, MUX_MAX = 3 }; static const char *apple_dpxbar_names[MUX_MAX] = { "dpphy", "dpin0", "dpin1" }; +enum { T8142_MUX_MAX = 5 }; + struct apple_dpxbar_hw { + unsigned int n_mux; unsigned int n_ufp; u32 tunable; const struct mux_control_ops *ops; @@ -108,12 +114,16 @@ struct apple_dpxbar { struct device *dev; const struct apple_dpxbar_hw *hw; void __iomem *regs; - int selected_dispext[MUX_MAX]; + int selected_dispext[T8142_MUX_MAX]; /* T6030 programs all its outputs itself, see apple_dpxbar_t6030_link() */ bool t6030_dpin; /* outputs holding a runtime PM reference (runtime_pm crossbars) */ unsigned long rpm_held; bool tunnel_link_up[MUX_MAX]; + u8 pclk[T8142_MUX_MAX]; + bool uhbr[T8142_MUX_MAX]; + bool enabled[T8142_MUX_MAX]; + bool faulted; spinlock_t lock; }; @@ -880,6 +890,225 @@ int apple_dpxbar_tunnel_select_source(struct mux_control *mux, int state) } EXPORT_SYMBOL_GPL(apple_dpxbar_tunnel_select_source); +/* + * T8142 (also used by T6050) has four pixel clocks and five outputs. + * Keep selection separate from clock activation: the PHY/DP IN owner supplies + * its actual clock and coding before link_up(), after configuring that clock. + * Only the four two-port external display sources are described here; AUX + * sources use a different selector-to-clock-bit mapping. + */ +static const u8 t8142_dest_shift[T8142_MUX_MAX] = { 8, 0, 4, 12, 16 }; +static const u8 t8142_select_shift[T8142_MUX_MAX] = { 8, 0, 4, 24, 0 }; +static const u8 t8142_select_shift2[T8142_MUX_MAX] = { 20, 12, 16, 28, 4 }; + +static int t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) +{ + u32 src, dst, clock, value; + int ret; + + if (!xbar->enabled[index]) + return 0; + src = BIT(xbar->selected_dispext[index]); + dst = BIT(t8142_dest_shift[index]); + clock = xbar->pclk[index] - 1; + + dpxbar_clear32(xbar, 0x3c, src); + dpxbar_clear32(xbar, 0x00, src); + dpxbar_clear32(xbar, 0x0c + 4 * clock, src); + dpxbar_clear32(xbar, 0x24 + 4 * clock, dst); + udelay(1); + ret = readl_poll_timeout_atomic(xbar->regs + 0x800, value, + !(value & src), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + ret = readl_poll_timeout_atomic(xbar->regs + 0x808 + 4 * clock, value, + !(value & src), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + ret = readl_poll_timeout_atomic(xbar->regs + 0x81c + 4 * clock, value, + !(value & dst), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + dpxbar_clear32(xbar, 0x08, src); + dpxbar_clear32(xbar, 0x20, 7 << (3 * xbar->selected_dispext[index])); + dpxbar_clear32(xbar, 0x38, 7 << t8142_dest_shift[index]); + dpxbar_set32(xbar, 0x04, src); + dpxbar_set32(xbar, 0x1c, src); + dpxbar_set32(xbar, 0x34, dst); + xbar->enabled[index] = false; + return 0; +} + +static int t8142_link_up(struct apple_dpxbar *xbar, unsigned int index) +{ + int state = xbar->selected_dispext[index]; + u32 clock, select, src, dst, value; + int ret; + + if (state < 0) + return -ENODEV; + if (!xbar->pclk[index]) + return -EINVAL; + if (xbar->enabled[index]) + return 0; + + /* The coding bit is shared by every output using this pixel clock. */ + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) + if (i != index && xbar->enabled[i] && + xbar->pclk[i] == xbar->pclk[index] && + xbar->uhbr[i] != xbar->uhbr[index]) + return -EBUSY; + + clock = xbar->pclk[index] - 1; + select = 2 * clock + 1; + src = BIT(state); + dst = BIT(t8142_dest_shift[index]); + dpxbar_clear32(xbar, 0x04, src); + dpxbar_clear32(xbar, 0x1c, src); + dpxbar_clear32(xbar, 0x34, dst); + udelay(1); + ret = readl_poll_timeout_atomic(xbar->regs + 0x804, value, + !(value & src), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + ret = readl_poll_timeout_atomic(xbar->regs + 0x818, value, + !(value & src), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + ret = readl_poll_timeout_atomic(xbar->regs + 0x82c, value, + !(value & dst), 1, 1000); + if (ret) { + xbar->faulted = true; + return ret; + } + dpxbar_set32(xbar, 0x08, src); + dpxbar_mask32(xbar, 0x20, 7 << (3 * state), select << (3 * state)); + dpxbar_mask32(xbar, 0x38, 7 << t8142_dest_shift[index], + select << t8142_dest_shift[index]); + dpxbar_mask32(xbar, 0x48, BIT(clock), xbar->uhbr[index] ? BIT(clock) : 0); + dpxbar_set32(xbar, 0x00, src); + dpxbar_set32(xbar, 0x0c + 4 * clock, src); + dpxbar_set32(xbar, 0x24 + 4 * clock, dst); + dpxbar_set32(xbar, 0x4c, dst); + dpxbar_set32(xbar, 0x3c, src); + xbar->enabled[index] = true; + return 0; +} + +static int apple_dpxbar_set_t8142(struct mux_control *mux, int state) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + u32 mask, value; + int ret = 0; + + if (state != MUX_IDLE_DISCONNECT && + (state < 0 || state >= 8 || (index == MUX_DPPHY && (state & 1)))) + return -EINVAL; + + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) { + ret = -EIO; + goto out; + } + if (state >= 0) { + for (unsigned int i = 0; i < T8142_MUX_MAX; i++) { + if (i != index && xbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&xbar->lock, flags); + return -EBUSY; + } + } + } + + if (xbar->selected_dispext[index] == state) + goto out; + if (xbar->selected_dispext[index] >= 0) { + ret = t8142_link_down(xbar, index); + if (ret) + goto out; + } + + /* No implicit clock choice or activation when selecting a new source. */ + xbar->pclk[index] = 0; + xbar->selected_dispext[index] = state; + if (state >= 0) { + mask = (0xfU << t8142_select_shift[index]) | + (0xfU << t8142_select_shift2[index]); + value = ((u32)state << t8142_select_shift[index]) | + ((u32)state << t8142_select_shift2[index]); + dpxbar_mask32(xbar, index == 4 ? 0x40 : 0x44, mask, value); + } +out: + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} + +static const struct mux_control_ops apple_dpxbar_t8142_ops = { + .set = apple_dpxbar_set_t8142, +}; + +/* Caller holds the mux selection throughout configuration and activation. */ +int apple_dpxbar_link_configure(struct mux_control *mux, unsigned int pclk, bool uhbr) +{ + struct apple_dpxbar *xbar; + unsigned int index; + unsigned long flags; + int ret = 0; + + if (!mux || mux->chip->ops != &apple_dpxbar_t8142_ops) + return -EOPNOTSUPP; + if (pclk < 1 || pclk > 4) + return -EINVAL; + xbar = mux_chip_priv(mux->chip); + index = mux_control_get_index(mux); + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) + ret = -EIO; + else if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (xbar->enabled[index]) + ret = -EBUSY; + else { + xbar->pclk[index] = pclk; + xbar->uhbr[index] = uhbr; + } + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dpxbar_link_configure); + +static int apple_dpxbar_t8142_link(struct mux_control *mux, bool up) +{ + struct apple_dpxbar *xbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&xbar->lock, flags); + if (xbar->faulted) + ret = -EIO; + else if (xbar->selected_dispext[index] < 0) + ret = -ENODEV; + else if (up) + ret = t8142_link_up(xbar, index); + else + ret = t8142_link_down(xbar, index); + spin_unlock_irqrestore(&xbar->lock, flags); + return ret; +} + + static int apple_dpxbar_t602x_link_up(struct mux_control *mux) { struct apple_dpxbar *xbar; @@ -961,6 +1190,9 @@ static int apple_dpxbar_t6030_link_set(struct mux_control *mux, bool up) int apple_dpxbar_link_up(struct mux_control *mux) { + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, true); + if (!mux) return -EINVAL; if (mux->chip->ops == &apple_dpxbar_t602x_ops) { @@ -1009,6 +1241,9 @@ static int apple_dpxbar_t602x_link_down(struct mux_control *mux) int apple_dpxbar_link_down(struct mux_control *mux) { + if (mux && mux->chip->ops == &apple_dpxbar_t8142_ops) + return apple_dpxbar_t8142_link(mux, false); + if (!mux) return -EINVAL; if (mux->chip->ops == &apple_dpxbar_t602x_ops) { @@ -1022,6 +1257,39 @@ int apple_dpxbar_link_down(struct mux_control *mux) } EXPORT_SYMBOL_GPL(apple_dpxbar_link_down); +static void apple_dpxbar_domain_release(void *data) +{ + struct device *pd = data; + + pm_runtime_put_sync_suspend(pd); + dev_pm_domain_detach(pd, true); +} + +static int apple_dpxbar_domains(struct device *dev) +{ + static const char * const names[] = { "usb-aon", "common" }; + int ret; + + /* Older handoff-only trees have no Linux-managed domains. */ + if (!of_property_present(dev->of_node, "power-domain-names")) + return 0; + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct device *pd = dev_pm_domain_attach_by_name(dev, names[i]); + + if (IS_ERR_OR_NULL(pd)) + return pd ? PTR_ERR(pd) : -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + ret = devm_add_action_or_reset(dev, apple_dpxbar_domain_release, pd); + if (ret) + return ret; + } + return 0; +} + static int apple_dpxbar_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1031,7 +1299,16 @@ static int apple_dpxbar_probe(struct platform_device *pdev) int ret; hw = of_device_get_match_data(dev); - mux_chip = devm_mux_chip_alloc(dev, MUX_MAX, sizeof(*dpxbar)); + if (hw->ops == &apple_dpxbar_t8142_ops) { + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + if (!res || resource_size(res) < 0x830) + return -EINVAL; + ret = apple_dpxbar_domains(dev); + if (ret) + return dev_err_probe(dev, ret, "cannot power crossbar controls\n"); + } + mux_chip = devm_mux_chip_alloc(dev, hw->n_mux, sizeof(*dpxbar)); if (IS_ERR(mux_chip)) return PTR_ERR(mux_chip); @@ -1052,14 +1329,14 @@ static int apple_dpxbar_probe(struct platform_device *pdev) return ret; } - if (!hw->no_tunable && + if (hw->ops == &apple_dpxbar_ops && !hw->no_tunable && !of_device_is_compatible(dev->of_node, "apple,t6020-display-crossbar")) { readl(dpxbar->regs + UNK_TUNABLE); writel(hw->tunable, dpxbar->regs + UNK_TUNABLE); readl(dpxbar->regs + UNK_TUNABLE); } - for (unsigned int i = 0; i < MUX_MAX; ++i) { + for (unsigned int i = 0; i < hw->n_mux; ++i) { mux_chip->mux[i].states = hw->n_ufp; mux_chip->mux[i].idle_state = MUX_IDLE_DISCONNECT; dpxbar->selected_dispext[i] = -1; @@ -1124,18 +1401,21 @@ static int apple_dpxbar_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_dpxbar_pm_ops, NULL, apple_dpxbar_resume); static const struct apple_dpxbar_hw apple_dpxbar_hw_t8103 = { + .n_mux = MUX_MAX, .n_ufp = 2, .tunable = 0, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t8112 = { + .n_mux = MUX_MAX, .n_ufp = 4, .tunable = 4278196325, .ops = &apple_dpxbar_ops, }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { + .n_mux = MUX_MAX, .n_ufp = 9, .tunable = 5, .ops = &apple_dpxbar_ops, @@ -1143,6 +1423,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { }; static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, }; @@ -1155,6 +1436,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { * The ADT lists nine UFP endpoints (dispext0-3, scodec0 and the DP IN ports). */ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6041 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, .t6030_flow = true, @@ -1170,6 +1452,7 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t6041 = { * output is selected; probe touches no register. */ static const struct apple_dpxbar_hw apple_dpxbar_hw_t8132 = { + .n_mux = MUX_MAX, .n_ufp = 9, .ops = &apple_dpxbar_t602x_ops, .t6030_flow = true, @@ -1177,7 +1460,15 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t8132 = { .runtime_pm = true, }; +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8142 = { + .n_mux = T8142_MUX_MAX, + .n_ufp = 8, + .ops = &apple_dpxbar_t8142_ops, +}; + static const struct of_device_id apple_dpxbar_ids[] = { + { .compatible = "apple,t6050-display-crossbar", .data = &apple_dpxbar_hw_t8142 }, + { .compatible = "apple,t8142-display-crossbar", .data = &apple_dpxbar_hw_t8142 }, { .compatible = "apple,t8103-display-crossbar", .data = &apple_dpxbar_hw_t8103, diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 5348987cf1dc8..36ed85b7935f2 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -62,3 +62,16 @@ config PHY_APPLE_DPTX such as the M2. This driver provides support for DisplayPort and is used on the Mac mini (M2 and M2 Pro, 2023). + +config PHY_APPLE_T6050_DP + tristate "Apple T6050 DisplayPort and Type-C combo PHY" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF && PM && TYPEC + select GENERIC_PHY + select APPLE_TUNABLE + select APPLE_RTKIT + select RESET_CONTROLLER + help + Fixed HDMI bridges and Type-C USB host/DisplayPort alternate mode. + One owner coordinates the shared PHY, firmware, PIPE and USB2 state. + Requires named register banks, power domains and firmware tunables. diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile index e7156408e792f..e0b94bb1dcaa5 100644 --- a/drivers/phy/apple/Makefile +++ b/drivers/phy/apple/Makefile @@ -8,3 +8,5 @@ phy-apple-t6040-usb2-y := t6040-usb2.o obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o phy-apple-dptx-y += dptx.o + +obj-$(CONFIG_PHY_APPLE_T6050_DP) += atc-t6050-dp.o diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h new file mode 100644 index 0000000000000..1066748dc1afe --- /dev/null +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -0,0 +1,394 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* One Type-C owner reuses the established DP and USB backend sequences. + * All calls below hold dp->lock; only PHY power_off authorizes PIPE teardown + * while the controller reset is deasserted. Lane changes require reset/off. + */ +static u32 combo_read(void *ctx, unsigned int bank, unsigned int offset) +{ + struct t6050_dp *dp = ctx; + return readl(dp->banks[bank] + offset); +} +static void combo_write(void *ctx, unsigned int bank, unsigned int offset, u32 value) +{ + struct t6050_dp *dp = ctx; + writel(value, dp->banks[bank] + offset); +} +static int combo_poll(void *ctx, unsigned int bank, unsigned int offset, + unsigned int mask, unsigned int expected, unsigned int usec) +{ + struct t6050_dp *dp = ctx; + u32 value; + return readl_poll_timeout(dp->banks[bank] + offset, value, + (value & mask) == expected, + (bank == 44 || bank == 35 || usec <= 1000) ? 1 : 1000, usec); +} +static void combo_delay(void *ctx, unsigned int usec) { udelay(usec); } +static void combo_pause(void *ctx, unsigned int nsec) { ndelay(nsec); } +static void combo_sleep(void *ctx, unsigned int msec) { msleep(msec); } +static int combo_wrapper(void *ctx, unsigned int active) +{ + struct t6050_dp *dp = ctx; + return atc_domain_set(&dp->domain[2], active); +} +static int combo_firmware(void *ctx, unsigned int active) +{ + return atc_firmware(ctx, active); +} +static int combo_tunable(void *ctx, unsigned int bank, const char *name) +{ + static const struct { u8 bank, index; const char *name; } common[] = { + {41, 0, "tunable_ATC0AXI2AF"}, {40, 1, "tunable_ATC_FABRIC"}, + {43, 2, "tunable_ATC_COMMON_CFG"}, {42, 3, "tunable_ATC0AXI2AF_LIOA"}, + {20, 4, "tunable_LN0_RX_CFG_TX_OF_RXCLK"}, {29, 5, "tunable_LN1_RX_CFG_TX_OF_RXCLK"}, + {18, 6, "tunable_LN0_RX_EQ_DFLT"}, {27, 7, "tunable_LN1_RX_EQ_DFLT"}, + {8, 8, "tunable_AUSPLL_CORE"}, {5, 9, "tunable_AUS40CMN_SHM"}, + {10, 10, "tunable_CIO4PLL_CORE"}, + }; + struct t6050_dp *dp = ctx; + + if (bank == 0 && !strcmp(name, "tunable_USB2PHY_DFLT")) { + apple_tunable_apply(dp->banks[0], dp->usb2_default); + return 0; + } + if (bank == 0 && !strcmp(name, "tunable_USB2PHY_HOST")) { + apple_tunable_apply(dp->banks[0], dp->usb2_host); + return 0; + } + for (unsigned int i = 0; i < ARRAY_SIZE(common); i++) + if (bank == common[i].bank && !strcmp(name, common[i].name)) { + apple_tunable_apply(dp->banks[bank], dp->tunables[common[i].index]); + return 0; + } + return -EINVAL; +} + +static int combo_quiesce(void *ctx) +{ + struct t6050_dp *dp = ctx; + int ret = 0; + + if (dp->usb_prepared) { + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_shutdown); + if (!ret) + dp->usb_prepared = false; + } else if (dp->enabled) { + /* Multifunction has a distinct native shutdown; never run the + * USB-only analog sequence over live DisplayPort lane state. */ + ret = atc_stop(dp); + } + return ret; +} + +static int combo_stop(struct t6050_dp *dp) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->faulted) + return -EIO; + if (dp->controller_active || dp->pipe_active) + return -EBUSY; + ret = combo_quiesce(dp); + if (!ret && dp->usb2_active) { + ret = apple_t6050_usb2_stop(&dp->usb_platform); + if (!ret) + dp->usb2_active = false; + } + if (!ret) + ret = atc_domain_set(&dp->domain[1], false); + if (!ret) + ret = atc_domain_set(&dp->domain[0], false); + if (ret) { + dp->faulted = true; + return ret < 0 ? ret : -EIO; + } + dp->combo_mode = COMBO_SAFE; + return 0; +} + +static int combo_start(struct t6050_dp *dp, enum combo_mode mode) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->faulted) + return -EIO; + if (dp->orientation == TYPEC_ORIENTATION_NONE) + return -ENOLINK; + ret = atc_domain_set(&dp->domain[0], true); + if (!ret) + ret = atc_domain_set(&dp->domain[1], true); + if (!ret) + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); + if (!ret) + dp->usb2_active = true; + if (!ret && mode == COMBO_USB) { + ret = dp->reverse ? + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_init_reverse) : + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_init_normal); + if (!ret) + dp->usb_prepared = true; + } else if (!ret) { + dp->multifunction = mode == COMBO_DP2; + ret = atc_start(dp); + } + if (ret) { + /* A partial shared transition has no cold-start ownership proof. */ + dp->faulted = true; + return ret < 0 ? ret : -EIO; + } + dp->combo_mode = mode; + return 0; +} + +static int combo_orientation(struct t6050_dp *dp, enum typec_orientation orientation) +{ + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (orientation != TYPEC_ORIENTATION_NONE && orientation != TYPEC_ORIENTATION_NORMAL && + orientation != TYPEC_ORIENTATION_REVERSE) + return -EINVAL; + if (dp->orientation == orientation) + return 0; + ret = combo_stop(dp); + if (!ret) { + dp->orientation = orientation; + dp->reverse = orientation == TYPEC_ORIENTATION_REVERSE; + } + return ret; +} + +static int combo_mux(struct t6050_dp *dp, struct typec_mux_state *state) +{ + enum combo_mode mode; + int ret; + + if (state->mode == TYPEC_STATE_SAFE) + mode = COMBO_SAFE; + else if (state->mode == TYPEC_STATE_USB && !state->alt) + mode = COMBO_USB; + else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: mode = COMBO_DP4; break; + case TYPEC_DP_STATE_D: mode = COMBO_DP2; break; + default: return -EOPNOTSUPP; + } + } else { + return -EOPNOTSUPP; + } + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode == dp->combo_mode && + (mode == COMBO_SAFE || dp->usb_prepared || dp->enabled)) + return 0; + if (mode != COMBO_SAFE && dp->orientation == TYPEC_ORIENTATION_NONE) + return -ENOLINK; + ret = combo_stop(dp); + if (ret || mode == COMBO_SAFE) + return ret; + return combo_start(dp, mode); +} + +static int combo_usb2_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode != PHY_MODE_USB_HOST) + return -EOPNOTSUPP; + if (!dp->usb_prepared && !dp->enabled) + return -ENOLINK; + if (dp->usb2_active) + return 0; + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); + if (ret) + dp->faulted = true; + else + dp->usb2_active = true; + return ret; +} + +static int combo_usb2_reset(struct phy *phy) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted || !dp->usb2_active) + return -ENOLINK; + ret = apple_t6050_usb2_port_reset(&dp->usb_io, dp, combo_sleep); + if (ret) + dp->faulted = true; + return ret ? -EIO : 0; +} + +static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (mode != PHY_MODE_USB_HOST) + return -EOPNOTSUPP; + if (!dp->controller_active || !dp->usb2_active) + return -ENOLINK; + /* Four DP lanes leave DWC3 on its established dummy PIPE; USB2 still + * uses its independent D+/D- PHY. Do not switch USB3 onto DP lanes. */ + if (dp->combo_mode == COMBO_DP4 && dp->enabled) + return 0; + if (!dp->usb_prepared && !(dp->combo_mode == COMBO_DP2 && dp->enabled)) + return -ENOLINK; + if (dp->pipe_active) + return (readl(dp->banks[44] + 0xc) & 0x73) == 0x10 ? 0 : -EIO; + apple_tunable_apply(dp->banks[44], dp->pipe_default); + ret = apple_t6050_usb3_pipe_switch(&dp->usb_io); + if (ret) + dp->faulted = true; + else + dp->pipe_active = true; + return ret ? -EIO : 0; +} + +static int combo_usb3_poweroff(struct phy *phy) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + int ret = 0; + + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + /* DWC3 calls power_off only after host/gadget shutdown and SUSPHY. + * PIPE must select its dummy producer before the controller is reset. */ + if (dp->pipe_active) { + ret = apple_t6050_usb3_pipe_stop(&dp->usb_io, combo_quiesce, dp); + if (!ret) + dp->pipe_active = false; + } + if (!ret && dp->usb2_active) { + ret = apple_t6050_usb2_stop(&dp->usb_platform); + if (!ret) + dp->usb2_active = false; + } + if (ret) + dp->faulted = true; + return ret ? -EIO : 0; +} +static const struct phy_ops combo_usb2_ops = { + .set_mode = combo_usb2_mode, .reset = combo_usb2_reset, .owner = THIS_MODULE, +}; +static const struct phy_ops combo_usb3_ops = { + .set_mode = combo_usb3_mode, .power_off = combo_usb3_poweroff, .owner = THIS_MODULE, +}; + +static int combo_reset_assert(struct reset_controller_dev *reset, unsigned long id) +{ + struct t6050_dp *dp = container_of(reset, struct t6050_dp, reset); + int ret; + + guard(mutex)(&dp->lock); + if (id || dp->pipe_active) + return -EBUSY; + if (dp->faulted) + return -EIO; + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 1, 0); + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0x10); + ret = apple_t6050_usb3_pipe_cold_dummy(&dp->usb_io); + if (ret) + dp->faulted = true; + else + dp->controller_active = false; + return ret ? -EIO : 0; +} +static int combo_reset_deassert(struct reset_controller_dev *reset, unsigned long id) +{ + struct t6050_dp *dp = container_of(reset, struct t6050_dp, reset); + + guard(mutex)(&dp->lock); + if (id) + return -EINVAL; + if (dp->faulted || !dp->usb2_active) + return -ENOLINK; + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0); + apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 1, 1); + dp->controller_active = true; + return 0; +} +static const struct reset_control_ops combo_reset_ops = { + .assert = combo_reset_assert, .deassert = combo_reset_deassert, +}; + +static struct phy *combo_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct t6050_dp *dp = dev_get_drvdata(dev); + if (args->args_count != 1) + return ERR_PTR(-EINVAL); + switch (args->args[0]) { + case PHY_TYPE_USB2: return dp->combo_phy[0]; + case PHY_TYPE_USB3: return dp->combo_phy[1]; + case PHY_TYPE_DP: return dp->combo_phy[2]; + default: return ERR_PTR(-EINVAL); + } +} + +static int combo_setup(struct platform_device *pdev, struct t6050_dp *dp) +{ + static const struct { u8 bank; u32 minimum; const char *name; } extra[] = { + {0, 0x20, "usb2phy"}, {1, 0x24, "usb2-digital"}, {44, 0x48, "pipehandler"}, + }; + struct device *dev = dp->dev; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(extra); i++) { + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, extra[i].name); + if (!res || resource_size(res) < extra[i].minimum) + return -EINVAL; + dp->banks[extra[i].bank] = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->banks[extra[i].bank])) + return PTR_ERR(dp->banks[extra[i].bank]); + dp->resources[extra[i].bank] = *res; + } + dp->usb2_default = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-usb2-default", &dp->resources[0]); + if (IS_ERR(dp->usb2_default)) + return PTR_ERR(dp->usb2_default); + dp->usb2_host = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-usb2-host", &dp->resources[0]); + if (IS_ERR(dp->usb2_host)) + return PTR_ERR(dp->usb2_host); + dp->pipe_default = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-pipe-default", &dp->resources[44]); + if (IS_ERR(dp->pipe_default)) + return PTR_ERR(dp->pipe_default); + dp->usb_io = (struct apple_t6050_usb_io) { + .context = dp, .read = combo_read, .write = combo_write, + .poll = combo_poll, .delay = combo_delay, + }; + dp->usb_platform = (struct apple_t6050_usb_platform) { + .context = dp, .io = &dp->usb_io, .wrapper_power = combo_wrapper, + .firmware = combo_firmware, .tunable = combo_tunable, .pause_ns = combo_pause, + }; + dp->reset = (struct reset_controller_dev) { + .owner = THIS_MODULE, .ops = &combo_reset_ops, .dev = dev, + .of_node = dev->of_node, .nr_resets = 1, + }; + ret = devm_reset_controller_register(dev, &dp->reset); + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) { + dp->combo_phy[i] = devm_phy_create(dev, NULL, i ? &combo_usb3_ops : &combo_usb2_ops); + if (IS_ERR(dp->combo_phy[i])) + return PTR_ERR(dp->combo_phy[i]); + phy_set_drvdata(dp->combo_phy[i], dp); + } + return 0; +} diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c new file mode 100644 index 0000000000000..cafee096a6250 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -0,0 +1,806 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* T6050 DisplayPort PHY for fixed bridges and Type-C DP alternate mode. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "usb-t6050/apple_t6050_usb_phy.h" +#include "usb-t6050/apple_t6050_usb3_pipe.h" +#include "usb-t6050/apple_t6050_usb2_reset.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum atc_operation { ATC_MASK, ATC_TUNABLE, ATC_DELAY, ATC_POLL, ATC_POWER, ATC_FIRMWARE }; +struct atc_op { + u8 op, bank; + u16 offset; + u32 mask, value, timeout; +}; +struct atc_rate_op { + struct atc_op op; + u8 rates; + u32 values[4]; +}; +#include "atc-t6050-dp-data.h" +#include "atc-t6050-typec-data.h" + +static const struct { u8 bank; u32 offset; } core_banks[] = { + {2, 0}, {3, 0x400}, {4, 0x800}, {5, 0xa00}, {6, 0x1000}, + {7, 0x3000}, {8, 0x3200}, {9, 0x3800}, {10, 0x3a00}, + {14, 0x7000}, {16, 0x9000}, {18, 0xb000}, {19, 0xc000}, + {20, 0xd000}, {21, 0xe000}, {22, 0xf000}, {25, 0x19000}, + {27, 0x1b000}, {28, 0x1c000}, {29, 0x1d000}, {30, 0x1e000}, + {31, 0x1f000}, {33, 0x28000}, {34, 0x28400}, {35, 0x29000}, +}; +static const struct { u8 bank; const char *name; } extra_banks[] = { + {39, "reset"}, {40, "fabric"}, {41, "axi2af"}, + {42, "lioa"}, {43, "common-cfg"}, + {38, "usb-control"}, +}; +static const struct { u8 bank; const char *name; } tunable_names[] = { + {41, "apple,tunable-axi2af"}, {40, "apple,tunable-fabric"}, + {43, "apple,tunable-common-cfg"}, {42, "apple,tunable-lioa"}, + {20, "apple,tunable-lane0-rx-clock"}, {29, "apple,tunable-lane1-rx-clock"}, + {18, "apple,tunable-lane0-rx-eq"}, {27, "apple,tunable-lane1-rx-eq"}, + {8, "apple,tunable-auspll"}, + {5, "apple,tunable-aus40cmn-shm"}, {10, "apple,tunable-cio4pll"}, +}; +static const unsigned int link_rates[] = {1620, 2700, 5400, 8100}; +static const char * const training_names[] = { + "apple,dp-training-table-rbr", "apple,dp-training-table-hbr", + "apple,dp-training-table-hbr2", "apple,dp-training-table-hbr3", +}; +struct atc_domain { + struct device *dev; + bool active; +}; +enum combo_mode { COMBO_SAFE, COMBO_USB, COMBO_DP4, COMBO_DP2 }; +struct t6050_dp { + bool combo, usb_prepared, usb2_active, pipe_active, controller_active; + enum combo_mode combo_mode; + enum typec_orientation orientation; + struct phy *combo_phy[3]; + struct reset_controller_dev reset; + struct apple_t6050_usb_io usb_io; + struct apple_t6050_usb_platform usb_platform; + struct apple_tunable *usb2_default, *usb2_host, *pipe_default; + struct device *dev; + struct mutex lock; + void __iomem *banks[45]; + struct resource resources[45]; + struct apple_tunable *tunables[ARRAY_SIZE(tunable_names)]; + struct atc_domain domain[3]; /* USB AON, common, wrapper: native order. */ + u32 training[4][16]; + u32 voltage[4], pre[4]; + int rate; + bool enabled, faulted; + bool typec, reverse, multifunction; + struct apple_rtkit *rtkit; + void __iomem *fw_data; + bool fw_running; +}; + +static int atc_domain_set(struct atc_domain *pd, bool active) +{ + int ret; + + if (pd->active == active) + return 0; + if (active) { + ret = pm_runtime_resume_and_get(pd->dev); + if (ret < 0) + return ret; + } else { + ret = pm_runtime_put_sync_suspend(pd->dev); + if (ret < 0) { + /* put decremented usage even when suspend failed. */ + pm_runtime_get_noresume(pd->dev); + return ret; + } + } + pd->active = active; + return 0; +} + +static void atc_domain_release(void *data) +{ + struct atc_domain *pd = data; + + if (pd->active) + pm_runtime_put_sync_suspend(pd->dev); + dev_pm_domain_detach(pd->dev, true); +} + +/* The bootloader preloads CIO80 firmware into local SRAM. The firmware's + * data segment is 0x10000000..0x10010000, not guest system RAM. */ +static int atc_fw_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct t6050_dp *dp = cookie; + + if (bfr->iova < 0x10000000 || + bfr->size > 0x10000 || bfr->iova - 0x10000000 > 0x10000 - bfr->size) { + dev_err(dp->dev, "unsupported PHY firmware buffer %llx size %zx\n", + (unsigned long long)bfr->iova, bfr->size); + return -EINVAL; + } + bfr->iomem = dp->fw_data + (bfr->iova - 0x10000000); + bfr->is_mapped = true; + return 0; +} + +static void atc_fw_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + /* SRAM is owned by the PHY and mapped for the device lifetime. */ +} + +static void atc_fw_message(void *cookie, u8 endpoint, u64 message) +{ + struct t6050_dp *dp = cookie; + + dev_dbg(dp->dev, "PHY firmware ep%x: %016llx\n", endpoint, message); +} + +static void atc_fw_crashed(void *cookie, const void *log, size_t size) +{ + struct t6050_dp *dp = cookie; + + WRITE_ONCE(dp->faulted, true); + dev_err(dp->dev, "PHY firmware crashed\n"); +} + +static const struct apple_rtkit_ops atc_fw_ops = { + .shmem_setup = atc_fw_shmem_setup, + .shmem_destroy = atc_fw_shmem_destroy, + .recv_message = atc_fw_message, + .crashed = atc_fw_crashed, +}; + +static int atc_firmware(struct t6050_dp *dp, bool start) +{ + int ret; + + if (!dp->rtkit) + return -ENODEV; + if (dp->fw_running == start) + return 0; + if (start) { + /* Native coprocessor run, after its +0x40 firmware enable. */ + writel(BIT(4), dp->banks[38] + 0x28); + ret = apple_rtkit_wake(dp->rtkit); + if (!ret) + ret = apple_rtkit_start_ep(dp->rtkit, 0x20); + } else { + /* CIO80PhyMBI::sendPowerDown, then the RTBuddy power transition. */ + ret = apple_rtkit_send_message(dp->rtkit, 0x20, BIT_ULL(56), NULL, false); + if (!ret) + ret = apple_rtkit_shutdown(dp->rtkit); + if (!ret) + writel((u32)~BIT(4), dp->banks[38] + 0x28); + } + if (ret) + dev_err(dp->dev, "PHY firmware %s failed: %d\n", start ? "start" : "stop", ret); + else + dp->fw_running = start; + return ret; +} + +static int atc_validate_step(struct t6050_dp *dp, const struct atc_op *s) +{ + if (s->op == ATC_DELAY) + return s->value <= 10000 ? 0 : -EINVAL; + if (s->op == ATC_POWER) + return s->value <= 1 ? 0 : -EINVAL; + if (s->op == ATC_FIRMWARE) + return dp->typec && s->value <= 1 ? 0 : -EINVAL; + if (s->bank >= ARRAY_SIZE(dp->banks) || !dp->banks[s->bank] || + s->offset % 4 || s->offset + 4 > resource_size(&dp->resources[s->bank])) + return -EINVAL; + if (s->op == ATC_TUNABLE) + return s->value < ARRAY_SIZE(tunable_names) && + tunable_names[s->value].bank == s->bank ? 0 : -EINVAL; + if (s->op == ATC_MASK || s->op == ATC_POLL) + return s->value & ~s->mask ? -EINVAL : 0; + return -EINVAL; +} + +static int atc_step(struct t6050_dp *dp, const struct atc_op *s) +{ + void __iomem *reg; + u32 value; + int ret = atc_validate_step(dp, s); + + if (ret) + return ret; + if (s->op == ATC_DELAY) { + udelay(s->value); + return 0; + } + if (s->op == ATC_FIRMWARE) + return atc_firmware(dp, s->value); + if (s->op == ATC_POWER) + return atc_domain_set(&dp->domain[2], s->value); + if (s->bank >= ARRAY_SIZE(dp->banks) || !dp->banks[s->bank] || + s->offset + 4 > resource_size(&dp->resources[s->bank])) + return -EINVAL; + reg = dp->banks[s->bank] + s->offset; + switch (s->op) { + case ATC_MASK: + value = readl(reg); + writel((value & ~s->mask) | s->value, reg); + return 0; + case ATC_TUNABLE: + if (s->value >= ARRAY_SIZE(dp->tunables)) + return -EINVAL; + apple_tunable_apply(dp->banks[s->bank], dp->tunables[s->value]); + return 0; + case ATC_POLL: + ret = readl_poll_timeout(reg, value, (value & s->mask) == s->value, + s->timeout > 1000 ? 1000 : 1, s->timeout); + if (ret) + dev_err(dp->dev, "bank%u+%x timeout: %x & %x != %x\n", + s->bank, s->offset, value, s->mask, s->value); + return ret; + default: + return -EINVAL; + } +} + +static int atc_sequence(struct t6050_dp *dp, const struct atc_op *s, size_t count) +{ + for (size_t i = 0; i < count; i++) { + int ret = atc_step(dp, &s[i]); + + if (ret) { + dp->faulted = true; + return ret; + } + } + return 0; +} +#define ATC_SEQUENCE(dp, seq) atc_sequence(dp, seq, ARRAY_SIZE(seq)) + +static int atc_link_rate(struct t6050_dp *dp, int rate) +{ + const struct atc_rate_op *on = dp_link_on; + size_t nr_on = ARRAY_SIZE(dp_link_on); + int ret; + + if (dp->multifunction) { + on = dp->reverse ? dp_group0_on : dp_group1_on; + nr_on = dp->reverse ? ARRAY_SIZE(dp_group0_on) : ARRAY_SIZE(dp_group1_on); + } + if (dp->rate == rate) + return 0; + if (dp->rate >= 0) { + if (dp->multifunction) + ret = dp->reverse ? ATC_SEQUENCE(dp, dp_group0_off) : + ATC_SEQUENCE(dp, dp_group1_off); + else + ret = ATC_SEQUENCE(dp, dp_link_off); + if (ret) + return ret; + dp->rate = -1; + } + if (rate < 0) + return 0; + for (size_t i = 0; i < nr_on; i++) { + struct atc_op op = on[i].op; + + if (!(on[i].rates & BIT(rate))) + continue; + op.value = on[i].values[rate]; + ret = atc_step(dp, &op); + if (ret) { + dp->faulted = true; + return ret; + } + } + dp->rate = rate; + memset(dp->voltage, 0, sizeof(dp->voltage)); + memset(dp->pre, 0, sizeof(dp->pre)); + return 0; +} + +static int atc_start(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (dp->enabled) + return 0; + ret = atc_domain_set(&dp->domain[0], true); + if (ret) + return ret; + ret = atc_domain_set(&dp->domain[1], true); + if (ret) { + atc_domain_set(&dp->domain[0], false); + return ret; + } + if (dp->multifunction) + ret = dp->reverse ? ATC_SEQUENCE(dp, mf_reverse_init) : + ATC_SEQUENCE(dp, mf_normal_init); + else + ret = dp->reverse ? ATC_SEQUENCE(dp, dp_reverse_init) : + ATC_SEQUENCE(dp, dp_init); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_on); + if (ret) + return ret; + dp->enabled = true; + return 0; +} + +static int atc_stop(struct t6050_dp *dp) +{ + int ret; + + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return 0; + ret = atc_link_rate(dp, -1); + if (!ret) + ret = ATC_SEQUENCE(dp, aux_off); + if (!ret) + ret = dp->multifunction ? ATC_SEQUENCE(dp, mf_poweroff) : + ATC_SEQUENCE(dp, dp_poweroff); + if (ret) + return ret; + dp->enabled = false; + /* Combo owns domain shutdown after USB2 and PIPE have also quiesced. */ + if (dp->combo) + return 0; + ret = atc_domain_set(&dp->domain[1], false); + if (!ret) + ret = atc_domain_set(&dp->domain[0], false); + return ret; +} + +static void atc_mask(void __iomem *reg, u32 mask, u32 value) +{ + writel((readl(reg) & ~mask) | value, reg); +} + +static void atc_drive(struct t6050_dp *dp, unsigned int lane, u32 preset) +{ + /* Native DP lane order is RX/TX in each orientation-selected group. */ + unsigned int group = (lane < 2 ? 1 : 0) ^ dp->reverse; + bool tx = lane & 1; + void __iomem *base = dp->banks[(tx ? 21 : 20) + 9 * group]; + u32 ctrl = ((preset >> 2) & 0xf) | ((preset & 3) << 14); + u32 eq = ((preset >> 5) & 0x7fffe) | 1; + + if (tx) + atc_mask(base + 0x100, BIT(20), BIT(20)); + atc_mask(base + (tx ? 0x58 : 0x68), 0xc00f, ctrl); + atc_mask(base + (tx ? 0x104 : 0xac), BIT(29), BIT(29)); + atc_mask(base + (tx ? 0x100 : 0xa8), 0x7ffff, eq); +} + +#include "atc-t6050-combo.h" + +static int t6050_dp_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + + guard(mutex)(&dp->lock); + if (mode == PHY_MODE_DP) + return dp->typec ? (dp->enabled ? 0 : -ENOLINK) : atc_start(dp); + if (mode == PHY_MODE_INVALID) + return dp->combo ? atc_link_rate(dp, -1) : atc_stop(dp); + return -EINVAL; +} + +static int t6050_dp_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + + guard(mutex)(&dp->lock); + if (mode != PHY_MODE_DP) + return -EINVAL; + memset(opts, 0, sizeof(*opts)); + opts->dp.lanes = dp->multifunction ? 2 : 4; + opts->dp.link_rate = 8100; + opts->dp.ssc = 0; + return 0; +} + +static int t6050_dp_configure(struct phy *phy, union phy_configure_opts *opts) +{ + struct t6050_dp *dp = phy_get_drvdata(phy); + struct phy_configure_opts_dp *cfg = &opts->dp; + int rate = -1, ret; + + /* Validate the whole request before altering a running transmitter. */ + if (cfg->set_rate && cfg->link_rate) { + for (unsigned int i = 0; i < ARRAY_SIZE(link_rates); i++) + if (cfg->link_rate == link_rates[i]) + rate = i; + if (rate < 0 || cfg->ssc) + return -EINVAL; + } + if ((cfg->set_lanes || cfg->set_voltages) && + cfg->lanes != 0 && cfg->lanes != 1 && cfg->lanes != 2 && cfg->lanes != 4) + return -EINVAL; + if (cfg->set_voltages) + for (unsigned int i = 0; i < cfg->lanes; i++) + if (cfg->voltage[i] > 3 || cfg->pre[i] > 3 || + cfg->voltage[i] + cfg->pre[i] > 3) + return -EINVAL; + if (cfg->set_voltages && cfg->set_rate && rate < 0 && cfg->lanes) + return -EINVAL; + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled) + return -EPIPE; + if ((cfg->set_lanes || cfg->set_voltages) && dp->multifunction && cfg->lanes > 2) + return -EINVAL; + if (cfg->set_rate) { + ret = atc_link_rate(dp, rate); + if (ret) + return ret; + } + /* Native setActiveLaneCount is a no-op; DCP controls the active count. */ + if (cfg->set_voltages) { + if (dp->rate < 0) + return -EPIPE; + for (unsigned int i = 0; i < cfg->lanes; i++) { + u32 preset = dp->training[dp->rate][4 * cfg->voltage[i] + cfg->pre[i]]; + + atc_drive(dp, i, preset); + dp->voltage[i] = cfg->voltage[i]; + dp->pre[i] = cfg->pre[i]; + } + } + return 0; +} + +static const struct phy_ops t6050_dp_ops = { + .set_mode = t6050_dp_set_mode, + .validate = t6050_dp_validate, + .configure = t6050_dp_configure, + .owner = THIS_MODULE, +}; + +int apple_dp_phy_link_config(struct phy *phy, unsigned int *pclk, bool *uhbr) +{ + struct t6050_dp *dp; + + if (!phy || phy->ops != &t6050_dp_ops) + return -EOPNOTSUPP; + dp = phy_get_drvdata(phy); + guard(mutex)(&dp->lock); + if (dp->faulted) + return -EIO; + if (!dp->enabled || dp->rate < 0) + return -ENOLINK; + /* Native direct DPPHY uses logical clock0, mapped to xbar PCLK1. + * The other four logical clocks are separately managed tunnel clocks. */ + *pclk = 1; + *uhbr = false; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dp_phy_link_config); + +/* The connector sets orientation before changing the alternate-mode mux. */ +static int t6050_set_orientation(struct t6050_dp *dp, + enum typec_orientation orientation) +{ + bool reverse = orientation == TYPEC_ORIENTATION_REVERSE; + int ret; + + if (dp->combo) + return combo_orientation(dp, orientation); + guard(mutex)(&dp->lock); + if (orientation == TYPEC_ORIENTATION_NONE) + return atc_stop(dp); + if (dp->reverse == reverse) + return 0; + ret = atc_stop(dp); + if (!ret) + dp->reverse = reverse; + return ret; +} + +static int t6050_set_typec_mode(struct t6050_dp *dp, struct typec_mux_state *state) +{ + bool multifunction; + int ret; + + if (dp->combo) + return combo_mux(dp, state); + guard(mutex)(&dp->lock); + if (state->mode == TYPEC_STATE_SAFE) + return atc_stop(dp); + /* USB-only and USB4 require their controller's separate PHY interface. */ + if (!state->alt || state->alt->svid != USB_TYPEC_DP_SID) + return -EOPNOTSUPP; + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + multifunction = false; + break; + case TYPEC_DP_STATE_D: + multifunction = true; + break; + default: + return -EOPNOTSUPP; + } + if (dp->enabled && dp->multifunction == multifunction) + return 0; + ret = atc_stop(dp); + if (ret) + return ret; + dp->multifunction = multifunction; + ret = atc_start(dp); + if (!ret) + dev_info(dp->dev, "Type-C DP: %u lanes, %s orientation\n", + multifunction ? 2 : 4, dp->reverse ? "reverse" : "normal"); + return ret; +} + +static int t6050_typec_orientation(struct typec_switch_dev *sw, + enum typec_orientation orientation) +{ + return t6050_set_orientation(typec_switch_get_drvdata(sw), orientation); +} + +static int t6050_typec_mode(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + return t6050_set_typec_mode(typec_mux_get_drvdata(mux), state); +} + +static void t6050_typec_switch_release(void *data) +{ + typec_switch_unregister(data); +} + +static void t6050_typec_mux_release(void *data) +{ + typec_mux_unregister(data); +} + +static int t6050_typec_register(struct t6050_dp *dp) +{ + struct typec_switch_desc sw_desc = { + .fwnode = dev_fwnode(dp->dev), .drvdata = dp, + .set = t6050_typec_orientation, + }; + struct typec_mux_desc mux_desc = { + .fwnode = dev_fwnode(dp->dev), .drvdata = dp, + .set = t6050_typec_mode, + }; + struct typec_switch_dev *sw; + struct typec_mux_dev *mux; + int ret; + + sw = typec_switch_register(dp->dev, &sw_desc); + if (IS_ERR(sw)) + return PTR_ERR(sw); + ret = devm_add_action_or_reset(dp->dev, t6050_typec_switch_release, sw); + if (ret) + return ret; + mux = typec_mux_register(dp->dev, &mux_desc); + if (IS_ERR(mux)) + return PTR_ERR(mux); + return devm_add_action_or_reset(dp->dev, t6050_typec_mux_release, mux); +} + +static int t6050_dp_probe(struct platform_device *pdev) +{ + static const char * const domains[] = {"usb-aon", "common", "wrapper"}; + struct device *dev = &pdev->dev; + struct t6050_dp *dp; + struct phy_provider *provider; + struct phy *phy; + struct resource *core; + void __iomem *base; + int ret; + + dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL); + if (!dp) + return -ENOMEM; + dp->dev = dev; + dp->combo = of_device_is_compatible(dev->of_node, "apple,t6050-combo-phy"); + if (dp->combo) { + u32 port; + if (of_property_read_u32(dev->of_node, "apple,port-index", &port) || + (port != 1 && port != 2) || + !of_property_present(dev->of_node, "apple,guest-port-reserved") || + !of_property_present(dev->of_node, "mode-switch")) + return -EINVAL; + } + dp->typec = of_property_present(dev->of_node, "mode-switch"); + dp->rate = -1; + mutex_init(&dp->lock); + platform_set_drvdata(pdev, dp); + core = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core"); + if (!core || resource_size(core) < (dp->combo ? 0x2c000 : 0x2c400)) + return -EINVAL; + base = devm_ioremap_resource(dev, core); + if (IS_ERR(base)) + return PTR_ERR(base); + for (unsigned int i = 0; i < ARRAY_SIZE(core_banks); i++) { + u8 bank = core_banks[i].bank; + + dp->banks[bank] = base + core_banks[i].offset; + dp->resources[bank] = *core; + dp->resources[bank].start += core_banks[i].offset; + dp->resources[bank].end = core->end; + } + for (unsigned int i = 0; i < ARRAY_SIZE(extra_banks); i++) { + u8 bank = extra_banks[i].bank; + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + extra_banks[i].name); + + if (bank == 38 && !dp->typec) + continue; + if (!res || resource_size(res) < 4) + return -EINVAL; + dp->banks[bank] = devm_ioremap_resource(dev, res); + if (IS_ERR(dp->banks[bank])) + return PTR_ERR(dp->banks[bank]); + dp->resources[bank] = *res; + } + /* Check every register step before any domain is enabled. */ + { + static const struct { const struct atc_op *ops; size_t count; } sequences[] = { + { dp_init, ARRAY_SIZE(dp_init) }, + { aux_on, ARRAY_SIZE(aux_on) }, + { aux_off, ARRAY_SIZE(aux_off) }, + { dp_link_off, ARRAY_SIZE(dp_link_off) }, + { dp_poweroff, ARRAY_SIZE(dp_poweroff) }, + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(sequences); i++) + for (size_t j = 0; j < sequences[i].count; j++) { + ret = atc_validate_step(dp, &sequences[i].ops[j]); + if (ret) + return ret; + } + for (unsigned int i = 0; i < ARRAY_SIZE(dp_link_on); i++) + for (unsigned int rate = 0; rate < ARRAY_SIZE(link_rates); rate++) { + struct atc_op op = dp_link_on[i].op; + + if (!(dp_link_on[i].rates & BIT(rate))) + continue; + op.value = dp_link_on[i].values[rate]; + ret = atc_validate_step(dp, &op); + if (ret) + return ret; + } + } + if (dp->typec) { + static const struct { const struct atc_op *ops; size_t count; } sequences[] = { + { dp_reverse_init, ARRAY_SIZE(dp_reverse_init) }, + { mf_normal_init, ARRAY_SIZE(mf_normal_init) }, + { mf_reverse_init, ARRAY_SIZE(mf_reverse_init) }, + { mf_poweroff, ARRAY_SIZE(mf_poweroff) }, + { dp_group0_off, ARRAY_SIZE(dp_group0_off) }, + { dp_group1_off, ARRAY_SIZE(dp_group1_off) }, + }; + const struct atc_rate_op *rates[] = { dp_group0_on, dp_group1_on }; + const size_t counts[] = { ARRAY_SIZE(dp_group0_on), ARRAY_SIZE(dp_group1_on) }; + + for (unsigned int i = 0; i < ARRAY_SIZE(sequences); i++) + for (size_t j = 0; j < sequences[i].count; j++) { + ret = atc_validate_step(dp, &sequences[i].ops[j]); + if (ret) + return ret; + } + for (unsigned int group = 0; group < ARRAY_SIZE(rates); group++) + for (size_t i = 0; i < counts[group]; i++) + for (unsigned int rate = 0; rate < ARRAY_SIZE(link_rates); rate++) { + struct atc_op op = rates[group][i].op; + if (!(rates[group][i].rates & BIT(rate))) + continue; + op.value = rates[group][i].values[rate]; + ret = atc_validate_step(dp, &op); + if (ret) + return ret; + } + } + for (unsigned int i = 0; i < ARRAY_SIZE(tunable_names); i++) { + if (i >= 9 && !dp->typec) + continue; + dp->tunables[i] = devm_apple_tunable_parse(dev, dev->of_node, + tunable_names[i].name, &dp->resources[tunable_names[i].bank]); + if (IS_ERR(dp->tunables[i])) + return dev_err_probe(dev, PTR_ERR(dp->tunables[i]), + "missing or invalid %s\n", tunable_names[i].name); + } + for (unsigned int i = 0; i < ARRAY_SIZE(training_names); i++) { + const char *name = training_names[i]; + + if (!of_property_present(dev->of_node, name)) + name = "apple,dp-training-table"; + if (!of_property_present(dev->of_node, name)) { + memcpy(dp->training[i], dp_training_default, sizeof(dp_training_default)); + continue; + } + if (of_property_count_u32_elems(dev->of_node, name) != 16) + return -EINVAL; + ret = of_property_read_u32_array(dev->of_node, name, dp->training[i], 16); + if (ret) + return ret; + } + /* Three named domains prevent automatic wrapper enable at probe time. */ + if (of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells") != 3) + return -EINVAL; + for (unsigned int i = 0; i < ARRAY_SIZE(domains); i++) { + dp->domain[i].dev = dev_pm_domain_attach_by_name(dev, domains[i]); + if (IS_ERR_OR_NULL(dp->domain[i].dev)) + return dev_err_probe(dev, dp->domain[i].dev ? PTR_ERR(dp->domain[i].dev) : -ENODEV, + "cannot attach %s domain\n", domains[i]); + ret = devm_add_action_or_reset(dev, atc_domain_release, &dp->domain[i]); + if (ret) + return ret; + } + if (dp->typec) { + dp->fw_data = devm_platform_ioremap_resource_byname(pdev, "firmware-data"); + if (IS_ERR(dp->fw_data)) + return PTR_ERR(dp->fw_data); + dp->rtkit = devm_apple_rtkit_init(dev, dp, NULL, 0, &atc_fw_ops); + if (IS_ERR(dp->rtkit)) + return dev_err_probe(dev, PTR_ERR(dp->rtkit), "cannot initialize PHY firmware IPC\n"); + } + if (dp->combo) { + ret = combo_setup(pdev, dp); + if (ret) + return ret; + } + phy = devm_phy_create(dev, NULL, &t6050_dp_ops); + if (IS_ERR(phy)) + return PTR_ERR(phy); + phy_set_drvdata(phy, dp); + dp->combo_phy[2] = phy; + provider = devm_of_phy_provider_register(dev, dp->combo ? combo_xlate : of_phy_simple_xlate); + if (IS_ERR(provider)) + return PTR_ERR(provider); + return dp->typec ? t6050_typec_register(dp) : 0; +} + +static void t6050_dp_shutdown(struct platform_device *pdev) +{ + struct t6050_dp *dp = platform_get_drvdata(pdev); + + guard(mutex)(&dp->lock); + if (dp->combo ? combo_stop(dp) : atc_stop(dp)) + dev_warn(dp->dev, "PHY shutdown incomplete after an earlier error\n"); +} + +static const struct of_device_id t6050_dp_match[] = { + { .compatible = "apple,t6050-dp-phy" }, + { .compatible = "apple,t6050-combo-phy" }, + {}, +}; +MODULE_DEVICE_TABLE(of, t6050_dp_match); +static struct platform_driver t6050_dp_driver = { + .probe = t6050_dp_probe, + .remove = t6050_dp_shutdown, + .shutdown = t6050_dp_shutdown, + .driver = { + .name = "apple-t6050-dp-phy", + .of_match_table = t6050_dp_match, + }, +}; +module_platform_driver(t6050_dp_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple T6050 DisplayPort and Type-C USB combo PHY"); diff --git a/drivers/phy/apple/atc-t6050-typec-data.h b/drivers/phy/apple/atc-t6050-typec-data.h new file mode 100644 index 0000000000000..096307b657c08 --- /dev/null +++ b/drivers/phy/apple/atc-t6050-typec-data.h @@ -0,0 +1,700 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Native firmware27 Type-C DP recipes; board tunables remain DT data. */ +static const struct atc_op dp_reverse_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x3c000fc, 0x48, 0 }, + { ATC_MASK, 2, 0x78, 0x1f, 0x14, 0 }, + { ATC_MASK, 2, 0x78, 0x3ffe0, 0x100, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_normal_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_TUNABLE, 5, 0x0, 0x0, 0x9, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_TUNABLE, 10, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x28000fc, 0x44, 0 }, + { ATC_MASK, 2, 0x78, 0x1ffff, 0x110, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x10000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x100, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x1000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x400, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 38, 0x40, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1388, 0 }, + { ATC_POLL, 2, 0x260, 0x400, 0x400, 1000000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_reverse_init[] = { + { ATC_MASK, 39, 0x0, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x1, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x2, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x8, 0 }, + { ATC_TUNABLE, 41, 0x0, 0x0, 0x0, 0 }, + { ATC_TUNABLE, 40, 0x0, 0x0, 0x1, 0 }, + { ATC_TUNABLE, 43, 0x0, 0x0, 0x2, 0 }, + { ATC_TUNABLE, 42, 0x0, 0x0, 0x3, 0 }, + { ATC_TUNABLE, 20, 0x0, 0x0, 0x4, 0 }, + { ATC_TUNABLE, 29, 0x0, 0x0, 0x5, 0 }, + { ATC_TUNABLE, 18, 0x0, 0x0, 0x6, 0 }, + { ATC_TUNABLE, 27, 0x0, 0x0, 0x7, 0 }, + { ATC_TUNABLE, 5, 0x0, 0x0, 0x9, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_TUNABLE, 10, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 4, 0x0, 0x3f0, 0x1c0, 0 }, + { ATC_MASK, 4, 0x18, 0xf000, 0x3000, 0 }, + { ATC_MASK, 9, 0x88, 0xff, 0x27, 0 }, + { ATC_MASK, 33, 0x4, 0x38, 0x8, 0 }, + { ATC_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0 }, + { ATC_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 20, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_MASK, 29, 0x90, 0x1f80000, 0x380000, 0 }, + { ATC_TUNABLE, 8, 0x0, 0x0, 0x8, 0 }, + { ATC_MASK, 2, 0x6c, 0x14000fc, 0x28, 0 }, + { ATC_MASK, 2, 0x78, 0x1ffff, 0x111, 0 }, + { ATC_MASK, 2, 0x6c, 0x3, 0x2, 0 }, + { ATC_MASK, 2, 0xc, 0x2a, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x20000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x200, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x2000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x800, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 38, 0x40, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1388, 0 }, + { ATC_POLL, 2, 0x260, 0x400, 0x400, 1000000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x64, 0 }, +}; + +static const struct atc_op mf_poweroff[] = { + { ATC_MASK, 35, 0x4, 0x1, 0x1, 0 }, + { ATC_MASK, 35, 0x10c, 0x100000, 0x0, 0 }, + { ATC_MASK, 35, 0x110, 0x60, 0x20, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x8, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x0, 0 }, + { ATC_POLL, 35, 0x104, 0x100, 0x100, 1000000 }, + { ATC_MASK, 35, 0x10c, 0x1e000000, 0x4000000, 0 }, + { ATC_MASK, 35, 0x110, 0x600, 0x200, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x8, 0 }, + { ATC_MASK, 35, 0x4, 0x8, 0x0, 0 }, + { ATC_POLL, 35, 0x104, 0x100, 0x100, 1000000 }, + { ATC_MASK, 35, 0x4, 0x1, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 5, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x10, 0x10, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xc8, 0 }, + { ATC_FIRMWARE, 0, 0x0, 0x0, 0x0, 0 }, + { ATC_POWER, 2, 0x0, 0x0, 0x0, 0 }, + { ATC_MASK, 39, 0x2c, 0x1, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x100, 0x100, 1000 }, + { ATC_MASK, 39, 0x0, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x2, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x0, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_POLL, 39, 0x4, 0x1, 0x0, 1000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 39, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, +}; + +static const struct atc_op dp_group0_off[] = { + { ATC_MASK, 39, 0x14, 0x1007, 0x7, 0 }, + { ATC_MASK, 39, 0x18, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x1c, 0x1, 0x0, 5000 }, + { ATC_MASK, 14, 0x0, 0x1, 0x1, 0 }, + { ATC_MASK, 22, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 22, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 22, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 22, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x100, 0x100, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 19, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 16, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 19, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x4000, 0x4000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x40, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x400, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 14, 0x0, 0xc, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x40000, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x1, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x80000, 0x80000, 0 }, + { ATC_MASK, 3, 0x8, 0x2, 0x2, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x40000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x1, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_POLL, 14, 0x34, 0x8, 0x0, 500000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 6, 0x34, 0xf, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x0, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_MASK, 8, 0x0, 0xaaa, 0x0, 0 }, +}; + +static const struct atc_rate_op dp_group0_on[] = { + { { ATC_MASK, 39, 0x14, 0x7, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 39, 0x18, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x6c, 0x1c, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 14, 0x0, 0x40fff0, 0, 0 }, 0xf, { 0x1110, 0x1110, 0x1110, 0x1110 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x1, { 0x1e0e021c, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x1, { 0x156600, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x1, { 0x70000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x1, { 0x1, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x2, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x2, { 0x0, 0x1e0e01c2, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x2, { 0x0, 0x7fffffe, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x2, { 0x0, 0x155200, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x2, { 0x0, 0x70000, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x2, { 0x0, 0x1, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x4, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x1e0e01c2, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x7fffffe, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x554800, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x4, { 0x0, 0x0, 0x70000, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x4, { 0x0, 0x0, 0x1, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1e0e02a3 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0xbff7ffc } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x564800 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x70000 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1 } }, + { { ATC_MASK, 8, 0x0, 0xaaa, 0, 0 }, 0xf, { 0xaaa, 0xaaa, 0xaaa, 0xaaa } }, + { { ATC_MASK, 6, 0x30, 0x4000, 0, 0 }, 0xf, { 0x4000, 0x4000, 0x4000, 0x4000 } }, + { { ATC_MASK, 6, 0x34, 0xf, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x18, 0x1fe000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_MASK, 6, 0x20, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x1c, 0x7, 0, 0 }, 0xf, { 0x7, 0x7, 0x7, 0x7 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 14, 0x34, 0x8, 0, 500000 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 14, 0x0, 0xc, 0, 0 }, 0xf, { 0xc, 0xc, 0xc, 0xc } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x120, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x100, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 21, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 21, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 21, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 21, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 22, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 22, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 22, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 22, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 22, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 22, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 22, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 22, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 19, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 16, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 19, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 19, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 19, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 19, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 19, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 19, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 19, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 19, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 19, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 19, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 19, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 19, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x10, 0, 0 }, 0xf, { 0x10, 0x10, 0x10, 0x10 } }, + { { ATC_MASK, 2, 0x10, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x4, 0, 0 }, 0xf, { 0x4, 0x4, 0x4, 0x4 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x400, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 2, 0xc, 0x1000, 0, 0 }, 0xf, { 0x1000, 0x1000, 0x1000, 0x1000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x40, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_MASK, 2, 0xc, 0x100, 0, 0 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x4000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x10000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 16, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 19, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 19, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 19, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 19, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 20, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 20, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 20, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 14, 0x0, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 39, 0x1c, 0x1, 0, 5000 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, +}; + +static const struct atc_op dp_group1_off[] = { + { ATC_MASK, 39, 0x14, 0x1007, 0x7, 0 }, + { ATC_MASK, 39, 0x18, 0x1, 0x0, 0 }, + { ATC_POLL, 39, 0x1c, 0x1, 0x0, 5000 }, + { ATC_MASK, 14, 0x0, 0x1, 0x1, 0 }, + { ATC_MASK, 31, 0x1c, 0xc00000, 0xc00000, 0 }, + { ATC_MASK, 31, 0x10, 0x600, 0x400, 0 }, + { ATC_MASK, 31, 0x4, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x100000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x400000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x4000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000000, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 31, 0x44, 0x1000, 0x0, 0 }, + { ATC_MASK, 2, 0x120, 0x200, 0x200, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x120, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x6000000, 0x4000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x44, 0x18000000, 0x10000000, 0 }, + { ATC_MASK, 28, 0x24, 0xc0000000, 0x80000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x24, 0x30000000, 0x20000000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x28, 0x10, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0xc000, 0xc000, 0 }, + { ATC_MASK, 25, 0x1d0, 0x3, 0x1, 0 }, + { ATC_MASK, 28, 0x40, 0x10000, 0x0, 0 }, + { ATC_MASK, 2, 0xc, 0x8000, 0x8000, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x80, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0xc, 0x800, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x2, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x10, 0x20, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 14, 0x0, 0xc, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x400000, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x4, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x800000, 0x800000, 0 }, + { ATC_MASK, 3, 0x8, 0x8, 0x8, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 2, 0x160, 0x400000, 0x0, 0 }, + { ATC_MASK, 3, 0x8, 0x4, 0x0, 0 }, + { ATC_DELAY, 0, 0x0, 0x0, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_POLL, 14, 0x34, 0x8, 0x0, 500000 }, + { ATC_DELAY, 0, 0x0, 0x0, 0xa, 0 }, + { ATC_MASK, 6, 0x34, 0xf, 0x1, 0 }, + { ATC_MASK, 6, 0x1c, 0x7, 0x0, 0 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x8, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x100, 500000 }, + { ATC_MASK, 6, 0x1c, 0x8, 0x0, 0 }, + { ATC_POLL, 6, 0x2c, 0xff00, 0x0, 500000 }, + { ATC_MASK, 8, 0x0, 0xaaa, 0x0, 0 }, +}; + +static const struct atc_rate_op dp_group1_on[] = { + { { ATC_MASK, 39, 0x14, 0x7, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 39, 0x18, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x6c, 0xe0, 0, 0 }, 0xf, { 0x40, 0x40, 0x40, 0x40 } }, + { { ATC_MASK, 14, 0x0, 0x40fff0, 0, 0 }, 0xf, { 0x1110, 0x1110, 0x1110, 0x1110 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x1, { 0x1e0e021c, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x1, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x1, { 0x156600, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x1, { 0x70000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x1, { 0x1, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x2, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x2, { 0x0, 0x1e0e01c2, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x2, { 0x0, 0x7fffffe, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x2, { 0x0, 0x155200, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x2, { 0x0, 0x70000, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x2, { 0x0, 0x1, 0x0, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x4, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x1e0e01c2, 0x0 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x7fffffe, 0x0 } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x4, { 0x0, 0x0, 0x554800, 0x0 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x4, { 0x0, 0x0, 0x70000, 0x0 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x4, { 0x0, 0x0, 0x1, 0x0 } }, + { { ATC_MASK, 8, 0x34, 0x3, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 7, 0x80, 0xffffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1e0e02a3 } }, + { { ATC_MASK, 7, 0x84, 0xfffffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0xbff7ffc } }, + { { ATC_MASK, 7, 0x88, 0x7fffff, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x564800 } }, + { { ATC_MASK, 8, 0x8, 0x1f0000, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x70000 } }, + { { ATC_MASK, 8, 0x18, 0x1, 0, 0 }, 0x8, { 0x0, 0x0, 0x0, 0x1 } }, + { { ATC_MASK, 8, 0x0, 0xaaa, 0, 0 }, 0xf, { 0xaaa, 0xaaa, 0xaaa, 0xaaa } }, + { { ATC_MASK, 6, 0x30, 0x4000, 0, 0 }, 0xf, { 0x4000, 0x4000, 0x4000, 0x4000 } }, + { { ATC_MASK, 6, 0x34, 0xf, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x18, 0x1fe000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_MASK, 6, 0x20, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 6, 0x1c, 0x7, 0, 0 }, 0xf, { 0x7, 0x7, 0x7, 0x7 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x100, 0x100, 0x100, 0x100 } }, + { { ATC_MASK, 6, 0x1c, 0x8, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 6, 0x2c, 0xff00, 0, 500000 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 14, 0x34, 0x8, 0, 500000 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 14, 0x0, 0xc, 0, 0 }, 0xf, { 0xc, 0xc, 0xc, 0xc } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x120, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x120, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x120, 0x200, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0x120, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0xfffffc0, 0, 0 }, 0xf, { 0xaafb800, 0xaafb800, 0xaafb800, 0xaafb800 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x1000000, 0, 0 }, 0xf, { 0x1000000, 0x1000000, 0x1000000, 0x1000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x4000000, 0, 0 }, 0xf, { 0x4000000, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x7c0, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x6c000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 30, 0x100, 0x100000, 0, 0 }, 0xf, { 0x100000, 0x100000, 0x100000, 0x100000 } }, + { { ATC_MASK, 30, 0x58, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 30, 0x104, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 30, 0x100, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 31, 0x44, 0x100000, 0, 0 }, 0x7, { 0x100000, 0x100000, 0x100000, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0x7, { 0x1, 0x1, 0x1, 0x0 } }, + { { ATC_MASK, 31, 0x44, 0x400000, 0, 0 }, 0x7, { 0x400000, 0x400000, 0x400000, 0x0 } }, + { { ATC_MASK, 31, 0x4, 0x3000000, 0, 0 }, 0xf, { 0x2000000, 0x2000000, 0x2000000, 0x2000000 } }, + { { ATC_MASK, 31, 0x4, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0x1, { 0x5000000, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 31, 0xc, 0x7000000, 0, 0 }, 0xe, { 0x0, 0x4000000, 0x4000000, 0x4000000 } }, + { { ATC_MASK, 31, 0x8, 0x3, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 31, 0x10, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_MASK, 31, 0x1c, 0xc00000, 0, 0 }, 0xf, { 0x800000, 0x800000, 0x800000, 0x800000 } }, + { { ATC_MASK, 28, 0xf4, 0x3000, 0, 0 }, 0xf, { 0x3000, 0x3000, 0x3000, 0x3000 } }, + { { ATC_MASK, 25, 0x188, 0x1, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d4, 0x700, 0, 0 }, 0xf, { 0x300, 0x300, 0x300, 0x300 } }, + { { ATC_MASK, 28, 0x24, 0x3f0000, 0, 0 }, 0xf, { 0x2a0000, 0x2a0000, 0x2a0000, 0x2a0000 } }, + { { ATC_MASK, 28, 0x3c, 0x18000, 0, 0 }, 0xf, { 0x10000, 0x10000, 0x10000, 0x10000 } }, + { { ATC_MASK, 28, 0x78, 0xc0000, 0, 0 }, 0xf, { 0x80000, 0x80000, 0x80000, 0x80000 } }, + { { ATC_MASK, 28, 0x7c, 0xc0, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 28, 0x84, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_MASK, 28, 0xd4, 0x600000, 0, 0 }, 0xf, { 0x400000, 0x400000, 0x400000, 0x400000 } }, + { { ATC_MASK, 28, 0xd8, 0x300, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_MASK, 28, 0x40, 0xc00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x48, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_MASK, 28, 0x28, 0x1ff030, 0, 0 }, 0xf, { 0x11c020, 0x11c020, 0x11c020, 0x11c020 } }, + { { ATC_MASK, 28, 0x2c, 0xfc, 0, 0 }, 0xf, { 0xb0, 0xb0, 0xb0, 0xb0 } }, + { { ATC_MASK, 28, 0x28, 0xe00, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 28, 0x28, 0x1c0, 0, 0 }, 0xf, { 0x180, 0x180, 0x180, 0x180 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x20, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 2, 0x10, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0x10, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 2, 0x10, 0x8, 0, 0 }, 0xf, { 0x8, 0x8, 0x8, 0x8 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x800, 0, 0 }, 0xf, { 0x800, 0x800, 0x800, 0x800 } }, + { { ATC_MASK, 2, 0xc, 0x2000, 0, 0 }, 0xf, { 0x2000, 0x2000, 0x2000, 0x2000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x80, 0, 0 }, 0xf, { 0x80, 0x80, 0x80, 0x80 } }, + { { ATC_MASK, 2, 0xc, 0x200, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 2, 0xc, 0x8000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 2, 0xc, 0x20000, 0, 0 }, 0xf, { 0x20000, 0x20000, 0x20000, 0x20000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x3, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 25, 0x1d0, 0x2, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0x30000, 0, 0 }, 0xf, { 0x30000, 0x30000, 0x30000, 0x30000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0xc000, 0xc000, 0xc000, 0xc000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3010, 0, 0 }, 0xf, { 0x1010, 0x1010, 0x1010, 0x1010 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x200, 0x200, 0x200, 0x200 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x3000, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0x30000000, 0, 0 }, 0xf, { 0x30000000, 0x30000000, 0x30000000, 0x30000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x24, 0xc0000000, 0, 0 }, 0xf, { 0xc0000000, 0xc0000000, 0xc0000000, 0xc0000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x600, 0x600, 0x600, 0x600 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x400, 0x400, 0x400, 0x400 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x28, 0x600, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x44, 0x18000000, 0, 0 }, 0xf, { 0x18000000, 0x18000000, 0x18000000, 0x18000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x2, 0x2, 0x2, 0x2 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0x1, { 0x28, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 28, 0x40, 0x38, 0, 0 }, 0xe, { 0x0, 0x20, 0x20, 0x20 } }, + { { ATC_MASK, 28, 0x44, 0x60000000, 0, 0 }, 0xf, { 0x40000000, 0x40000000, 0x40000000, 0x40000000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 28, 0x44, 0x6000000, 0, 0 }, 0xf, { 0x6000000, 0x6000000, 0x6000000, 0x6000000 } }, + { { ATC_MASK, 28, 0x40, 0xc000, 0, 0 }, 0xf, { 0x8000, 0x8000, 0x8000, 0x8000 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x68, 0xc00f, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_MASK, 29, 0xac, 0x20000000, 0, 0 }, 0xf, { 0x20000000, 0x20000000, 0x20000000, 0x20000000 } }, + { { ATC_MASK, 29, 0xa8, 0x7ffff, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 29, 0x18, 0x38, 0, 0 }, 0xf, { 0x20, 0x20, 0x20, 0x20 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_DELAY, 0, 0x0, 0x0, 0, 0 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, + { { ATC_MASK, 14, 0x0, 0x1, 0, 0 }, 0xf, { 0x0, 0x0, 0x0, 0x0 } }, + { { ATC_POLL, 39, 0x1c, 0x1, 0, 5000 }, 0xf, { 0x1, 0x1, 0x1, 0x1 } }, +}; diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h new file mode 100644 index 0000000000000..503cbb537f865 --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb2_reset.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef APPLE_T6050_USB2_RESET_H +#define APPLE_T6050_USB2_RESET_H +#include "apple_t6050_usb3_bist.h" + +/* Native T6050 Type-C PHY USB2 port reset block, native27: + * bank0+4 bit1 assert, sleep 5 ms, release. + * Caller holds exclusive PHY ownership with its USB power domains active. + * sleep_ms is supplied by the OS frontend; there is no packet-path polling. */ +static inline int apple_t6050_usb2_port_reset( + const struct apple_t6050_usb_io *io, void *sleep_context, + void (*sleep_ms)(void *, unsigned int)) +{ + if (!io || !io->read || !io->write || !sleep_ms) + return -1; + apple_t6050_usb_mask(io, 0, 4, 2U, 2U); + if (!(io->read(io->context, 0, 4) & 2U)) + return -1; + sleep_ms(sleep_context, 5); + apple_t6050_usb_mask(io, 0, 4, 2U, 0); + return io->read(io->context, 0, 4) & 2U ? -1 : 0; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h new file mode 100644 index 0000000000000..3af8fcf2a3a13 --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_bist.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: MIT */ +/* T6050-specific native PHY recipe; no Windows/Linux frontend policy. */ +#ifndef APPLE_T6050_USB3_BIST_H +#define APPLE_T6050_USB3_BIST_H +struct apple_t6050_usb_io { + void *context; + unsigned int (*read)(void *, unsigned int bank, unsigned int offset); + void (*write)(void *, unsigned int bank, unsigned int offset, unsigned int value); + int (*poll)(void *, unsigned int bank, unsigned int offset, unsigned int mask, unsigned int value, unsigned int usec); + void (*delay)(void *, unsigned int usec); +}; +static inline void apple_t6050_usb_mask(const struct apple_t6050_usb_io *io, unsigned int bank, unsigned int offset, unsigned int mask, unsigned int value) +{ io->write(io->context, bank, offset, (io->read(io->context, bank, offset) & ~mask) | value); } +static inline int apple_t6050_usb_bist_off(const struct apple_t6050_usb_io *io) +{ + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x660, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x1, 0x0); + return 0; +} +static inline int apple_t6050_usb_bist_on(const struct apple_t6050_usb_io *io) +{ + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x1, 0x1); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x10c, 0x1e100000, 0x4100000); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x660, 0x220); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + if (io->poll(io->context, 35, 0x104, 0x100, 0x0, 20000)) return -1; + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x20); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x0); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x10c, 0x1e000000, 0x6000000); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x110, 0x600, 0x200); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x8); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x4, 0x8, 0x0); + if (io->poll(io->context, 35, 0x104, 0x40, 0x40, 130000)) return -1; + if (io->poll(io->context, 35, 0x104, 0x100, 0x0, 1000)) return -1; + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x20); + /* native sequence step */ + apple_t6050_usb_mask(io, 35, 0x8, 0x20, 0x0); + return 0; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h new file mode 100644 index 0000000000000..953a42dbc7aab --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: MIT */ +/* Platform backend for native direct USB3, not a Type-C policy engine. + * The caller owns the port, has powered the PHY/firmware for native USB3, + * completed DWC3 core init and enabled SUSPHY, and has valid DART mappings. + * Bank 35 is T6050's ADT PHY reg[35]. Bank 44 is usb-drdN reg[3]. + * The first left/debug port must never reach this backend. + */ +#ifndef APPLE_T6050_USB3_PIPE_H +#define APPLE_T6050_USB3_PIPE_H +#include "apple_t6050_usb3_bist.h" + +/* Cold controller setup while RESET_N is clear. Linux's existing ATC + * frontend supplies a dummy PIPE clock before DWC3 reads its core registers. + * T6050 uses the native-RE 6:4 clock field and dummy encoding 0x42. + * This is not an active producer handoff; never use it for USB4 teardown. + */ +static inline int +apple_t6050_usb3_pipe_cold_dummy(const struct apple_t6050_usb_io *io) +{ + unsigned int mux; + if (!io || !io->read || !io->write || !io->delay) + return -1; + if (io->read(io->context, 44, 0x1c) & 1) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + if (mux != 0 && mux != 2 && mux != 0x42) + return -1; + if (mux == 0x42) + return 0; + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 10); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 2); + io->delay(io->context, 10); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x40); + io->delay(io->context, 10); + return (io->read(io->context, 44, 0x0c) & 0x73) == 0x42 ? 0 : -1; +} + +static inline int +apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) +{ + unsigned int aon, mux; + int result = -1, locked = 0, bist = 0; + + if (!io || !io->read || !io->write || !io->poll || !io->delay) + return -1; + aon = io->read(io->context, 44, 0x1c); + mux = io->read(io->context, 44, 0x0c) & 0x73; + /* Never reroute an active USB4 producer or guess an unknown state. */ + if (!(aon & 1) || (mux != 0x42 && mux != 0x10)) + return -1; + + /* Native T8142 xHCI USB3 mode switch, native27 T6050-compatible path. */ + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + locked = 1; + if (io->poll(io->context, 44, 0x14, 1, 1, 6000)) + goto out; + + bist = 1; + if (apple_t6050_usb_bist_on(io)) + goto out; + apple_t6050_usb_mask(io, 44, 0x20, 0x100f, 3); + if (io->poll(io->context, 44, 0x20, 0xf, 3, 1000)) + goto out; + if (apple_t6050_usb_bist_off(io)) + goto out; + bist = 0; + + /* T6050 clock bits are 6:4, not the older 5:3. Native USB3 is 0x10. */ + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0); + apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0x10); + result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x10 ? 0 : -1; + +out: + if (bist && apple_t6050_usb_bist_off(io)) + result = -1; + apple_t6050_usb_mask(io, 44, 0x00, 5, 0); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + if (io->poll(io->context, 44, 0x14, 1, 0, 6000)) + result = -1; + } + return result; +} + +/* The native close() call quiesces the ACIO PHY while PIPE is locked, + * before selecting the dummy producer. The caller supplies that lifecycle + * operation; merely changing MUX is not a complete disconnect sequence. + */ +static inline int +apple_t6050_usb3_pipe_stop(const struct apple_t6050_usb_io *io, + int (*quiesce)(void *), void *context) +{ + unsigned int mux; + int result = -1; + + if (!io || !io->read || !io->write || !io->poll || !io->delay || !quiesce) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + if (mux != 0x10 && mux != 0x42) + return -1; + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + apple_t6050_usb_mask(io, 44, 0x00, 4, 4); + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + /* Native mode0-from2 waits 150 ms, logs a missing lock ACK, and + * continues teardown, as the saved native driver does. + * The caller has already stopped the controller's DMA/PIPE consumer; + * this is not the active-producer handoff used by pipe_switch(). + * The nonselected override, final mux readback and unlock checks below + * remain mandatory, as does successful PHY quiescence. + */ + io->poll(io->context, 44, 0x14, 1, 1, 150000); + if (quiesce(context)) + goto out; + apple_t6050_usb_mask(io, 44, 0x20, 0x100f, 0x1002); + if (io->poll(io->context, 44, 0x20, 0xf, 2, 1000)) + goto out; + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 2); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x40); + result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x42 ? 0 : -1; +out: + apple_t6050_usb_mask(io, 44, 0x00, 1, 0); + apple_t6050_usb_mask(io, 44, 0x00, 4, 0); + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + if (io->poll(io->context, 44, 0x14, 1, 0, 6000)) + result = -1; + return result; +} +#endif diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h new file mode 100644 index 0000000000000..5c96da7d82c2d --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb_phy.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: MIT */ +/* Generated by generate-phy.py; modeled ACKs are not hardware proof. */ +#ifndef APPLE_T6050_USB_PHY_H +#define APPLE_T6050_USB_PHY_H +#include "apple_t6050_usb3_bist.h" +struct apple_t6050_usb_platform { + const struct apple_t6050_usb_io *io; + void *context; + int (*wrapper_power)(void *, unsigned int); + int (*firmware)(void *, unsigned int); + int (*tunable)(void *, unsigned int, const char *); + void (*pause_ns)(void *, unsigned int); +}; +enum apple_t6050_usb_op { T6050_MASK, T6050_POLL, T6050_DELAY, + T6050_POWER, T6050_FIRMWARE, T6050_TUNABLE }; +struct apple_t6050_usb_step { + unsigned int op, bank, offset, mask, value, timeout; + const char *name; +}; +static const struct apple_t6050_usb_step apple_t6050_usb2_host[] = { + { T6050_TUNABLE, 0, 0x0, 0x0, 0x0, 0, "tunable_USB2PHY_DFLT" }, + { T6050_TUNABLE, 0, 0x0, 0x0, 0x0, 0, "tunable_USB2PHY_HOST" }, + { T6050_DELAY, 0, 0x0, 0x0, 0x2710, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x1, 0x0, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x2, 0x0, 0, 0 }, + { T6050_MASK, 1, 0x0, 0x9, 0x9, 0, 0 }, + { T6050_MASK, 0, 0x4, 0x4, 0x4, 0, 0 }, + { T6050_MASK, 0, 0x1c, 0x20000000, 0x0, 0, 0 }, + { T6050_MASK, 0, 0x1c, 0x40000000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1e, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_MASK, 0, 0x0, 0x7, 0x2, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_init_normal[] = { + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 }, + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, + { T6050_MASK, 4, 0x0, 0x3f0, 0x1c0, 0, 0 }, + { T6050_MASK, 4, 0x18, 0xf000, 0x3000, 0, 0 }, + { T6050_MASK, 9, 0x88, 0xff, 0x27, 0, 0 }, + { T6050_MASK, 33, 0x4, 0x38, 0x8, 0, 0 }, + { T6050_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0, 0 }, + { T6050_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 20, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 29, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_MASK, 2, 0x6c, 0x28000fc, 0x44, 0, 0 }, + { T6050_MASK, 2, 0x78, 0x1ffff, 0x110, 0, 0 }, + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x40, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x100, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x40, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x400, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_init_reverse[] = { + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 }, + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, + { T6050_MASK, 4, 0x0, 0x3f0, 0x1c0, 0, 0 }, + { T6050_MASK, 4, 0x18, 0xf000, 0x3000, 0, 0 }, + { T6050_MASK, 9, 0x88, 0xff, 0x27, 0, 0 }, + { T6050_MASK, 33, 0x4, 0x38, 0x8, 0, 0 }, + { T6050_MASK, 10, 0xdc, 0x3ff0, 0x1760, 0, 0 }, + { T6050_MASK, 21, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 30, 0xcc, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 20, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_MASK, 29, 0x90, 0x1f80000, 0x380000, 0, 0 }, + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, + { T6050_MASK, 2, 0x6c, 0x14000fc, 0x28, 0, 0 }, + { T6050_MASK, 2, 0x78, 0x1ffff, 0x111, 0, 0 }, + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x10, 0x80, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x20000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x200, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x2000, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x80, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x800, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x2, 0x2, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 }, +}; +static const struct apple_t6050_usb_step apple_t6050_usb3_shutdown[] = { + { T6050_MASK, 35, 0x4, 0x1, 0x1, 0, 0 }, + { T6050_MASK, 35, 0x10c, 0x100000, 0x0, 0, 0 }, + { T6050_MASK, 35, 0x110, 0x60, 0x20, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x8, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_POLL, 35, 0x104, 0x100, 0x100, 1000000, 0 }, + { T6050_MASK, 35, 0x10c, 0x1e000000, 0x4000000, 0, 0 }, + { T6050_MASK, 35, 0x110, 0x600, 0x200, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x8, 0, 0 }, + { T6050_MASK, 35, 0x4, 0x8, 0x0, 0, 0 }, + { T6050_POLL, 35, 0x104, 0x100, 0x100, 1000000, 0 }, + { T6050_MASK, 35, 0x4, 0x1, 0x0, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x100, 0x100, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x200, 0x200, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0x120, 0x20, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 5, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0xc8, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x100, 1000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x4, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, +}; + +static inline int +apple_t6050_usb_phy_run(const struct apple_t6050_usb_platform *p, + const struct apple_t6050_usb_step *steps, unsigned int count) +{ + if (!p || !p->io || !p->io->read || !p->io->write || + !p->io->poll || !p->io->delay || !p->wrapper_power || + !p->firmware || !p->tunable) + return -1; + for (unsigned int i = 0; i < count; i++) { + const struct apple_t6050_usb_step *s = &steps[i]; + int ret = 0; + switch (s->op) { + case T6050_MASK: + apple_t6050_usb_mask(p->io, s->bank, s->offset, s->mask, s->value); + break; + case T6050_POLL: + ret = p->io->poll(p->io->context, s->bank, s->offset, + s->mask, s->value, s->timeout); + break; + case T6050_DELAY: p->io->delay(p->io->context, s->value); break; + case T6050_POWER: ret = p->wrapper_power(p->context, s->value); break; + case T6050_FIRMWARE: ret = p->firmware(p->context, s->value); break; + case T6050_TUNABLE: ret = p->tunable(p->context, s->bank, s->name); break; + default: return -1; + } + if (ret) + return ret; + } + return 0; +} +#define APPLE_T6050_USB_RUN(p, steps) \ + apple_t6050_usb_phy_run((p), (steps), sizeof(steps) / sizeof((steps)[0])) + +/* Native shutdown first asserts PORT_RESET only if it was not already set. + * The 5 ms wait belongs to that transition, not every redundant stop call. + * Preserve IOPause(500 ns), whose unit differs from IODelay(microseconds). + */ +static inline int apple_t6050_usb2_stop(const struct apple_t6050_usb_platform *p) +{ + if (!p || !p->io || !p->io->read || !p->io->write || + !p->io->delay || !p->pause_ns) + return -1; + if (!(p->io->read(p->io->context, 0, 4) & 2)) { + apple_t6050_usb_mask(p->io, 0, 4, 2, 2); + p->io->delay(p->io->context, 5000); + } + apple_t6050_usb_mask(p->io, 0, 0, 7, 4); + apple_t6050_usb_mask(p->io, 0, 4, 8, 8); + apple_t6050_usb_mask(p->io, 0, 4, 1, 1); + apple_t6050_usb_mask(p->io, 0, 0x1c, 0x20000000, 0x20000000); + apple_t6050_usb_mask(p->io, 0, 0x1c, 0x40000000, 0x40000000); + p->pause_ns(p->context, 500); + return 0; +} +#endif From ad990d38db1b3ba42bc9a5023defa866e3b42606 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 14:19:46 -0800 Subject: [PATCH 201/392] dt-bindings: phy: apple,t6050-dp-phy: Allow the USB domain on combo PHYs The combo PHY's USB2 PHY and digital banks are in the USB controller's power domain. Let a combo PHY name it fourth, as "usb", so the PHY can hold it on a port with no USB controller bound. The fixed DisplayPort PHY keeps exactly three domains. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- .../devicetree/bindings/phy/apple,t6050-dp-phy.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml index 52cb96f4a965b..b055369faf9f6 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml @@ -74,14 +74,20 @@ properties: description: Offset, mask and value tuples for the PIPE handler power-domains: + description: + A combo PHY may name the USB controller's domain fourth ("usb"). Its + USB2 PHY and digital banks are in that domain, and the PHY then holds it + while it runs, also on a port with no USB controller bound. minItems: 3 - maxItems: 3 + maxItems: 4 power-domain-names: + minItems: 3 items: - const: usb-aon - const: common - const: wrapper + - const: usb apple,tunable-axi2af: $ref: /schemas/types.yaml#/definitions/uint32-array @@ -184,6 +190,10 @@ allOf: properties: '#phy-cells': const: 0 + power-domains: + maxItems: 3 + power-domain-names: + maxItems: 3 if: required: - mode-switch From 42b87ef7de672fdab3cc5fc4175e976c8840f807 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 14:19:48 -0800 Subject: [PATCH 202/392] phy: apple: t6050: Hold the USB domain for the combo PHY's USB2 banks The combo PHY's USB2 PHY and digital banks are in the USB controller's power domain. Only a bound USB controller holds that domain, so on a port with no controller (a DisplayPort-only port, as on J775D with Linux on the hardware) the first Type-C mode switch wrote an unclocked bank and took an SError. A combo PHY node may now name that domain as a fourth power domain, "usb". When it does, the PHY holds it from combo start to combo stop, after the common domain on the way up and before it on the way down. Nodes that name three domains probe and run exactly as before, and the fixed DisplayPort PHY still takes exactly three. Testing: on J775D (M5 Ultra), with the fourth domain named on each combo port, Type-C mode switches and DisplayPort on the rear ports ran without an SError on every boot since 2026-10-30. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 8 ++++++++ drivers/phy/apple/atc-t6050-dp.c | 20 ++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index 1066748dc1afe..3c3454ceb64de 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -94,6 +94,8 @@ static int combo_stop(struct t6050_dp *dp) if (!ret) dp->usb2_active = false; } + if (!ret && dp->ndomains > 3) + ret = atc_domain_set(&dp->domain[3], false); if (!ret) ret = atc_domain_set(&dp->domain[1], false); if (!ret) @@ -118,6 +120,12 @@ static int combo_start(struct t6050_dp *dp, enum combo_mode mode) ret = atc_domain_set(&dp->domain[0], true); if (!ret) ret = atc_domain_set(&dp->domain[1], true); + /* + * The USB2 PHY and digital banks are unclocked without the USB domain, + * which a USB controller holds only while one is bound to this port. + */ + if (!ret && dp->ndomains > 3) + ret = atc_domain_set(&dp->domain[3], true); if (!ret) ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); if (!ret) diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c index cafee096a6250..4479153919488 100644 --- a/drivers/phy/apple/atc-t6050-dp.c +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -83,7 +83,9 @@ struct t6050_dp { void __iomem *banks[45]; struct resource resources[45]; struct apple_tunable *tunables[ARRAY_SIZE(tunable_names)]; - struct atc_domain domain[3]; /* USB AON, common, wrapper: native order. */ + /* USB AON, common, wrapper: native order; a combo may add USB. */ + struct atc_domain domain[4]; + unsigned int ndomains; u32 training[4][16]; u32 voltage[4], pre[4]; int rate; @@ -605,14 +607,14 @@ static int t6050_typec_register(struct t6050_dp *dp) static int t6050_dp_probe(struct platform_device *pdev) { - static const char * const domains[] = {"usb-aon", "common", "wrapper"}; + static const char * const domains[] = {"usb-aon", "common", "wrapper", "usb"}; struct device *dev = &pdev->dev; struct t6050_dp *dp; struct phy_provider *provider; struct phy *phy; struct resource *core; void __iomem *base; - int ret; + int ndomains, ret; dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL); if (!dp) @@ -741,10 +743,16 @@ static int t6050_dp_probe(struct platform_device *pdev) if (ret) return ret; } - /* Three named domains prevent automatic wrapper enable at probe time. */ - if (of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells") != 3) + /* + * Named domains prevent automatic wrapper enable at probe time. A combo + * PHY may also name the USB controller's domain, which its USB2 banks + * are in (combo_start()). + */ + ndomains = of_count_phandle_with_args(dev->of_node, "power-domains", "#power-domain-cells"); + if (ndomains != 3 && !(dp->combo && ndomains == 4)) return -EINVAL; - for (unsigned int i = 0; i < ARRAY_SIZE(domains); i++) { + dp->ndomains = ndomains; + for (unsigned int i = 0; i < dp->ndomains; i++) { dp->domain[i].dev = dev_pm_domain_attach_by_name(dev, domains[i]); if (IS_ERR_OR_NULL(dp->domain[i].dev)) return dev_err_probe(dev, dp->domain[i].dev ? PTR_ERR(dp->domain[i].dev) : -ENODEV, From 4ef9db44d4d951531b431a4d8ed26bbd1f720e33 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 17:06:32 -0800 Subject: [PATCH 203/392] dt-bindings: phy: apple,t6050-dp-phy: Type apple,port-index as uint32 apple,port-index has an enum but no type, so dtschema cannot decode the cell and every combo PHY node fails with "b'\x00\x00\x00\x02' is not one of [1, 2]" and a core-schema type error, whatever port it names. Give it the uint32 type. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml index b055369faf9f6..fdc2e60461700 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml @@ -37,6 +37,7 @@ properties: const: 1 apple,port-index: + $ref: /schemas/types.yaml#/definitions/uint32 enum: [1, 2] description: Native Type-C port; the debug/proxy port is excluded From 8b23bd4d3468912947de0790651739028b6d30b9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 204/392] usb: dwc3: apple: Add the bare-metal T6050 controller Add apple,t6050-dwc3, the T6050 (J775D) DWC3 on bare metal. It uses the shared tunable table, extended with the tables Diljit Singh's T6050 USB support applies, in the same order: the T6040 tables plus CIO (the Apple region from 0xcd20, after the fixed CIO values), bulk fabric and AUSBC debug (the optional drd-bulk and drd-ausbc regions). It is host-only, like T6040. The USB2 port reset follows the root hub's start (a no-op without a PHY reset op), and a USB3 PHY failure is fatal, as on T8132. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- .../devicetree/bindings/usb/apple,dwc3.yaml | 36 ++++++- drivers/usb/dwc3/dwc3-apple.c | 95 +++++++++++++++---- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 706d2866f61ba..7745fef5b686f 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -35,6 +35,23 @@ allOf: required: - resets - usb-role-switch + - if: + properties: + compatible: + contains: + const: apple,t6050-dwc3 + then: + properties: + reg: + minItems: 3 + reg-names: + minItems: 3 + else: + properties: + reg: + maxItems: 3 + reg-names: + maxItems: 3 $defs: apple,tunable: @@ -48,7 +65,7 @@ $defs: - description: Bits to be set after applying the mask description: > Controller table from the device's Apple device tree, applied after - every core reset (T6040). + every core reset (T6040, T6050). properties: compatible: @@ -74,7 +91,9 @@ properties: items: - description: Core DWC3 region - description: Apple-specific DWC3 region - - description: Controller configuration block (T6040) + - description: Controller configuration block (T6040, T6050) + - description: Bulk fabric block (T6050) + - description: AUSBC debug block (T6050) reg-names: minItems: 2 @@ -82,6 +101,8 @@ properties: - const: dwc3-core - const: dwc3-apple - const: drd-cfg + - const: drd-bulk + - const: drd-ausbc interrupts: maxItems: 1 @@ -118,7 +139,16 @@ properties: apple,tunable-cio: $ref: "#/$defs/apple,tunable" description: - Tunables applied to the CIO (USB4 tunnel) registers of the Apple region. + Tunables applied to the CIO registers of the Apple region, from offset + 0x20 of that region, after the driver's fixed CIO values. + + apple,tunable-bulk: + $ref: "#/$defs/apple,tunable" + description: Tunables applied to the drd-bulk region. + + apple,tunable-ausbc: + $ref: "#/$defs/apple,tunable" + description: Tunables applied to the drd-ausbc region. required: - compatible diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f16aa6f21014b..f2484ee172862 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -100,7 +100,8 @@ * enum dwc3_apple_state is defined in dwc3-apple-state.h. */ -#define DWC3_APPLE_NR_TUNABLES 5 +#define DWC3_APPLE_NR_TUNABLES 8 +#define DWC3_APPLE_NR_DRD_REGS 3 /** * struct dwc3_apple - Apple-specific DWC3 USB controller @@ -127,7 +128,8 @@ * @rpm_held: The glue holds a runtime PM reference for the running core * @pre_core_rpm: Runtime PM is the glue's own until the first core bring-up * @dt_dis_u3_susphy: The device tree keeps USB3 PHY suspend off itself - * @drd_cfg: The controller's configuration block (optional, for its tunable) + * @drd_regs: The controller's optional blocks outside dwc3-core (drd-cfg, drd-bulk, drd-ausbc), + * mapped for their tunables * @tunables: Controller tunables from the device tree, see dwc3_apple_tunable_list */ struct dwc3_apple { @@ -158,7 +160,7 @@ struct dwc3_apple { bool pre_core_rpm; bool dt_dis_u3_susphy; - void __iomem *drd_cfg; + void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; }; @@ -232,22 +234,43 @@ enum dwc3_apple_tunable_block { DWC3_APPLE_TUNABLE_GLOBALS, /* DWC3_GLOBALS_REGS_START */ DWC3_APPLE_TUNABLE_LINK, /* dwc3-core + 0xd000 */ DWC3_APPLE_TUNABLE_DBG, /* dwc3-core + 0xd800 */ + DWC3_APPLE_TUNABLE_CIO, /* dwc3-core + 0xcd20, in dwc3-apple */ + /* The optional blocks outside dwc3-core, in drd_regs[] order */ DWC3_APPLE_TUNABLE_DRD_CFG, /* drd-cfg */ + DWC3_APPLE_TUNABLE_DRD_BULK, /* drd-bulk */ + DWC3_APPLE_TUNABLE_DRD_AUSBC, /* drd-ausbc */ + DWC3_APPLE_NR_TUNABLE_BLOCKS }; +static const char * const dwc3_apple_drd_reg_names[DWC3_APPLE_NR_DRD_REGS] = { + "drd-cfg", "drd-bulk", "drd-ausbc", +}; + +static_assert(DWC3_APPLE_NR_TUNABLE_BLOCKS - DWC3_APPLE_TUNABLE_DRD_CFG == DWC3_APPLE_NR_DRD_REGS); + #define DWC3_APPLE_LINK_REGS_START 0xd000 #define DWC3_APPLE_DBG_REGS_START 0xd800 #define DWC3_APPLE_LINK_DBG_SIZE 0x800 +/* The CIO registers, the fixed ones of dwc3_apple_setup_cio() included */ +#define DWC3_APPLE_CIO_REGS_START 0xcd20 +/* + * In the order they are applied. A table is optional, and one for a block that the device tree + * does not describe is refused. The CIO table follows dwc3_apple_setup_cio() and overrides its + * fixed values (T6050). + */ static const struct { const char *prop; enum dwc3_apple_tunable_block block; } dwc3_apple_tunable_list[] = { + { "apple,tunable-bulk", DWC3_APPLE_TUNABLE_DRD_BULK }, { "apple,tunable-gbl-default", DWC3_APPLE_TUNABLE_GLOBALS }, { "apple,tunable-gbl-host", DWC3_APPLE_TUNABLE_GLOBALS }, + { "apple,tunable-cio", DWC3_APPLE_TUNABLE_CIO }, { "apple,tunable-dbg", DWC3_APPLE_TUNABLE_DBG }, { "apple,tunable-cfg-host", DWC3_APPLE_TUNABLE_DRD_CFG }, { "apple,tunable-link", DWC3_APPLE_TUNABLE_LINK }, + { "apple,tunable-ausbc", DWC3_APPLE_TUNABLE_DRD_AUSBC }, }; static_assert(ARRAY_SIZE(dwc3_apple_tunable_list) == DWC3_APPLE_NR_TUNABLES); @@ -262,8 +285,14 @@ static void __iomem *dwc3_apple_tunable_base(struct dwc3_apple *appledwc, return appledwc->apple_regs + DWC3_APPLE_LINK_REGS_START - APPLE_DWC3_REGS_START; case DWC3_APPLE_TUNABLE_DBG: return appledwc->apple_regs + DWC3_APPLE_DBG_REGS_START - APPLE_DWC3_REGS_START; + case DWC3_APPLE_TUNABLE_CIO: + return appledwc->apple_regs + DWC3_APPLE_CIO_REGS_START - APPLE_DWC3_REGS_START; case DWC3_APPLE_TUNABLE_DRD_CFG: - return appledwc->drd_cfg; + case DWC3_APPLE_TUNABLE_DRD_BULK: + case DWC3_APPLE_TUNABLE_DRD_AUSBC: + return appledwc->drd_regs[block - DWC3_APPLE_TUNABLE_DRD_CFG]; + case DWC3_APPLE_NR_TUNABLE_BLOCKS: + break; } return NULL; } @@ -281,8 +310,9 @@ static void dwc3_apple_apply_tunables(struct dwc3_apple *appledwc) /* The tunables are optional; a malformed one fails the probe. */ static int dwc3_apple_parse_tunables(struct dwc3_apple *appledwc, struct platform_device *pdev) { - struct resource *core = appledwc->mmio_resource, *drd_cfg; - struct resource blocks[] = { + struct resource *core = appledwc->mmio_resource; + /* A block the device tree does not describe stays empty, and its table is refused. */ + struct resource blocks[DWC3_APPLE_NR_TUNABLE_BLOCKS] = { [DWC3_APPLE_TUNABLE_GLOBALS] = DEFINE_RES_MEM(core->start + DWC3_GLOBALS_REGS_START, DWC3_GLOBALS_REGS_END - DWC3_GLOBALS_REGS_START + 1), @@ -290,17 +320,24 @@ static int dwc3_apple_parse_tunables(struct dwc3_apple *appledwc, struct platfor DWC3_APPLE_LINK_DBG_SIZE), [DWC3_APPLE_TUNABLE_DBG] = DEFINE_RES_MEM(core->start + DWC3_APPLE_DBG_REGS_START, DWC3_APPLE_LINK_DBG_SIZE), - [DWC3_APPLE_TUNABLE_DRD_CFG] = DEFINE_RES_MEM(0, 0), + [DWC3_APPLE_TUNABLE_CIO] = DEFINE_RES_MEM(core->start + DWC3_APPLE_CIO_REGS_START, + APPLE_DWC3_REGS_END - + DWC3_APPLE_CIO_REGS_START + 1), }; struct device *dev = appledwc->dev; int i; - drd_cfg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "drd-cfg"); - if (drd_cfg) { - appledwc->drd_cfg = devm_ioremap_resource(dev, drd_cfg); - if (IS_ERR(appledwc->drd_cfg)) - return PTR_ERR(appledwc->drd_cfg); - blocks[DWC3_APPLE_TUNABLE_DRD_CFG] = *drd_cfg; + for (i = 0; i < DWC3_APPLE_NR_DRD_REGS; i++) { + struct resource *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + dwc3_apple_drd_reg_names[i]); + if (!res) + continue; + appledwc->drd_regs[i] = devm_ioremap_resource(dev, res); + if (IS_ERR(appledwc->drd_regs[i])) + return PTR_ERR(appledwc->drd_regs[i]); + blocks[DWC3_APPLE_TUNABLE_DRD_CFG + i] = *res; } for (i = 0; i < ARRAY_SIZE(dwc3_apple_tunable_list); i++) { @@ -628,8 +665,13 @@ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_stat dwc3_enable_susphy(&appledwc->dwc, true); ret = phy_set_mode(appledwc->usb3_phy, host ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE); if (ret) { - /* A failed T8132 BIST or PIPE handoff must not start xHCI. */ - if (of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3")) + /* + * A failed T8132 BIST or PIPE handoff must not start xHCI. Neither must a failed + * T6050 one: that PHY reports a port left on USB2 (four DisplayPort lanes) as + * success, and its errors leave the PIPE without a known producer. + */ + if (of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3") || + of_device_is_compatible(appledwc->dev->of_node, "apple,t6050-dwc3")) return ret; dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); } else if (host && appledwc->usb3_phy && @@ -648,18 +690,19 @@ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_stat } /* - * No device mode: the T6040 USB2 PHY implements host mode only (phy-apple-t6040-usb2), and a - * kernel built without gadget support turns dwc3_gadget_init() into a stub that succeeds. Such a - * controller must not follow a USB_ROLE_DEVICE request from the Type-C port controller (a charger - * or a host on the other end), or the core would end up in device mode on top of a PHY powered - * in host mode. + * No device mode: the T6040 USB2 PHY (phy-apple-t6040-usb2) and the T6050 Type-C PHY implement + * host mode only, and a kernel built without gadget support turns dwc3_gadget_init() into a stub + * that succeeds. Such a controller must not follow a USB_ROLE_DEVICE request from the Type-C port + * controller (a charger or a host on the other end), or the core would end up in device mode on + * top of a PHY powered in host mode. */ static bool dwc3_apple_host_only(struct device *dev) { if (!IS_ENABLED(CONFIG_USB_DWC3_GADGET) && !IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE)) return true; - return of_device_is_compatible(dev->of_node, "apple,t6040-dwc3"); + return of_device_is_compatible(dev->of_node, "apple,t6040-dwc3") || + of_device_is_compatible(dev->of_node, "apple,t6050-dwc3"); } static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state state) @@ -691,6 +734,16 @@ static int dwc3_apple_init(struct dwc3_apple *appledwc, enum dwc3_apple_state st dev_err(appledwc->dev, "Failed to initialize host, ret=%d\n", ret); goto core_exit; } + /* + * A USB2 PHY with a port reset (the T6050 Type-C PHY) takes it once the root hub is + * up; for the other PHYs, which have none, this is a no-op. + */ + ret = phy_reset(appledwc->usb2_phy); + if (ret) { + dev_err(appledwc->dev, "Failed to reset the USB2 port, ret=%d\n", ret); + dwc3_host_exit(&appledwc->dwc); + goto core_exit; + } break; case DWC3_APPLE_DEVICE: ret = dwc3_gadget_init(&appledwc->dwc); From 27e8000edb6ec0a888e7982ba39160860ba3bd05 Mon Sep 17 00:00:00 2001 From: Justin Pfister <119864532+JJRPF@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:59:56 -0500 Subject: [PATCH 205/392] arm64: dts: apple: t602x: add the CPU energy model (enables EAS on M2 Pro/Max) T602x has no CPU energy model, so energy-aware scheduling stays off even though apple-soc-cpufreq already registers the model from the OPP table: the cluster OPP tables carry no opp-microwatt, while T8103, T600x, T8112, T8132 and T8140 do. Add opp-microwatt to every OPP of both core types. The values are derived, not measured on T602x: T8112 (M2) has the same cores, Blizzard and Avalanche, on the same process, and its opp-microwatt was measured with the SMC cluster power meters. Its per-OPP power is fitted to P = (a + b f) V^2 f per core type with T6020's voltage for each OPP (from the ADT's pmgr voltage-states), and the fit gives T6020's OPPs, within 6% rms of T8112's measurements for both core types. The energy per cycle rises from each OPP to the next, so no OPP is marked inefficient and cpufreq keeps choosing among all of them; only the scheduler's placement changes. Same-boot A/B on a J414s (M2 Pro), headless idle, kernel.sched_energy_aware 1 vs 0, 12 paired rounds of 20 s: system power is 0.230 W lower with the energy model (lower in 12 of 12; SoC power 0.156 W). All six T602x boards build with W=1 and no new warnings. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01RuRMQncGu3ukpyW44vHHjQ Signed-off-by: Justin Pfister <119864532+JJRPF@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t602x-common.dtsi | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index e7b88dfff6bce..b5275a098397b 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -268,31 +268,37 @@ opp-hz = /bits/ 64 <912000000>; opp-level = <2>; clock-latency-ns = <7700>; + opp-microwatt = <53000>; }; opp03 { opp-hz = /bits/ 64 <1284000000>; opp-level = <3>; clock-latency-ns = <25000>; + opp-microwatt = <92000>; }; opp04 { opp-hz = /bits/ 64 <1752000000>; opp-level = <4>; clock-latency-ns = <33000>; + opp-microwatt = <162000>; }; opp05 { opp-hz = /bits/ 64 <2004000000>; opp-level = <5>; clock-latency-ns = <38000>; + opp-microwatt = <214000>; }; opp06 { opp-hz = /bits/ 64 <2256000000>; opp-level = <6>; clock-latency-ns = <44000>; + opp-microwatt = <271000>; }; opp07 { opp-hz = /bits/ 64 <2424000000>; opp-level = <7>; clock-latency-ns = <48000>; + opp-microwatt = <331000>; }; }; @@ -304,88 +310,105 @@ opp-hz = /bits/ 64 <702000000>; opp-level = <1>; clock-latency-ns = <7400>; + opp-microwatt = <123000>; }; opp02 { opp-hz = /bits/ 64 <948000000>; opp-level = <2>; clock-latency-ns = <18000>; + opp-microwatt = <184000>; }; opp03 { opp-hz = /bits/ 64 <1188000000>; opp-level = <3>; clock-latency-ns = <21000>; + opp-microwatt = <253000>; }; opp04 { opp-hz = /bits/ 64 <1452000000>; opp-level = <4>; clock-latency-ns = <24000>; + opp-microwatt = <358000>; }; opp05 { opp-hz = /bits/ 64 <1704000000>; opp-level = <5>; clock-latency-ns = <28000>; + opp-microwatt = <469000>; }; opp06 { opp-hz = /bits/ 64 <1968000000>; opp-level = <6>; clock-latency-ns = <31000>; + opp-microwatt = <620000>; }; opp07 { opp-hz = /bits/ 64 <2208000000>; opp-level = <7>; clock-latency-ns = <33000>; + opp-microwatt = <775000>; }; opp08 { opp-hz = /bits/ 64 <2400000000>; opp-level = <8>; clock-latency-ns = <45000>; + opp-microwatt = <927000>; }; opp09 { opp-hz = /bits/ 64 <2568000000>; opp-level = <9>; clock-latency-ns = <47000>; + opp-microwatt = <1083000>; }; opp10 { opp-hz = /bits/ 64 <2724000000>; opp-level = <10>; clock-latency-ns = <50000>; + opp-microwatt = <1275000>; }; opp11 { opp-hz = /bits/ 64 <2868000000>; opp-level = <11>; clock-latency-ns = <52000>; + opp-microwatt = <1467000>; }; opp12 { opp-hz = /bits/ 64 <3000000000>; opp-level = <12>; clock-latency-ns = <57000>; + opp-microwatt = <1701000>; }; opp13 { opp-hz = /bits/ 64 <3132000000>; opp-level = <13>; clock-latency-ns = <60000>; + opp-microwatt = <1933000>; }; opp14 { opp-hz = /bits/ 64 <3264000000>; opp-level = <14>; clock-latency-ns = <64000>; + opp-microwatt = <2216000>; }; opp15 { opp-hz = /bits/ 64 <3360000000>; opp-level = <15>; clock-latency-ns = <64000>; + opp-microwatt = <2299000>; turbo-mode; }; opp16 { opp-hz = /bits/ 64 <3408000000>; opp-level = <16>; clock-latency-ns = <64000>; + opp-microwatt = <2340000>; turbo-mode; }; opp17 { opp-hz = /bits/ 64 <3504000000>; opp-level = <17>; clock-latency-ns = <64000>; + opp-microwatt = <2424000>; turbo-mode; }; }; From 84bb560eae24d18e014da4ab3450e125ab70b157 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:03:34 -0500 Subject: [PATCH 206/392] dt-bindings: soc: apple: apple,sep: Take the T6050 SEP through its mailbox node The T6050 SEP was described from the ADT with its four mailbox interrupts on its own node and had to stay disabled. Its mailbox is an ASC v4 mailbox at the SEP + 0x8000, as on the other SoCs, so describe it as its own node there and require mboxes for every SEP. T6050 keeps its second reg and its clock. Signed-off-by: Diljit Singh --- .../bindings/soc/apple/apple,sep.yaml | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml index 4ab4e2439b7aa..1a20d2ce27eb5 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -39,12 +39,6 @@ properties: mbox-names: const: mbox - interrupts: - maxItems: 4 - description: - The coprocessor's mailbox interrupts, on a SoC whose SEP mailbox is not - described as a separate node. - clocks: maxItems: 1 @@ -83,6 +77,8 @@ required: - compatible - reg - iommus + - mboxes + - mbox-names allOf: - if: @@ -91,26 +87,12 @@ allOf: contains: const: apple,t6050-sep then: - description: - The T6050 SEP is described from the ADT and stays disabled until its - mailbox and firmware handoff are supported. properties: reg: minItems: 2 - interrupts: - minItems: 4 - status: - const: disabled - required: - - interrupts - - status else: properties: - interrupts: false clocks: false - required: - - mboxes - - mbox-names additionalProperties: false From 7bdc93485d2db05dbc422cae019480d4686aa558 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:03:35 -0500 Subject: [PATCH 207/392] arm64: dts: apple: t6050: Add the SEP's mailbox and DART window iBoot hands over a running SEP on the M5 Pro (sepfw-booted in the J714sAP ADT). Describe what apple_sep needs to attach to it: the ASC v4 mailbox at the SEP + 0x8000, with the first four ADT interrupts in mailbox order (533 to 536: send empty, send not empty, receive empty, receive not empty), and the DMA window iBoot programmed in the SEP DART (dart-sep vm-base-0 0x10000004000, vm-size-0 0xffff0000), which T8140 has too. Everything stays disabled: a board enables the mailbox, and apple_sep enables the SEP and its DART when it loads. Signed-off-by: Diljit Singh --- .../boot/dts/apple/t6050-peripherals.dtsi | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index eef667d647d3d..c2faa9065b0f7 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2200,25 +2200,44 @@ status = "disabled"; }; + /* iBoot hands over a running SEP; apple_sep enables it and its DART when it loads. */ sep: sep@502600000 { compatible = "apple,t6050-sep", "apple,sep"; reg = <0x5 0x2600000 0x0 0x88000>, <0x5 0x2050000 0x0 0x4000>; - interrupts = , - , - , - ; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; iommus = <&sep_dart 0>; power-domains = <&ps_sep>; clocks = <&peripheral_clocks 447>; status = "disabled"; }; + /* The ASC v4 mailbox at sep + 0x8000; the first four ADT interrupts in mailbox order */ + sep_mbox: mailbox@502608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x5 0x2608000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + sep_dart: iommu@500b80000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x5 0xb80000 0x0 0x20000>; interrupts = ; #iommu-cells = <1>; + /* + * The window iBoot programmed (ADT dart-sep vm-base-0, vm-size-0), which + * already carries the BIT(40) alias the DART translates. + */ + apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; + apple,dma-offset = /bits/ 64 <0x0>; status = "disabled"; }; From fcd9b17e92585cb89b6f9f1c91d12723701ee985 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:03:36 -0500 Subject: [PATCH 208/392] soc: apple: sep: Add the M5 Pro (J714s) profile The J714sAP ADT (26A428) describes the enclave as J616s's: pre-booted (sepfw-booted), an iop-sep,ascwrap-v6 SEP behind a DART whose window starts at the BIT(40) alias (vm-base-0 0x10000004000, 0xffff0000 long), and J700's Mesa sensor (sensor-id 0x3356) on spi2, here at 0x505108000, 8 MHz, chip select 0. Take T6040's profile with that sensor bus. Signed-off-by: Diljit Singh --- drivers/soc/apple/profile.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index b824b00c16042..5accaf70cb404 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -26,6 +26,8 @@ //! the T8140 SEP DART aperture above 4 GiB. //! * `T6040` / J616s (MacBook Pro 16", M4 Pro): iBoot boots sepOS, so the //! driver uses the warm registration path; the sensor is the J700 part. +//! * `T6050` / J714s (MacBook Pro 14", M5 Pro): T6040's SEP and sensor, with +//! the sensor's bus at its own address. // The boot handshake, identity source, sensor and DART fields are consumed by // the boot-endpoint, identity and sensor-transport paths. @@ -457,6 +459,21 @@ pub(crate) const T6041: PlatformProfile = PlatformProfile { ..T6040 }; +/// MacBook Pro 14" M5 Pro (J714s). The J714sAP ADT (26A428) records a +/// pre-booted SEP (`sepfw-booted`), an `iop-sep,ascwrap-v6` SEP whose DART +/// aperture starts at the BIT(40) alias (`vm-base-0` 0x10000004000, +/// `vm-size-0` 0xffff0000), and the Mesa sensor J700 carries (`sensor-id` +/// 0x3356) on spi2 at `0x505108000`, 8 MHz, chip select 0. That is J616s's +/// SEP, so T6040's profile with this sensor bus. +pub(crate) const T6050: PlatformProfile = PlatformProfile { + name: "T6050/J714s", + sensor: SensorProfile { + controller_base: 0x5_0510_8000, + ..T6040.sensor + }, + ..T6040 +}; + /// MacBook Air M4 (J713). The J713 ADT (26.6.2, 25G83) puts the Mesa sensor /// on spi2 at `0x3ad108000`, 8 MHz, chip select 0, with power on the NUB GPIO /// controller (pin 28) and data-ready on AP GPIO 199. The SEP is @@ -663,6 +680,9 @@ fn select( if has(b"apple,t6041") && has(b"apple,j616c") { return Ok(&T6041); } + if has(b"apple,t6050") && has(b"apple,j714s") { + return Ok(&T6050); + } // Only the M4 MacBook Airs: the other T8132 Macs are unread. if has(b"apple,t8132") && (has(b"apple,j713") || has(b"apple,j715")) { return Ok(&T8132); @@ -745,6 +765,8 @@ mod tests { assert_eq!(picked(&[b"apple,j614s", b"apple,t6040"]), Some(T6040.name)); assert_eq!(picked(&[b"apple,j616c", b"apple,t6041"]), Some(T6041.name)); assert_eq!(picked(&[b"apple,j713", b"apple,t6041"]), None); + assert_eq!(picked(&[b"apple,j714s", b"apple,t6050"]), Some(T6050.name)); + assert_eq!(picked(&[b"apple,t6050"]), None); } /// Touch ID has one path for every Mac. The retry limits, the lock @@ -755,7 +777,7 @@ mod tests { /// profile naming one of the known Mesa revisions. #[test] fn every_touch_id_mac_gets_a_sensor_the_shared_path_decodes() { - let macs: [&[&[u8]]; 14] = [ + let macs: [&[&[u8]]; 15] = [ &[b"apple,j313", b"apple,t8103"], &[b"apple,j316s", b"apple,t6000"], &[b"apple,j316c", b"apple,t6001"], @@ -770,6 +792,7 @@ mod tests { &[b"apple,j616c", b"apple,t6041"], &[b"apple,j713", b"apple,t8132"], &[b"apple,j700", b"apple,t8140"], + &[b"apple,j714s", b"apple,t6050"], ]; for compatibles in macs { let has = |c: &[u8]| compatibles.iter().any(|x| *x == c); @@ -829,6 +852,7 @@ mod dart_window_tests { &T8122, &T6040, &T6041, + &T6050, &T8132, ] { assert_eq!(offset(profile.dart_window), 1 << 40); From 208b1496c2036506cea526f8b19abc7ffca52c14 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:46:48 -0500 Subject: [PATCH 209/392] arm64: dts: apple: t6050: Leave the SEP's power domain to its firmware iBoot starts the enclave, and the enclave gates its own domain for a sleep (DEV_DISABLE in the state word, which macOS clears at the wake). Its state word idles at actual 0 under auto-enable while it runs, so the pwrstate driver would take it for off and write it. Mark it firmware-owned: genpd keeps it on and never touches the register, and the SEP driver reads the word itself. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-peripherals.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index c2faa9065b0f7..e82926b257398 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -545,6 +545,7 @@ power-domains = <&ps_fab8_soc>; status = "disabled"; }; + /* The enclave runs from iBoot and gates its own domain: never Linux's to change. */ ps_sep: power-controller@298 { compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; reg = <0x298 4>; @@ -552,6 +553,7 @@ #reset-cells = <0>; label = "sep"; power-domains = <&ps_fab8_soc>; + apple,firmware-owned; status = "disabled"; }; ps_sio_cpu: power-controller@2a0 { From 8c09eadb70516dda2b41d876c5487a13b9837897 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:56:14 +1100 Subject: [PATCH 210/392] arm64: dts: apple: t8140-j700: Enable the AOP and always-on GPIO banks Enable the two GPIO banks the AP can reach next to the AOP: - pinctrl@310824000 (aop-gpio, 57 pins) with only the host interrupt groups 4-6 (AIC 384-386) and apple,irq-group-base = <4>, so the chained handler never reads the pending bits of the groups the AOP firmware programs. - pinctrl@3081f0000 (nub-gpio, 30 pins) without interrupts, the shape the J616s runs, and line 20 (the secure-repair enable) in gpio-reserved-ranges so it can be neither requested nor accessed through the regmap. No consumer requests a line. Both power states are always-on. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-j700.dts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index e4634a965e642..bbaf941493c2f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -699,6 +699,30 @@ status = "okay"; }; +/* + * The AOP's GPIO bank. The AOP keeps interrupt groups 0-3; the AP's are 4-6 + * (AIC 384-386). No consumer requests a line yet. + */ +&pinctrl_aop { + interrupts = , + , + ; + apple,irq-group-base = <4>; + status = "okay"; +}; + +/* + * The always-on GPIO bank, as on the J616s: the AP takes none of its + * interrupts. Line 20 is the secure-repair enable and is never requested. + */ +&pinctrl_nub { + gpio-reserved-ranges = <20 1>; + status = "okay"; + /delete-property/ interrupt-controller; + /delete-property/ #interrupt-cells; + /delete-property/ interrupts; +}; + &smc_gpio { apple,pcio-templates = <10 0x800000 0xa0000>; }; From 78a9f60f8e5f9720dfed07c2c3918932ffc8b6eb Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:56:49 +1100 Subject: [PATCH 211/392] dt-bindings: pwm: apple,s5l-fpwm: Allow the controller interrupt Each FPWM instance has its own interrupt line. Allow one optional interrupts entry so nodes can describe it; the driver does not use it. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index 6988fb5da2d95..f7dcfa5bb4d29 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -36,6 +36,9 @@ properties: clocks: maxItems: 1 + interrupts: + maxItems: 1 + power-domains: maxItems: 1 From 44672bdce7c2157a891de6736655b4ed58760e75 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:56:49 +1100 Subject: [PATCH 212/392] arm64: dts: apple: t8112-j413, t8140-j700: Enable the spare FPWM0 channel The T8112 block at 0x235040000 sits next to FPWM1 (the keyboard backlight) and is gated by FPWM0. Describe it as an FPWM instance for pwm-apple and enable it on the J413. Enable the T8140 FPWM0 on the J700. No consumer references either channel: each shows up as a one-channel pwmchip that stays disabled until a user configures it. Probe only reads the control register. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-j413.dts | 5 +++++ arch/arm64/boot/dts/apple/t8112-peripherals.dtsi | 10 ++++++++-- arch/arm64/boot/dts/apple/t8140-j700.dts | 5 +++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 9c9e7532ae048..a596e8f8c53e0 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -168,6 +168,11 @@ status = "okay"; }; +/* A spare PWM channel for developers; no consumer is known. */ +&fpwm0 { + status = "okay"; +}; + &fpwm1 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi index 1869b990d571f..03d32043891eb 100644 --- a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi @@ -72,12 +72,18 @@ status = "disabled"; }; - m2_pwm: pwm@235040000 { - compatible = "apple,t8112-pwm", "apple,pwm"; + /* + * The FPWM0 instance, next to the keyboard backlight's FPWM1. No board + * wires it to anything known: a spare PWM channel with no consumer. + */ + fpwm0: pwm@235040000 { + compatible = "apple,t8112-fpwm", "apple,s5l-fpwm"; reg = <0x2 0x35040000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index bbaf941493c2f..4b5697595bc3b 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -695,6 +695,11 @@ }; }; +/* A spare PWM channel for developers; no consumer is known. */ +&fpwm0 { + status = "okay"; +}; + &smc { status = "okay"; }; From 498e9b1a3ab1fb2c3f637ecb5f82c110bf987f4b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 23:01:02 +1100 Subject: [PATCH 213/392] selftests: apple: Model phy_reset() in the dwc3-apple host tests dwc3_apple_init() now resets the USB2 port through phy_reset() once the host is up. The dwc3 teardown model extracts dwc3_apple_init() and the PHY core functions it calls, so it needs phy_reset() and a reset member in its phy_ops; none of its PHYs has one, as on every SoC before T6050. neo-stability-tests also builds dwc3_apple_init() against a reduced dwc3_apple. Give it the usb2_phy member and a no-op phy_reset(). Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/neo-stability-tests.py | 4 +++- tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c | 1 + tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index 067a9d502801c..1d0a5302af4ce 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -80,7 +80,7 @@ def function(source, name): enum dwc3_apple_state {DWC3_APPLE_PROBE_PENDING,DWC3_APPLE_NO_CABLE,DWC3_APPLE_HOST,DWC3_APPLE_DEVICE,DWC3_APPLE_SUSPENDED}; #define DWC3_GCTL_PRTCAP_HOST 1 struct dwc3 {int num_usb3_ports,num_usb2_ports;int usb3_generic_phy[2],usb2_generic_phy[2],usb3_phy,usb2_phy;int current_dr_role;bool dis_u3_susphy_quirk;}; -struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy;unsigned usb3_power_off_pending,usb2_power_off_pending;}; +struct dwc3_apple {struct device*dev;struct dwc3 dwc;int lock,reset;enum dwc3_apple_state state;bool fixed_hub,host_only,rpm_held,core_active,core_exit_pending,reset_pending,dt_dis_u3_susphy;unsigned usb3_power_off_pending,usb2_power_off_pending;int usb2_phy;}; static int core_error,host_error,gadget_error,reset_error,role_error,start_role; static unsigned rpm_puts; /* A started core holds the glue runtime PM reference until core_stop drops it. */ @@ -98,6 +98,8 @@ def function(source, name): #define BIT(x) (1UL << (x)) static void dwc3_event_buffers_cleanup(struct dwc3*d){} static int phy_power_off(int p){return 0;} +/* Only the T6050 Type-C USB2 PHY has a port reset; for the PHYs here it is a no-op. */ +static int phy_reset(int p){return 0;} static void usb_phy_set_suspend(int p,int suspend){} static void dwc3_core_exit_after_phy_power_off(struct dwc3*d){core_exits++;} static u16 get_unaligned_le16(const u8*p){return p[0]|p[1]<<8;} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 806ec38dd7fd4..4c9f0ac4faad6 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -36,6 +36,7 @@ struct phy_ops { int (*exit)(struct phy *); int (*power_on)(struct phy *); int (*power_off)(struct phy *); + int (*reset)(struct phy *); }; struct phy { struct apple_atcphy *data; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index f42f8251fee56..0bf6edeac8c19 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1091,7 +1091,7 @@ def fn(n, name): 'atcphy_usb2_init', 'atcphy_usb3_power_off', 'atcphy_usb3_route', 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), - 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), + 'phy': ('phy_init', 'phy_exit', 'phy_reset', 'phy_power_on', 'phy_power_off'), 'core': ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', 'dwc3_core_exit'), 'apple': ('dwc3_apple_host_before_core_init', 'dwc3_apple_core_stop', 'dwc3_apple_core_init', 'dwc3_apple_core_start', 'dwc3_apple_init', 'dwc3_apple_exit', 'dwc3_usb_role_switch_set'), From 44826a01ea0b34b1745b5255eb72985e7e9b7ab9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 214/392] phy: apple: t6050: Enable the dummy PIPE PHY for the USB3 lock The PIPE lock that starts the USB3 handoff is acknowledged only while the combo PHY's dummy PIPE PHY is enabled (pipehandler NONSELECTED_OVERRIDE bit 15), as on the earlier ATC PHYs (T8122, T8140), and the bit resets to clear with the controller's domain. Without it the lock timed out, the PHY faulted and rear port 2 had no USB at all. Enable it before the controller leaves reset. Each PIPE-switch step returns its own code, and every fault of the PHY is logged with its reason (the pipehandler registers for PIPE faults); dwc3-apple logs a fatal USB3 PHY failure. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 37 ++++++++++++++++--- .../apple/usb-t6050/apple_t6050_usb3_pipe.h | 19 +++++++--- drivers/usb/dwc3/dwc3-apple.c | 4 +- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index 3c3454ceb64de..6a9d97d233dcd 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -79,6 +79,17 @@ static int combo_quiesce(void *ctx) return ret; } +/* The pipehandler bank is the controller's: read it only while the controller holds its domain. */ +static void combo_pipe_fault(struct t6050_dp *dp, const char *what, int ret) +{ + dev_err(dp->dev, "%s failed (%d): pipe 00=%08x 04=%08x 0c=%08x 10=%08x 14=%08x 1c=%08x 20=%08x\n", + what, ret, readl(dp->banks[44]), readl(dp->banks[44] + 0x4), + readl(dp->banks[44] + 0xc), readl(dp->banks[44] + 0x10), + readl(dp->banks[44] + 0x14), readl(dp->banks[44] + 0x1c), + readl(dp->banks[44] + 0x20)); + dp->faulted = true; +} + static int combo_stop(struct t6050_dp *dp) { int ret; @@ -101,6 +112,7 @@ static int combo_stop(struct t6050_dp *dp) if (!ret) ret = atc_domain_set(&dp->domain[0], false); if (ret) { + dev_err(dp->dev, "stop failed (%d)\n", ret); dp->faulted = true; return ret < 0 ? ret : -EIO; } @@ -142,6 +154,7 @@ static int combo_start(struct t6050_dp *dp, enum combo_mode mode) } if (ret) { /* A partial shared transition has no cold-start ownership proof. */ + dev_err(dp->dev, "start of mode %d failed (%d)\n", mode, ret); dp->faulted = true; return ret < 0 ? ret : -EIO; } @@ -217,10 +230,12 @@ static int combo_usb2_mode(struct phy *phy, enum phy_mode mode, int submode) if (dp->usb2_active) return 0; ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb2_host); - if (ret) + if (ret) { + dev_err(dp->dev, "USB2 host start failed (%d)\n", ret); dp->faulted = true; - else + } else { dp->usb2_active = true; + } return ret; } @@ -233,8 +248,10 @@ static int combo_usb2_reset(struct phy *phy) if (dp->faulted || !dp->usb2_active) return -ENOLINK; ret = apple_t6050_usb2_port_reset(&dp->usb_io, dp, combo_sleep); - if (ret) + if (ret) { + dev_err(dp->dev, "USB2 port reset failed (%d)\n", ret); dp->faulted = true; + } return ret ? -EIO : 0; } @@ -261,7 +278,7 @@ static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) apple_tunable_apply(dp->banks[44], dp->pipe_default); ret = apple_t6050_usb3_pipe_switch(&dp->usb_io); if (ret) - dp->faulted = true; + combo_pipe_fault(dp, "USB3 PIPE switch", ret); else dp->pipe_active = true; return ret ? -EIO : 0; @@ -287,8 +304,10 @@ static int combo_usb3_poweroff(struct phy *phy) if (!ret) dp->usb2_active = false; } - if (ret) + if (ret) { + dev_err(dp->dev, "USB3 power-off failed (%d)\n", ret); dp->faulted = true; + } return ret ? -EIO : 0; } static const struct phy_ops combo_usb2_ops = { @@ -312,7 +331,7 @@ static int combo_reset_assert(struct reset_controller_dev *reset, unsigned long apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0x10); ret = apple_t6050_usb3_pipe_cold_dummy(&dp->usb_io); if (ret) - dp->faulted = true; + combo_pipe_fault(dp, "dummy PIPE park", ret); else dp->controller_active = false; return ret ? -EIO : 0; @@ -326,6 +345,12 @@ static int combo_reset_deassert(struct reset_controller_dev *reset, unsigned lon return -EINVAL; if (dp->faulted || !dp->usb2_active) return -ENOLINK; + /* + * The PIPE lock that starts the USB3 handoff is acknowledged only while the dummy PHY + * behind the mux is enabled (NONSELECTED_OVERRIDE bit 15), as on the earlier ATC PHYs. + * The bit resets to clear with the controller's domain, so set it for every session. + */ + apple_t6050_usb_mask(&dp->usb_io, 44, 0x20, BIT(15), BIT(15)); apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 0x10, 0); apple_t6050_usb_mask(&dp->usb_io, 44, 0x1c, 1, 1); dp->controller_active = true; diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h index 953a42dbc7aab..f70f6f1090381 100644 --- a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -36,6 +36,11 @@ apple_t6050_usb3_pipe_cold_dummy(const struct apple_t6050_usb_io *io) return (io->read(io->context, 44, 0x0c) & 0x73) == 0x42 ? 0 : -1; } +/* + * Returns 0, or the failing step: -2 the PIPE was not parked on a known producer, -3 the lock was + * not acknowledged, -4..-6 the BIST, -7 the mux did not read back USB3, -8 the BIST did not stop, + * -9 the lock release was not acknowledged. Up to -6 the mux is untouched. + */ static inline int apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) { @@ -48,22 +53,26 @@ apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) mux = io->read(io->context, 44, 0x0c) & 0x73; /* Never reroute an active USB4 producer or guess an unknown state. */ if (!(aon & 1) || (mux != 0x42 && mux != 0x10)) - return -1; + return -2; /* Native T8142 xHCI USB3 mode switch, native27 T6050-compatible path. */ apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); apple_t6050_usb_mask(io, 44, 0x00, 1, 1); apple_t6050_usb_mask(io, 44, 0x10, 1, 1); locked = 1; + result = -3; if (io->poll(io->context, 44, 0x14, 1, 1, 6000)) goto out; bist = 1; + result = -4; if (apple_t6050_usb_bist_on(io)) goto out; apple_t6050_usb_mask(io, 44, 0x20, 0x100f, 3); + result = -5; if (io->poll(io->context, 44, 0x20, 0xf, 3, 1000)) goto out; + result = -6; if (apple_t6050_usb_bist_off(io)) goto out; bist = 0; @@ -73,16 +82,16 @@ apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) io->delay(io->context, 5); apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0); apple_t6050_usb_mask(io, 44, 0x0c, 0x73, 0x10); - result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x10 ? 0 : -1; + result = (io->read(io->context, 44, 0x0c) & 0x73) == 0x10 ? 0 : -7; out: if (bist && apple_t6050_usb_bist_off(io)) - result = -1; + result = -8; apple_t6050_usb_mask(io, 44, 0x00, 5, 0); if (locked) { apple_t6050_usb_mask(io, 44, 0x10, 1, 0); - if (io->poll(io->context, 44, 0x14, 1, 0, 6000)) - result = -1; + if (io->poll(io->context, 44, 0x14, 1, 0, 6000) && !result) + result = -9; } return result; } diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index f2484ee172862..f66370e6b5f88 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -671,8 +671,10 @@ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_stat * success, and its errors leave the PIPE without a known producer. */ if (of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3") || - of_device_is_compatible(appledwc->dev->of_node, "apple,t6050-dwc3")) + of_device_is_compatible(appledwc->dev->of_node, "apple,t6050-dwc3")) { + dev_err(appledwc->dev, "USB3 PHY setup failed, err=%d\n", ret); return ret; + } dev_warn(appledwc->dev, "USB3 PHY setup failed, USB2 only, err=%d\n", ret); } else if (host && appledwc->usb3_phy && of_device_is_compatible(appledwc->dev->of_node, "apple,t8132-dwc3")) { From 4e780eb1f424544e685674f2460dd0f0cbdf1860 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 215/392] phy: apple: t6050: Stay on USB2 when the PIPE never left the dummy A USB3 PIPE switch can fail before the mux moves, with the PIPE still on its dummy producer. The PHY then faulted and lost USB2 with it. Leave the port on USB2 in that case, as four DisplayPort lanes do. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index 6a9d97d233dcd..d4aa80de61ac8 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -277,6 +277,11 @@ static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) return (readl(dp->banks[44] + 0xc) & 0x73) == 0x10 ? 0 : -EIO; apple_tunable_apply(dp->banks[44], dp->pipe_default); ret = apple_t6050_usb3_pipe_switch(&dp->usb_io); + if (ret >= -6 && ret < 0 && (readl(dp->banks[44] + 0xc) & 0x73) == 0x42) { + /* The PIPE never left its dummy producer: the port stays USB2, as with four DP lanes. */ + dev_warn(dp->dev, "USB3 PIPE switch failed (%d), the port runs USB2 only\n", ret); + return 0; + } if (ret) combo_pipe_fault(dp, "USB3 PIPE switch", ret); else From 931f247beeb2cdc67b078f511e3825373e1f95c1 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 01:50:54 -0800 Subject: [PATCH 216/392] phy: apple: t6050: Accept every Type-C port but the debug port The combo PHY refused any apple,port-index other than 1 or 2, the two ports a guest description produced. Nothing else in the driver reads the index: the firmware buffer, banks and sequences are port-agnostic. Validate it against the board's port set instead: 1 to 5, with 0 (the port that carries the boot loader's debug link) still excluded, here and in the binding. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml | 4 ++-- drivers/phy/apple/atc-t6050-dp.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml index fdc2e60461700..8cbfe94b62ecd 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml @@ -38,8 +38,8 @@ properties: apple,port-index: $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2] - description: Native Type-C port; the debug/proxy port is excluded + enum: [1, 2, 3, 4, 5] + description: Type-C port; the debug port (0) is excluded apple,guest-port-reserved: type: boolean diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c index 4479153919488..01c666bb9e7dc 100644 --- a/drivers/phy/apple/atc-t6050-dp.c +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -623,8 +623,9 @@ static int t6050_dp_probe(struct platform_device *pdev) dp->combo = of_device_is_compatible(dev->of_node, "apple,t6050-combo-phy"); if (dp->combo) { u32 port; + /* Every Type-C port but 0, the debug port that carries the boot link. */ if (of_property_read_u32(dev->of_node, "apple,port-index", &port) || - (port != 1 && port != 2) || + port < 1 || port > 5 || !of_property_present(dev->of_node, "apple,guest-port-reserved") || !of_property_present(dev->of_node, "mode-switch")) return -EINVAL; From 03a0475542f1a47ffdc5cb4b452a5086b31b1ec8 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:22:09 -0500 Subject: [PATCH 217/392] arm64: dts: apple: j716c: Add the right USB-C port (ATC2) Describe the right-hand USB-C port: its SN201202x PD controller, the ATC2 combo PHY, usb-drd2 and its DARTs, and their power domains. On J716c the port's HPM is ADT /arm-io/nub-spmi-a1/hpm2 (SPMI address 0xc, port-location "right"), on its own SPMI bus, not on nub-spmi-a0 at address 8 as on J775D; on J716c that address is hpm5 (port-type 0x11), not a port's controller. Add nub-spmi-a1 to t6050-max.dtsi, polled as nub-spmi-a0 is, put the HPM on it, and enable the ATC2 chain that J775D leaves to the display handoff. Add the atcphy2 alias so that the boot loader fills the PHY's tunables from /arm-io/atc-phy2, and turn dcpext0's Type-C route off with dcpext0: a mode-switch endpoint whose mux is never registered would defer the PD controller forever. Developed with Claude Code. Signed-off-by: Erik Bourget --- arch/arm64/boot/dts/apple/t6050-j716c.dts | 122 +++++++++++++++++- arch/arm64/boot/dts/apple/t6050-max.dtsi | 13 ++ .../boot/dts/apple/t6050-peripherals.dtsi | 2 +- 3 files changed, 135 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index 33fd4e1c9b03c..550aef6f8768f 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -13,6 +13,7 @@ /dts-v1/; +#define T6050_J716C #include "t6050-max.dtsi" / { @@ -220,5 +221,124 @@ apple,output-commands = /bits/ 64 <0x001a000000000000 0x000f000000800000 0x011c000000010000>; }; -#define T6050_J716C #include "t6050-peripherals.dtsi" + +/* + * ==== The right USB-C port (ATC2) ==== + * + * ADT /arm-io/nub-spmi-a1/hpm2 (SN201202x, SPMI address 0xc, port-number 3, port-location + * "right", acio-parent acio2), atc-phy2, usb-drd2 and dart-usb2. The HPM is on its own bus, + * nub-spmi-a1, and nothing hands the port off, so the chain is enabled outright. Aura's + * kboot_setup_atc() fills the PHY's tunables through the atcphy2 alias. + */ +/ { + aliases { + atcphy2 = &atc2_phy; + }; +}; + +&ps_atc2_common { + status = "okay"; +}; + +&ps_atc2_usb_aon { + status = "okay"; +}; + +&ps_atc2_phymxwrap { + status = "okay"; +}; + +&ps_atc2_usb { + status = "okay"; +}; + +&atc2_phy_mbox { + status = "okay"; +}; + +&atc2_phy { + status = "okay"; + + port { + atc2_phy_ep: endpoint { + remote-endpoint = <&hpm2_ss>; + }; + }; +}; + +&usb_drd2 { + status = "okay"; + + port { + usb_drd2_hs: endpoint { + remote-endpoint = <&hpm2_hs>; + }; + }; +}; + +&usb_drd2_dart_0 { + status = "okay"; +}; + +&usb_drd2_dart_1 { + status = "okay"; +}; + +/* + * dcpext0's Type-C route stays off with dcpext0: a mode-switch endpoint whose mux is never + * registered would defer the PD controller forever. + */ +&dcpext0 { + typec-routes { + route@0 { + status = "disabled"; + + port { + dcpext0_typec: endpoint { + remote-endpoint = <&hpm2_dp>; + }; + }; + }; + }; +}; + +&nub_spmi_a1 { + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + hpm2_hs: endpoint { + remote-endpoint = <&usb_drd2_hs>; + }; + }; + + port@1 { + reg = <1>; + hpm2_ss: endpoint { + remote-endpoint = <&atc2_phy_ep>; + }; + }; + + port@2 { + reg = <2>; + hpm2_dp: endpoint { + remote-endpoint = <&dcpext0_typec>; + }; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index 760094fce8097..55cf22a98e2a2 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -115,4 +115,17 @@ interrupt-controller; #interrupt-cells = <2>; }; + +#ifdef T6050_J716C + /* + * /arm-io/nub-spmi-a1: reg[0] 0x8a1a8000, generation 4, a bus of USB-C PD controllers (the + * board's). They are polled, as on nub-spmi-a0, so the bus takes no interrupt. + */ + nub_spmi_a1: spmi@28a1a8000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a1a8000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + }; +#endif }; diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index eef667d647d3d..ae450252f2acb 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -1319,7 +1319,7 @@ }; #endif -#if defined(T6050_J714C) || defined(T6050_J714S) || defined(T6050_J716C) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) +#if defined(T6050_J714C) || defined(T6050_J714S) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) nub_spmi_a1: spmi@28a1a8000 { compatible = "apple,t6050-spmi", "apple,t8142-spmi"; reg = <0x2 0x8a1a8000 0x0 0x4000>, From a1d086a6f287ddb74788f6a634ecfc4de343c4e8 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:35:50 -0500 Subject: [PATCH 218/392] phy: apple: t6050: carry on when the PIPE lock is not acknowledged On J716c the PIPE handler does not acknowledge the lock request when the controller switches from the dummy producer to USB3, even with the dummy PHY behind the mux enabled. The switch then fails and the port falls back to USB2. The switch works without the ACK. Wait 6 ms for the lock ACK, and for an unlock ACK to clear, then log the PIPE handler state if it is missing and complete the switch. With this, USB3 comes up on that port. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 12 +++++++++++- .../phy/apple/usb-t6050/apple_t6050_usb3_pipe.h | 14 ++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index d4aa80de61ac8..1da782c4f9de4 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -18,9 +18,19 @@ static int combo_poll(void *ctx, unsigned int bank, unsigned int offset, { struct t6050_dp *dp = ctx; u32 value; - return readl_poll_timeout(dp->banks[bank] + offset, value, + int ret = readl_poll_timeout(dp->banks[bank] + offset, value, (value & mask) == expected, (bank == 44 || bank == 35 || usec <= 1000) ? 1 : 1000, usec); + + /* The PIPE switch carries on without the lock ACK; say so. */ + if (ret && bank == 44 && offset == 0x14 && usec == 6000) + dev_warn(dp->dev, + "PIPE %slock not acknowledged; continuing (override=%x values=%x mux=%x req=%x ack=%x aon=%x nonsel=%x)\n", + expected ? "" : "un", readl(dp->banks[44]), + readl(dp->banks[44] + 4), readl(dp->banks[44] + 0xc), + readl(dp->banks[44] + 0x10), value, + readl(dp->banks[44] + 0x1c), readl(dp->banks[44] + 0x20)); + return ret; } static void combo_delay(void *ctx, unsigned int usec) { udelay(usec); } static void combo_pause(void *ctx, unsigned int nsec) { ndelay(nsec); } diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h index f70f6f1090381..812227cac03e9 100644 --- a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -37,9 +37,10 @@ apple_t6050_usb3_pipe_cold_dummy(const struct apple_t6050_usb_io *io) } /* - * Returns 0, or the failing step: -2 the PIPE was not parked on a known producer, -3 the lock was - * not acknowledged, -4..-6 the BIST, -7 the mux did not read back USB3, -8 the BIST did not stop, - * -9 the lock release was not acknowledged. Up to -6 the mux is untouched. + * Returns 0, or the failing step: -2 the PIPE was not parked on a known producer, -4..-6 the + * BIST, -7 the mux did not read back USB3, -8 the BIST did not stop. Up to -6 the mux is + * untouched. A lock or unlock that is not acknowledged within 6 ms is not fatal: it is logged + * and the switch carries on. On J716c the lock ACK does not come. */ static inline int apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) @@ -60,9 +61,7 @@ apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) apple_t6050_usb_mask(io, 44, 0x00, 1, 1); apple_t6050_usb_mask(io, 44, 0x10, 1, 1); locked = 1; - result = -3; - if (io->poll(io->context, 44, 0x14, 1, 1, 6000)) - goto out; + io->poll(io->context, 44, 0x14, 1, 1, 6000); bist = 1; result = -4; @@ -90,8 +89,7 @@ apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) apple_t6050_usb_mask(io, 44, 0x00, 5, 0); if (locked) { apple_t6050_usb_mask(io, 44, 0x10, 1, 0); - if (io->poll(io->context, 44, 0x14, 1, 0, 6000) && !result) - result = -9; + io->poll(io->context, 44, 0x14, 1, 0, 6000); } return result; } From 1856c663bd2501de5e7f3aa86d3216ab3c34d071 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 2 Nov 2026 23:41:32 -0500 Subject: [PATCH 219/392] reset: apple-cio: add the T6050 CIO reconfig request On T6050 the CPU wrapper of each USB4 router is clocked through the CIO reconfig request (the router CPU's clock gate CIOn_RECONFIG_V). Until it has run, any access to the wrapper faults. The request register uses the T8142 bit positions (request BIT(2 * port), four ports) and sits at offset 0x60 of a page in the die's PMGR block, 0x280620000 on die 0. Its odd bit is a done flag rather than a busy flag: it is set once the request is taken, then request and done clear by themselves about 4 ms later. The wrapper takes an SError when it is accessed before they clear. Wait for both bits to clear (a reconfiguration still under way), store the request, wait for the done flag, then wait for both bits to clear again. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../bindings/reset/apple,t8103-cio-reset.yaml | 3 + drivers/reset/reset-apple-cio.c | 65 +++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index a7bc6bf4885c9..5d7e40b564f91 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -22,6 +22,9 @@ description: | On t8142, a standalone register page contains a shared request register with alternating request and busy bits for four ports. + On t6050 the same request register sits at offset 0x60 of a page in the + die's PMGR block. + In all cases the reset id is the CIO port index on the die. properties: diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index af299473a6c87..e5755ebb39aff 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -37,7 +37,12 @@ * separate 16 KiB page at 0x350780000, so the register is 0x350780008. The * device tree describes that page as its own syscon. t8122 is unresolved. * - * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. + * On t8142, RegMap 0x62 is a separate MMIO region at 0x380780000. On t6050 + * the register is at +0x60 in a page of the die's PMGR block (0x280620000 + * on die 0). There the odd bit is a done flag rather than a busy flag: it + * is set once the request is taken, then request and done clear by + * themselves within a few ms. Only after that is the port's CPU wrapper + * accessible. * The t8132 layout has not been decoded. * * Reference firmware polls the busy bits, stores the request, then polls @@ -56,6 +61,8 @@ #define APPLE_CIO_RESET_POLL_US 100 #define APPLE_CIO_RESET_TIMEOUT_US 100000 +/* A t6050 reconfiguration takes a few ms; allow ample margin. */ +#define APPLE_CIO_RESET_T6050_TIMEOUT_US 192000 #define T8103_CIO_CTRL_STRIDE 0x4000 #define T8103_CIO_CTRL_INIT_REQ BIT(0) @@ -75,6 +82,7 @@ struct apple_cio_reset; struct apple_cio_reset_variant { unsigned int nr_resets; bool pmgr_child; + u32 ctrl; /* request register, t8142 layout */ int (*deassert)(struct apple_cio_reset *priv, unsigned long id); }; @@ -174,19 +182,19 @@ static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) guard(mutex)(&priv->lock); /* Serialize all four ports sharing the request register. */ - ret = regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, !(val & T8142_CIO_CTRL_BUSY_MASK), APPLE_CIO_RESET_POLL_US, APPLE_CIO_RESET_TIMEOUT_US); if (ret) return ret; - ret = regmap_write(priv->regmap, T8142_CIO_CTRL, + ret = regmap_write(priv->regmap, priv->variant->ctrl, T8142_CIO_CTRL_INIT_REQ(id)); if (ret) return ret; - return regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + return regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, !(val & T8142_CIO_CTRL_INIT_BUSY(id)), APPLE_CIO_RESET_POLL_US, APPLE_CIO_RESET_TIMEOUT_US); @@ -194,9 +202,54 @@ static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) static const struct apple_cio_reset_variant apple_t8142_cio_reset = { .nr_resets = 4, + .ctrl = T8142_CIO_CTRL, .deassert = t8142_cio_deassert, }; +static int t6050_cio_deassert(struct apple_cio_reset *priv, unsigned long id) +{ + u32 mask = T8142_CIO_CTRL_INIT_REQ(id) | T8142_CIO_CTRL_INIT_BUSY(id); + u32 val; + int ret; + + guard(mutex)(&priv->lock); + + /* Let a reconfiguration still under way finish first. */ + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + !(val & mask), APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, priv->variant->ctrl, + T8142_CIO_CTRL_INIT_REQ(id)); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + val & T8142_CIO_CTRL_INIT_BUSY(id), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); + if (ret) + return ret; + + /* + * The CPU wrapper is usable only once the reconfiguration is over, when + * the request and done bits clear again, a few ms later. An access + * before that takes an SError. + */ + return regmap_read_poll_timeout(priv->regmap, priv->variant->ctrl, val, + !(val & mask), APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_T6050_TIMEOUT_US); +} + +/* The t8142 bit layout in a register mapped on its own, with a done flag. */ +static const struct apple_cio_reset_variant apple_t6050_cio_reset = { + .nr_resets = 4, + .ctrl = 0x60, + .deassert = t6050_cio_deassert, +}; + static const struct apple_cio_reset_variant apple_t8103_cio_reset = { .nr_resets = 2, .deassert = t8103_cio_deassert, @@ -277,6 +330,10 @@ static int apple_cio_reset_probe(struct platform_device *pdev) } static const struct of_device_id apple_cio_reset_match[] = { + { + .compatible = "apple,t6050-cio-reset", + .data = &apple_t6050_cio_reset, + }, { .compatible = "apple,t8142-cio-reset", .data = &apple_t8142_cio_reset, From 4aee080f4ed7f9a5eee673772f1dc8aa8906a4da Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 18:08:07 -0500 Subject: [PATCH 220/392] dt-bindings: phy: apple,t6050-dp-phy: Add the CIO tunables The CIO (USB4/Thunderbolt) mode has its own firmware register and receiver tunables. They are optional: an absent one is skipped. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../bindings/phy/apple,t6050-dp-phy.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml index 8cbfe94b62ecd..3b22c3f19ee9b 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml @@ -126,6 +126,28 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32-array description: Offset, mask and value tuples within the named bank + apple,tunable-uc-regs-cio: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Offset, mask and value tuples for the PHY firmware's registers in CIO + (USB4/Thunderbolt) mode. Optional; an absent CIO tunable is skipped. + + apple,tunable-lane0-rx-top-cio: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: As apple,tunable-uc-regs-cio, for lane 0's receiver top bank + + apple,tunable-lane1-rx-top-cio: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: As apple,tunable-uc-regs-cio, for lane 1's receiver top bank + + apple,tunable-lane0-rx-eq-cio: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: As apple,tunable-uc-regs-cio, for lane 0's receiver equalizer + + apple,tunable-lane1-rx-eq-cio: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: As apple,tunable-uc-regs-cio, for lane 1's receiver equalizer + patternProperties: '^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$': $ref: /schemas/types.yaml#/definitions/uint32-array From 8726ee82db1b8939997fa9468ddc962f56348f19 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:29:27 -0500 Subject: [PATCH 221/392] phy: apple: t6050: add the CIO (USB4/Thunderbolt) mode The T6050 combo PHY rejects TYPEC_MODE_USB4 and the Thunderbolt alternate mode, so a USB4 dock that the PD controller has already entered USB4 with gets no link: the PHY is never configured and DWC3 cannot start. Add the CIO mode as a step recipe in the style of the USB ones: - the mode-independent power-up; - the common tunables, then the CIO set (AUS40CMN_SHM, AUSPLL_CORE, CIO4PLL_CORE) and the CIO80 set (UC_REGS_CIO, LN0/LN1_RX_TOP_CIO, LN0/LN1_RX_EQ_CIO), with no USB or DP lane masks; - the crossbar in CIO, where orientation only selects TOP +0x78 bit 0; - the sleep-control release of both lanes; - the firmware start, UC_TNBL_DONE and the ACIOPHY reset release, without the PIPE BIST and the ATCPHY_CFG +0x8 orientation bit. The shutdown powers both transmitters and receivers down, shuts the common block down, stops the firmware and runs the usual sleep, clamp and reset sequence. USB2 runs beside the link, so the USB2 PHY is brought up as for USB. USB3 arrives through a tunnel later; in CIO the USB3 PHY set_mode leaves DWC3 on its dummy PIPE producer until a tunnel exists. The five CIO80 tunables are optional DT properties (apple,tunable-uc-regs-cio, -lane0/1-rx-top-cio, -lane0/1-rx-eq-cio); a missing one is skipped. On J716c only the RX_TOP pair is non-empty. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 55 ++++- drivers/phy/apple/atc-t6050-dp.c | 7 +- .../phy/apple/usb-t6050/apple_t6050_cio_phy.h | 189 ++++++++++++++++++ 3 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index 1da782c4f9de4..c57779075ec05 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -54,6 +54,11 @@ static int combo_tunable(void *ctx, unsigned int bank, const char *name) {8, 8, "tunable_AUSPLL_CORE"}, {5, 9, "tunable_AUS40CMN_SHM"}, {10, 10, "tunable_CIO4PLL_CORE"}, }; + static const struct { u8 bank, index; const char *name; } cio[] = { + {3, 0, "tunable_UC_REGS_CIO_DFLT"}, {16, 1, "tunable_LN0_RX_TOP_CIO_DFLT"}, + {25, 2, "tunable_LN1_RX_TOP_CIO_DFLT"}, {18, 3, "tunable_LN0_RX_EQ_CIO_DFLT"}, + {27, 4, "tunable_LN1_RX_EQ_CIO_DFLT"}, + }; struct t6050_dp *dp = ctx; if (bank == 0 && !strcmp(name, "tunable_USB2PHY_DFLT")) { @@ -69,6 +74,12 @@ static int combo_tunable(void *ctx, unsigned int bank, const char *name) apple_tunable_apply(dp->banks[bank], dp->tunables[common[i].index]); return 0; } + /* A missing CIO80 tunable is skipped. */ + for (unsigned int i = 0; i < ARRAY_SIZE(cio); i++) + if (bank == cio[i].bank && !strcmp(name, cio[i].name)) { + apple_tunable_apply(dp->banks[bank], dp->cio_tunables[cio[i].index]); + return 0; + } return -EINVAL; } @@ -81,6 +92,10 @@ static int combo_quiesce(void *ctx) ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_shutdown); if (!ret) dp->usb_prepared = false; + } else if (dp->cio_prepared) { + ret = APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_shutdown); + if (!ret) + dp->cio_prepared = false; } else if (dp->enabled) { /* Multifunction has a distinct native shutdown; never run the * USB-only analog sequence over live DisplayPort lane state. */ @@ -158,6 +173,19 @@ static int combo_start(struct t6050_dp *dp, enum combo_mode mode) APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_usb3_init_normal); if (!ret) dp->usb_prepared = true; + } else if (!ret && mode == COMBO_CIO) { + ktime_t start = ktime_get(); + + /* USB2 stays native beside a USB4/TBT link; USB3 comes tunnelled. */ + ret = dp->reverse ? + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_init_reverse) : + APPLE_T6050_USB_RUN(&dp->usb_platform, apple_t6050_cio_init_normal); + if (!ret) + dp->cio_prepared = true; + dev_info(dp->dev, "CIO mode %s (%d), %s orientation, %lld ms\n", + ret ? "failed" : "up: UC_TNBL_DONE", ret, + dp->reverse ? "reverse" : "normal", + ktime_ms_delta(ktime_get(), start)); } else if (!ret) { dp->multifunction = mode == COMBO_DP2; ret = atc_start(dp); @@ -201,6 +229,11 @@ static int combo_mux(struct t6050_dp *dp, struct typec_mux_state *state) mode = COMBO_SAFE; else if (state->mode == TYPEC_STATE_USB && !state->alt) mode = COMBO_USB; + else if (!state->alt && state->mode == TYPEC_MODE_USB4) + mode = COMBO_CIO; + else if (state->alt && state->alt->svid == USB_TYPEC_TBT_SID && + state->mode == TYPEC_TBT_MODE) + mode = COMBO_CIO; else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { switch (state->mode) { case TYPEC_DP_STATE_C: @@ -215,7 +248,7 @@ static int combo_mux(struct t6050_dp *dp, struct typec_mux_state *state) if (dp->faulted) return -EIO; if (mode == dp->combo_mode && - (mode == COMBO_SAFE || dp->usb_prepared || dp->enabled)) + (mode == COMBO_SAFE || dp->usb_prepared || dp->cio_prepared || dp->enabled)) return 0; if (mode != COMBO_SAFE && dp->orientation == TYPEC_ORIENTATION_NONE) return -ENOLINK; @@ -235,7 +268,7 @@ static int combo_usb2_mode(struct phy *phy, enum phy_mode mode, int submode) return -EIO; if (mode != PHY_MODE_USB_HOST) return -EOPNOTSUPP; - if (!dp->usb_prepared && !dp->enabled) + if (!dp->usb_prepared && !dp->cio_prepared && !dp->enabled) return -ENOLINK; if (dp->usb2_active) return 0; @@ -281,6 +314,10 @@ static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) * uses its independent D+/D- PHY. Do not switch USB3 onto DP lanes. */ if (dp->combo_mode == COMBO_DP4 && dp->enabled) return 0; + /* In CIO the lanes carry USB4/TBT. DWC3 keeps its dummy PIPE until a + * USB3 tunnel hands PIPE to the router; native USB3 mode is 0 too. */ + if (dp->combo_mode == COMBO_CIO && dp->cio_prepared) + return 0; if (!dp->usb_prepared && !(dp->combo_mode == COMBO_DP2 && dp->enabled)) return -ENOLINK; if (dp->pipe_active) @@ -390,6 +427,12 @@ static struct phy *combo_xlate(struct device *dev, const struct of_phandle_args static int combo_setup(struct platform_device *pdev, struct t6050_dp *dp) { + /* Same order as the cio[] table in combo_tunable(). */ + static const struct { u8 bank; const char *name; } cio_tunables[] = { + {3, "apple,tunable-uc-regs-cio"}, {16, "apple,tunable-lane0-rx-top-cio"}, + {25, "apple,tunable-lane1-rx-top-cio"}, {18, "apple,tunable-lane0-rx-eq-cio"}, + {27, "apple,tunable-lane1-rx-eq-cio"}, + }; static const struct { u8 bank; u32 minimum; const char *name; } extra[] = { {0, 0x20, "usb2phy"}, {1, 0x24, "usb2-digital"}, {44, 0x48, "pipehandler"}, }; @@ -417,6 +460,14 @@ static int combo_setup(struct platform_device *pdev, struct t6050_dp *dp) "apple,tunable-pipe-default", &dp->resources[44]); if (IS_ERR(dp->pipe_default)) return PTR_ERR(dp->pipe_default); + for (unsigned int i = 0; i < ARRAY_SIZE(cio_tunables); i++) { + struct apple_tunable *t = devm_apple_tunable_parse(dev, dev->of_node, + cio_tunables[i].name, &dp->resources[cio_tunables[i].bank]); + + if (IS_ERR(t) && PTR_ERR(t) != -ENOENT) + return PTR_ERR(t); + dp->cio_tunables[i] = IS_ERR(t) ? NULL : t; + } dp->usb_io = (struct apple_t6050_usb_io) { .context = dp, .read = combo_read, .write = combo_write, .poll = combo_poll, .delay = combo_delay, diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c index 01c666bb9e7dc..e6941cf86c8ad 100644 --- a/drivers/phy/apple/atc-t6050-dp.c +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -14,6 +14,7 @@ #include "usb-t6050/apple_t6050_usb_phy.h" #include "usb-t6050/apple_t6050_usb3_pipe.h" #include "usb-t6050/apple_t6050_usb2_reset.h" +#include "usb-t6050/apple_t6050_cio_phy.h" #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include enum atc_operation { ATC_MASK, ATC_TUNABLE, ATC_DELAY, ATC_POLL, ATC_POWER, ATC_FIRMWARE }; struct atc_op { @@ -68,9 +70,9 @@ struct atc_domain { struct device *dev; bool active; }; -enum combo_mode { COMBO_SAFE, COMBO_USB, COMBO_DP4, COMBO_DP2 }; +enum combo_mode { COMBO_SAFE, COMBO_USB, COMBO_DP4, COMBO_DP2, COMBO_CIO }; struct t6050_dp { - bool combo, usb_prepared, usb2_active, pipe_active, controller_active; + bool combo, usb_prepared, usb2_active, pipe_active, controller_active, cio_prepared; enum combo_mode combo_mode; enum typec_orientation orientation; struct phy *combo_phy[3]; @@ -78,6 +80,7 @@ struct t6050_dp { struct apple_t6050_usb_io usb_io; struct apple_t6050_usb_platform usb_platform; struct apple_tunable *usb2_default, *usb2_host, *pipe_default; + struct apple_tunable *cio_tunables[5]; /* optional; empty on J716c except RX_TOP */ struct device *dev; struct mutex lock; void __iomem *banks[45]; diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h b/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h new file mode 100644 index 0000000000000..4af28f8c2ccff --- /dev/null +++ b/drivers/phy/apple/usb-t6050/apple_t6050_cio_phy.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: MIT */ +/* ACIO80 PHY CIO (USB4/Thunderbolt) mode. + * + * The bring-up runs the mode-independent power-up (the USB recipes' first + * block), the common and CIO/CIO80 tunables, the crossbar, the sleep-control + * release and the firmware start. CIO applies no DP or USB lane masks, + * no DP clock switch and no orientation bit in ATCPHY_CFG +0x8; orientation + * only selects ACIOPHY_TOP +0x78 bit 0. Both lanes are released. + * Spread-spectrum stays on. + */ +#ifndef APPLE_T6050_CIO_PHY_H +#define APPLE_T6050_CIO_PHY_H +#include "apple_t6050_usb_phy.h" + +/* The power-up and reset release, identical to the USB recipes. */ +#define APPLE_T6050_CIO_POWERUP \ + { T6050_MASK, 39, 0x0, 0x1, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x1, 0x1, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x2, 0x2, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x2, 0x2, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x4, 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_POWER, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x2c, 0x1, 0x0, 0, 0 }, \ + { T6050_POLL, 39, 0x4, 0x100, 0x0, 1000, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 39, 0x0, 0x8, 0x8, 0, 0 } + +/* The common tunables, then the CIO set and the CIO80 set. An absent CIO80 + * tunable is skipped. + */ +#define APPLE_T6050_CIO_TUNABLES \ + { T6050_TUNABLE, 41, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF" }, \ + { T6050_TUNABLE, 40, 0x0, 0x0, 0x0, 0, "tunable_ATC_FABRIC" }, \ + { T6050_TUNABLE, 43, 0x0, 0x0, 0x0, 0, "tunable_ATC_COMMON_CFG" }, \ + { T6050_TUNABLE, 42, 0x0, 0x0, 0x0, 0, "tunable_ATC0AXI2AF_LIOA" }, \ + { T6050_TUNABLE, 20, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_CFG_TX_OF_RXCLK" }, \ + { T6050_TUNABLE, 29, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_CFG_TX_OF_RXCLK" }, \ + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_DFLT" }, \ + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_DFLT" }, \ + { T6050_TUNABLE, 5, 0x0, 0x0, 0x0, 0, "tunable_AUS40CMN_SHM" }, \ + { T6050_TUNABLE, 8, 0x0, 0x0, 0x0, 0, "tunable_AUSPLL_CORE" }, \ + { T6050_TUNABLE, 10, 0x0, 0x0, 0x0, 0, "tunable_CIO4PLL_CORE" }, \ + { T6050_TUNABLE, 3, 0x0, 0x0, 0x0, 0, "tunable_UC_REGS_CIO_DFLT" }, \ + { T6050_TUNABLE, 16, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_TOP_CIO_DFLT" }, \ + { T6050_TUNABLE, 25, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_TOP_CIO_DFLT" }, \ + { T6050_TUNABLE, 18, 0x0, 0x0, 0x0, 0, "tunable_LN0_RX_EQ_CIO_DFLT" }, \ + { T6050_TUNABLE, 27, 0x0, 0x0, 0x0, 0, "tunable_LN1_RX_EQ_CIO_DFLT" } + +/* The crossbar: clear the lane selects, set the orientation bit (+0x78) + * and select CIO. + */ +#define APPLE_T6050_CIO_XBAR(reverse) \ + { T6050_MASK, 2, 0x6c, 0xfc, 0x0, 0, 0 }, \ + { T6050_MASK, 2, 0x78, 0x1f, (reverse), 0, 0 }, \ + { T6050_MASK, 2, 0x6c, 0x3, 0x2, 0, 0 } + +/* Power a lane's receiver up, then release its transmitter overrides. */ +#define APPLE_T6050_CIO_LANE(l) \ + { T6050_MASK, 2, 0x10, 0x10 << (l), 0x10 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x40 << (l), 0x40 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x1 << (l), 0x1 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x4 << (l), 0x4 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x400 << (l), 0x400 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x1000 << (l), 0x1000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x40 << (l), 0x40 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x100 << (l), 0x100 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x4000 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x10000 << (l), 0x10000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x40 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x4 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x400 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* The sleep-control release: the common block's overrides, then both lanes. */ +#define APPLE_T6050_CIO_SLPCTRL \ + { T6050_MASK, 2, 0xc, 0x2a, 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + APPLE_T6050_CIO_LANE(0), \ + APPLE_T6050_CIO_LANE(1) + +/* Start the PHY firmware, wait for UC_TNBL_DONE and + * deassert the ACIOPHY reset. No PIPE BIST and no ATCPHY_CFG +0x8 write. + */ +#define APPLE_T6050_CIO_START \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_MASK, 38, 0x40, 0x1, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0xa, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1388, 0, 0 }, \ + { T6050_POLL, 2, 0x260, 0x400, 0x400, 1000000, 0 }, \ + { T6050_MASK, 39, 0x0, 0x10, 0x10, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x64, 0, 0 } + +static const struct apple_t6050_usb_step apple_t6050_cio_init_normal[] = { + APPLE_T6050_CIO_POWERUP, + APPLE_T6050_CIO_TUNABLES, + APPLE_T6050_CIO_XBAR(0x0), + APPLE_T6050_CIO_SLPCTRL, + APPLE_T6050_CIO_START, +}; + +static const struct apple_t6050_usb_step apple_t6050_cio_init_reverse[] = { + APPLE_T6050_CIO_POWERUP, + APPLE_T6050_CIO_TUNABLES, + APPLE_T6050_CIO_XBAR(0x1), + APPLE_T6050_CIO_SLPCTRL, + APPLE_T6050_CIO_START, +}; + +/* Power a lane's transmitter down. */ +#define APPLE_T6050_CIO_TX_DOWN(l) \ + { T6050_MASK, 2, 0x120, 0x100 << (l), 0x100 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x1 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x120, 0x10 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* Power a lane's receiver down; CIO powers the receivers down, USB does not. */ +#define APPLE_T6050_CIO_RX_DOWN(l) \ + { T6050_MASK, 2, 0xc, 0x4000 << (l), 0x4000 << (l), 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x40 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0xc, 0x400 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x1 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, \ + { T6050_MASK, 2, 0x10, 0x10 << (l), 0x0, 0, 0 }, \ + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 } + +/* The shutdown: both lanes down without the PIPE BIST, the firmware + * power-down, then the USB recipes' sleep, clamp and reset sequence. + */ +static const struct apple_t6050_usb_step apple_t6050_cio_shutdown[] = { + APPLE_T6050_CIO_TX_DOWN(0), + APPLE_T6050_CIO_TX_DOWN(1), + { T6050_MASK, 5, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x10, 0x10, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 2, 0xc, 0x4, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + APPLE_T6050_CIO_RX_DOWN(0), + APPLE_T6050_CIO_RX_DOWN(1), + { T6050_DELAY, 0, 0x0, 0x0, 0xc8, 0, 0 }, + { T6050_FIRMWARE, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_POWER, 0, 0x0, 0x0, 0x0, 0, 0 }, + { T6050_MASK, 39, 0x2c, 0x1, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x100, 0x100, 1000, 0 }, + { T6050_MASK, 39, 0x0, 0x10, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x8, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x4, 0x4, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x2, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x2, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x0, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_POLL, 39, 0x4, 0x1, 0x0, 1000, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, + { T6050_MASK, 39, 0x8, 0x1, 0x0, 0, 0 }, + { T6050_DELAY, 0, 0x0, 0x0, 0x1, 0, 0 }, +}; +#endif From 024beea2bff7b3aed7af357dcf713465f1c93235 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 18:08:06 -0500 Subject: [PATCH 222/392] dt-bindings: thunderbolt: apple: Add the T6050 router and NHI The T6050 host router has the T8132 MxWrap CPU wrapper and LSTX pin, four ACIO power domains instead of three, and a PCIe router AP firmware block with its own tunables. Its NHI has a single register range, one DART stream per ring, and an SRAM parity interrupt after its 24 ring interrupts. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../thunderbolt/apple,t8103-usb4-acio.yaml | 48 +++++++++++++++++-- .../thunderbolt/apple,t8103-usb4-nhi.yaml | 20 +++++++- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index d9dd4cd1aa756..fe61df65c5c27 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -52,7 +52,8 @@ properties: PCIe adapter configuration space of the USB4 router. Present where the root complex exposes the adapter separately, so that the tunnel can be configured before a PCIe-C link is brought up. - - description: MxWrap co-processor control registers + - description: MxWrap co-processor control registers, on T8132 and T6050 + - description: PCIe router AP firmware registers, on T6050 reg-names: oneOf: @@ -62,6 +63,7 @@ properties: - const: sram - const: pcie-adapter - const: cpu + - const: router-ap-fw - description: T6050, which has no separate PCIe adapter window items: - const: rc @@ -87,12 +89,12 @@ properties: power-domains: minItems: 3 - maxItems: 4 + maxItems: 5 description: | - The first three domains power the ACIO co-processor complex. When - apple,pcie-tunnel is present, the fourth domain powers its PCIe-C + The first three domains (four on T6050, the fourth being the routers' + shared audio (AUSS) domain) power the ACIO co-processor complex. When + apple,pcie-tunnel is present, the domain after them powers its PCIe-C controller and is enabled only after the USB4 PCIe tunnel is active. - On T6050 the fourth domain is the routers' shared audio (AUSS) domain. thunderbolt-switch: type: boolean @@ -135,6 +137,18 @@ properties: named by the pcie-adapter reg entry. Only meaningful where that entry is present. + apple,tunable-router-ap-fw: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask of bits to clear + - description: Bits to set + description: | + As apple,tunable-rc, but applied to the PCIe router AP firmware + registers named by the router-ap-fw reg entry, with the other router + tunables once the co-processor firmware is ready. + '#address-cells': const: 1 @@ -216,6 +230,16 @@ allOf: minItems: 4 reg-names: minItems: 4 + - if: + properties: + compatible: + contains: + enum: + - apple,t6040-usb4-acio + - apple,t6050-usb4-acio + - apple,t8132-usb4-acio + then: + properties: pinctrl-names: items: - const: lstx @@ -237,6 +261,20 @@ allOf: required: - iommu@a80000 - nhi@f00000 + - if: + properties: + compatible: + contains: + const: apple,t6050-usb4-acio + then: + properties: + power-domains: + minItems: 4 + else: + properties: + power-domains: + maxItems: 4 + apple,tunable-router-ap-fw: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml index ede4cea33eb2d..28718fed167c5 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -52,10 +52,12 @@ properties: interrupts: description: One interrupt for each of the 12 transmit rings followed by one for each - of the 12 receive rings. - maxItems: 24 + of the 12 receive rings. On T6050 an SRAM parity interrupt follows them. + minItems: 24 + maxItems: 25 interrupt-names: + minItems: 24 items: - const: txring0 - const: txring1 @@ -81,6 +83,7 @@ properties: - const: rxring9 - const: rxring10 - const: rxring11 + - const: parity iommus: minItems: 1 @@ -177,4 +180,17 @@ allOf: iommus: maxItems: 1 + - if: + properties: + compatible: + not: + contains: + const: apple,t6050-usb4-nhi + then: + properties: + interrupts: + maxItems: 24 + interrupt-names: + maxItems: 24 + additionalProperties: false From ea31a9a60c40b660641dc5368d5b3fca8968c6f4 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:31:48 -0500 Subject: [PATCH 223/392] thunderbolt: apple: support the T6050 host routers The M5 Max (T6050) host routers are Type 7 routers with an MxWrap CPU wrapper, close to the M3 ones. They differ at start: - The wrapper is only clocked once the CIO reset (the reconfig request) has run; any access before it faults. The reset is required, as on the other generations, and runs before the wrapper is touched. - The CPU is released with a plain store of the RUN value rather than a read-modify-write, and halted again when the router stops. - Both the rc block and the wrapper take an external abort when read before the CPU runs, so the M3_CTRL start of the other generations is skipped and their state is not logged. The rc block is first touched to hand LSTX over, once RTKit is up. - The router takes the LSTX pin over only once its RTKit is up: write the LSTX control from the level the AP held, then mux the pin to the router. While the router is off, hold the pin as a GPIO at the level the router last drove. - The PCIe router AP firmware tunable is applied with the other router tunables once the firmware is ready. - The router firmware trains Gen4 links only when the cable word says the cable is Gen4. For a USB4 Gen4 cable from Enter_USB it expects 0x801 (present and Gen4), without the 20 Gb/s bit. - The NHI has a named SRAM parity interrupt after its 24 ring interrupts, which is not a ring. Match apple,t6050-usb4-acio and apple,t6050-usb4-nhi. DisplayPort tunnels stay off on T6050 for now. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/thunderbolt/apple.c | 158 +++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 13 deletions(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 3374e0d3e794f..0b0c25d5adb69 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -141,6 +141,17 @@ struct apple_cio_hw { bool strict_fw_ready; /* Keep the links of the host router out of CL states. */ bool no_host_clx; + /* Type 7 router: Enter_USB cable words, bootloader RC tuning, Type 7 NHI. */ + bool type7; + /* + * Release the CPU with a plain store of the RUN value rather than a + * read-modify-write, and halt it again when the router stops. + */ + bool run_store; + /* Hold LSTX low as a GPIO from probe until the router first takes it. */ + bool lstx_hold_from_probe; + /* The router trains Gen4 only on a cable word that says Gen4. */ + bool gen4_cable; const struct apple_dpin_policy *dp; }; @@ -178,11 +189,30 @@ static const struct apple_cio_hw apple_cio_t8132_hw = { /* Type 7 uses its existing ready-state check and has no qualified DP route. */ static const struct apple_cio_hw apple_cio_t8142_hw = { + .type7 = true, + .dp = &apple_dpin_disabled, +}; + +/* + * T6050 is Type 7 with an MxWrap CPU wrapper that is only clocked once the + * CIO reset (the reconfig request) has run. Its CPU is released with a plain + * store of RUN; as on T8132 it takes LSTX over only once its RTKit is up, and + * the AP holds the pin while the router is off. DP tunnels stay off for now. + */ +static const struct apple_cio_hw apple_cio_t6050_hw = { + .mxwrap = true, + .type7 = true, + .run_store = true, + .lstx_after_boot = true, + .lstx_hold_from_probe = true, + .strict_fw_ready = true, + .gen4_cable = true, .dp = &apple_dpin_disabled, }; static const struct apple_cio_hw apple_cio_t6040_hw = { .mxwrap = true, + .type7 = true, .lstx_explicit = true, .skip_run = true, .lstx_after_boot = true, @@ -256,6 +286,10 @@ MODULE_PARM_DESC(s2idle_teardown, #define TB_VSE_CAP_APPLE_CABLE_INFO_20_GBPS BIT(4) #define TB_VSE_CAP_APPLE_CABLE_INFO_LEGACY_ADAPTER BIT(9) #define TB_VSE_CAP_APPLE_CABLE_INFO_TBT2_3 BIT(10) +#define TB_VSE_CAP_APPLE_CABLE_INFO_GEN4 BIT(11) + +/* Enter_USB cable speed: USB PD 3.2 adds USB4 Gen4 after Gen3. */ +#define APPLE_EUDO_CABLE_SPEED_USB4_GEN4 4 static unsigned int dpin_mode_value = 8; module_param(dpin_mode_value, uint, 0644); @@ -306,6 +340,10 @@ struct apple_cio { struct apple_tunable *rc_tunable; void __iomem *pcie_adapter_base; struct apple_tunable *pcie_adapter_tunable; + void __iomem *router_ap_fw_base; + struct apple_tunable *router_ap_fw_tunable; + /* A run_store router CPU that apple_cio_start() released */ + bool cpu_running; struct resource *sram_res; void __iomem *sram_base; @@ -1108,6 +1146,10 @@ static int apple_nhi_probe_irqs(struct apple_nhi *anhi) n_irqs = platform_irq_count(anhi->pdev); if (n_irqs < 0) return dev_err_probe(anhi->dev, n_irqs, "platform_irq_count failed\n"); + /* T6050 adds a named SRAM parity interrupt after the ring interrupts. */ + if (n_irqs > 0 && + platform_get_irq_byname_optional(anhi->pdev, "parity") > 0) + n_irqs--; if (n_irqs == 0) { dev_err(anhi->dev, "No interrupts found\n"); return -EINVAL; @@ -2084,9 +2126,13 @@ static int apple_nhi_probe(struct platform_device *pdev) int cap_apple; int ret = 0; + /* The T6050 router is the Type 7 one released by a RUN store. */ if (acio->type7 != (of_device_is_compatible(pdev->dev.of_node, "apple,t8142-usb4-nhi") || - of_device_is_compatible(pdev->dev.of_node, "apple,t6040-usb4-nhi"))) { + of_device_is_compatible(pdev->dev.of_node, "apple,t6040-usb4-nhi") || + of_device_is_compatible(pdev->dev.of_node, "apple,t6050-usb4-nhi")) || + acio->hw->run_store != + !!of_device_is_compatible(pdev->dev.of_node, "apple,t6050-usb4-nhi")) { ret = dev_err_probe(&pdev->dev, -EINVAL, "ACIO/NHI generation mismatch\n"); goto err; } @@ -2515,6 +2561,9 @@ static const struct of_device_id apple_nhi_match[] = { { .compatible = "apple,t6040-usb4-nhi", }, + { + .compatible = "apple,t6050-usb4-nhi", + }, { .compatible = "apple,t8142-usb4-nhi", }, @@ -2648,6 +2697,22 @@ static int apple_cio_lstx_release(struct apple_cio *acio) return pinctrl_select_state(acio->pinctrl, acio->lstx_pins); } +/* + * Halt a run_store router CPU that apple_cio_start() released. Its wrapper is + * only clocked after the CIO reset, so never touch it otherwise. + */ +static void apple_cio_cpu_halt(struct apple_cio *acio) +{ + u32 val; + + if (!acio->hw->run_store || !acio->cpu_running) + return; + acio->cpu_running = false; + val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + writel(val & ~APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); +} + /** * apple_cio_stop - Stop and power down the ACIO complex * @acio: ACIO complex to stop @@ -2672,6 +2737,7 @@ static void apple_cio_stop(struct apple_cio *acio) "Failed to shutdown M3 RTKit, continuing ACIO shutdown anyway\n"); apple_rtkit_free(acio->rtk); acio->rtk = NULL; + apple_cio_cpu_halt(acio); /* High only if the router left it driven high with the cable still in */ if (acio->hw->lstx_after_boot) @@ -2691,9 +2757,15 @@ static void apple_cio_stop(struct apple_cio *acio) acio->resume_link_failed = false; } -/* Only called while the ACIO power domains are on. */ +/* + * Only called while the ACIO power domains are on. Neither a T6050 router's rc + * block nor its CPU wrapper is known to be readable while it starts (see + * apple_cio_start()), so log nothing for it. + */ static void apple_cio_log_state(struct apple_cio *acio, const char *when) { + if (acio->hw->run_store) + return; if (acio->cpu_base) dev_info(acio->dev, "%s: LSTX 0x%08x, FW state 0x%08x, CPU ctrl 0x%08x, status 0x%08x\n", when, readl(acio->rc_base + APPLE_CIO_M3_CTRL), @@ -2809,7 +2881,11 @@ static int apple_cio_start(struct apple_cio *acio) * them with a bare store. */ if (acio->hw->lstx_after_boot) { - /* Written below, once RTKit is up. */ + /* + * Written below, once RTKit is up. A T6050 CPU is started through + * its wrapper below, and its rc block takes an external abort until + * the CPU runs, so M3_CTRL is left alone here. + */ } else if (acio->hw->lstx_explicit) { writel(APPLE_CIO_LSTX_CTRL_ENABLE | APPLE_CIO_LSTX_CTRL_LEVEL, acio->rc_base + APPLE_CIO_M3_CTRL); @@ -2820,9 +2896,21 @@ static int apple_cio_start(struct apple_cio *acio) if (acio->hw->mxwrap && !acio->hw->skip_run) { apple_cio_log_state(acio, "before RUN"); - val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); - writel(val | APPLE_MXWRAP_CPU_CTRL_RUN, - acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + if (acio->hw->run_store) { + /* + * The CIO reset above clocks the wrapper; the CPU runs the + * image the boot loader left in its SRAM once RUN is set. + * The run value is stored outright, not merged. + */ + writel(APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + acio->cpu_running = true; + apple_cio_log_state(acio, "after RUN"); + } else { + val = readl(acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + writel(val | APPLE_MXWRAP_CPU_CTRL_RUN, + acio->cpu_base + APPLE_MXWRAP_CPU_CTRL); + } } acio->rtk = apple_rtkit_init(acio->dev, acio, NULL, 0, &apple_cio_rtkit_ops); @@ -2881,6 +2969,8 @@ static int apple_cio_start(struct apple_cio *acio) apple_tunable_apply(acio->pcie_adapter_base, acio->pcie_adapter_tunable); } + /* The router AP firmware tunable goes with the others once the firmware is ready. */ + apple_tunable_apply(acio->router_ap_fw_base, acio->router_ap_fw_tunable); /* * Bring up devices which are part of ACIO and are now accessible by the main SoC @@ -2918,6 +3008,7 @@ static int apple_cio_start(struct apple_cio *acio) apple_rtkit_free(acio->rtk); remove_links: acio->rtk = NULL; + apple_cio_cpu_halt(acio); /* Cut power to reset the entire block */ for (i = 0; i < acio->pd_list->num_pds; i++) { if (acio->pd_list->pd_links[i]) @@ -3131,7 +3222,16 @@ static int apple_cio_tbt_switch_set(struct typec_thunderbolt_switch_dev *sw, acio->target_cable_info = TB_VSE_CAP_APPLE_CABLE_INFO_PRESENT; if (FIELD_GET(EUDO_CABLE_TYPE_MASK, data->usb4.eudo) != EUDO_CABLE_TYPE_PASSIVE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ACTIVE_CABLE; - if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= EUDO_CABLE_SPEED_USB4_GEN3) + /* + * The T6050 router firmware trains Gen4 only on a Gen4 cable, and + * takes Gen4 alone: no 20 Gb/s bit beside it. + */ + if (acio->hw->gen4_cable && + FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= + APPLE_EUDO_CABLE_SPEED_USB4_GEN4) + acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_GEN4; + else if (FIELD_GET(EUDO_CABLE_SPEED_MASK, data->usb4.eudo) >= + EUDO_CABLE_SPEED_USB4_GEN3) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_20_GBPS; if (data->orientation == TYPEC_ORIENTATION_REVERSE) acio->target_cable_info |= TB_VSE_CAP_APPLE_CABLE_INFO_ORIENTATION_REVERSE; @@ -3335,16 +3435,15 @@ static int apple_cio_probe(struct platform_device *pdev) init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; - acio->type7 = of_device_is_compatible(acio->np, "apple,t8142-usb4-acio") || - of_device_is_compatible(acio->np, "apple,t6040-usb4-acio"); - if (acio->type7 && - of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) - return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); - acio->hw = of_device_get_match_data(dev); if (!acio->hw) return -EINVAL; + acio->type7 = acio->hw->type7; + if (acio->type7 && + of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) + return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); + if (acio->hw->mxwrap) { struct resource *cpu_res; @@ -3367,6 +3466,9 @@ static int apple_cio_probe(struct platform_device *pdev) acio->lstx_pins = NULL; acio->lstx_hold = acio->lstx_pins && of_property_present(dev->of_node, "lstx-gpios"); + /* Hold the line low until the router first takes it. */ + if (acio->hw->lstx_hold_from_probe) + apple_cio_lstx_hold(acio, false); } acio->pcie_tunnel_np = of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); @@ -3435,6 +3537,32 @@ static int apple_cio_probe(struct platform_device *pdev) } } + { + struct resource *ap_fw; + + ap_fw = platform_get_resource_byname(pdev, IORESOURCE_MEM, "router-ap-fw"); + if (ap_fw) { + acio->router_ap_fw_base = devm_ioremap_resource(dev, ap_fw); + if (IS_ERR(acio->router_ap_fw_base)) + return dev_err_probe(dev, PTR_ERR(acio->router_ap_fw_base), + "Unable to map router AP firmware regs\n"); + acio->router_ap_fw_tunable = + devm_apple_tunable_parse(dev, acio->np, + "apple,tunable-router-ap-fw", ap_fw); + if (IS_ERR(acio->router_ap_fw_tunable)) { + if (PTR_ERR(acio->router_ap_fw_tunable) != -ENOENT) + return dev_err_probe(dev, + PTR_ERR(acio->router_ap_fw_tunable), + "Unable to load router AP firmware tunable\n"); + acio->router_ap_fw_tunable = NULL; + } + } + } + + /* + * On T6050 this is the CIO reconfig request, which also clocks the + * router CPU's wrapper: nothing may touch the wrapper before it. + */ acio->reset = devm_reset_control_get_exclusive(dev, NULL); if (IS_ERR(acio->reset)) return dev_err_probe(dev, PTR_ERR(acio->reset), "Unable to get CIO reset\n"); @@ -3548,6 +3676,10 @@ static const struct of_device_id apple_acio_match[] = { .compatible = "apple,t6040-usb4-acio", .data = &apple_cio_t6040_hw, }, + { + .compatible = "apple,t6050-usb4-acio", + .data = &apple_cio_t6050_hw, + }, { .compatible = "apple,t8142-usb4-acio", .data = &apple_cio_t8142_hw, From ab6654bbbc8e6204c7d3a78eebce8fbd9a4267da Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 18:08:06 -0500 Subject: [PATCH 224/392] dt-bindings: iommu: apple,dart: Describe the setup of a DART nothing initialises A T8110 DART that no boot loader sets up, such as a USB4 router's or its PCIe-C tunnel's, needs the device tree to describe what the boot loader otherwise does: its tunables, its bypass streams, every error reported and the context cache flushed. Add apple,full-error-reporting and apple,ctc-flush, and extend apple,bypass-sids beyond the PCIe-C tunnel DARTs. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../devicetree/bindings/iommu/apple,dart.yaml | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..0c91127ba8505 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -151,9 +151,25 @@ properties: items: maximum: 255 description: - Reserved streams of a USB4 PCIe tunnel DART that its PCIe-C port issues + Streams set to bypass after every reset of the DART, outside any domain. + On a USB4 PCIe tunnel DART, the reserved streams its PCIe-C port issues its own writes on, such as the interrupt controller writes its MSIs - turn into. They are set to bypass after every reset of the DART. + turn into. On a T8110 DART that nothing initialises before Linux, such + as a USB4 router's, the streams its co-processor uses without + translation. + + apple,full-error-reporting: + type: boolean + description: + Unmask every error and enable the error interrupt for every stream, + with the registers of the DART's revision, and leave the DART unlocked + on error. For a T8110 DART that nothing initialises before Linux. + + apple,ctc-flush: + type: boolean + description: + Flush the DART's context cache after a stream flush, where the DART + has one. For a T8110 DART that nothing initialises before Linux. apple,remap-streams: $ref: /schemas/types.yaml#/definitions/uint32-matrix From 331043fa765040e3e2b370894660e191ff3d45e4 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:34:56 -0500 Subject: [PATCH 225/392] iommu: apple-dart: set up a T8110 DART that nothing initialised A T8110 DART that the boot loader leaves alone, such as the M5 Max USB4 router's, comes up with its tunables unapplied, its errors masked and no stream in bypass. Its own driver otherwise only clears the streams, enables them and flushes the TLBs. Let the device tree describe the rest, for non-tunneled T8110 DARTs: - apple,tunable: the DART's own tunables, applied at every reset before the first stream is configured. Client partition registers (0x800..0xfff) are skipped unless PARAMS1 bit 29 says they exist. - apple,bypass-sids: streams set to bypass translation at every reset, outside any domain. - apple,full-error-reporting: unmask every error and enable the error interrupt for every stream with the registers of the DART's revision (the error mask moved to 0x4060 in 2.2; status, enable and the per-stream enables moved from 0x700 to 0x4200 in 2.4), and leave the DART unlocked on error. On 2.4 the interrupt handler also reports and acknowledges the error status. - apple,ctc-flush: where PARAMS2 bit 4 says there is a context cache, flush it (0x600) after a V2 stream flush, which leaves it alone. Log the identification registers of a DART set up this way at probe. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/iommu/apple-dart.c | 165 +++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..eb79cad148163 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -190,6 +190,26 @@ #define DART_T8110_ENABLE_STREAMS 0xc00 #define DART_T8110_DISABLE_STREAMS 0xc20 +/* + * Revision 2.2 and later moved the error mask; 2.4 also moved the error + * status, the error interrupt enable and the per-stream error enables. + */ +#define DART_T8110_V2_ERROR_MASK 0x4060 +#define DART_T8110_ERR_IRQ_STATUS 0x704 +#define DART_T8110_ERR_IRQ_ENABLE 0x700 +#define DART_T8110_ERR_SID_ENABLE(i) (0x720 + 4 * (i)) +#define DART_T8110_V24_ERR_IRQ_STATUS 0x4204 +#define DART_T8110_V24_ERR_IRQ_ENABLE 0x4200 +#define DART_T8110_V24_ERR_SID_ENABLE(i) (0x4220 + 4 * (i)) + +/* PARAMS1 bit 29: client partitions (the 0x800.. registers) exist. */ +#define DART_T8110_PARAMS1_CLIENT_PARTITIONS BIT(29) +#define DART_T8110_CLIENT_PARTITION_BASE 0x800 +#define DART_T8110_CLIENT_PARTITION_END 0x1000 +/* PARAMS2 bit 4: a translation context cache that a full flush leaves alone. */ +#define DART_T8110_PARAMS2_CTC BIT(4) +#define DART_T8110_CTC_FLUSH 0x600 + #define DART_T8110_TCR 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_ENABLE BIT(7) @@ -311,6 +331,16 @@ struct apple_dart { struct apple_tunable *tunables; u16 version; + /* + * Set up from the device tree for a DART nothing else initialised: + * tunables applied at every reset, streams kept in static bypass, every + * error reported, and the context cache flushed after a full TLB flush. + */ + DECLARE_BITMAP(static_bypass, DART_MAX_STREAMS); + bool full_errors; + bool ctc_flush; + bool client_partitions; + /* J713 internal display DARTs, borrowed from the running DCP. */ struct apple_dart_dcp *dcp; struct apple_dart_dcp *frontend[2]; @@ -1377,6 +1407,10 @@ apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, if (ret) break; } + /* A V2 flush leaves the context cache alone: flush all of it after. */ + if (!ret && dart->ctc_flush && dart->version >= 0x0202 && + command == DART_T8110_TLB_CMD_OP_FLUSH_SID) + apple_dart_writel(dart, 0, DART_T8110_CTC_FLUSH); spin_unlock_irqrestore(&dart->lock, flags); if (ret) dev_err_ratelimited(dart->dev, @@ -1750,6 +1784,59 @@ apple_dart_t8110_hw_enable_remap(struct apple_dart *dart, int sid, int remap_to) * acknowledge the errors latched so far, unlock a DART that locks on errors, * unmask the error interrupt and flush the TLBs. */ +/* + * Unmask every error and enable its interrupt for each stream, with the + * registers of this revision: the T8110 reset value leaves them masked on + * 2.2 and later, where the error mask is no longer at ERROR_MASK. + */ +static void apple_dart_t8110_enable_errors(struct apple_dart *dart) +{ + bool v24 = dart->version >= 0x0204; + unsigned int i; + + apple_dart_writel(dart, 0, dart->version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : DART_T8110_ERROR_MASK); + apple_dart_readl(dart, dart->version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : DART_T8110_ERROR_MASK); + apple_dart_writel(dart, U32_MAX, v24 ? DART_T8110_V24_ERR_IRQ_STATUS : + DART_T8110_ERR_IRQ_STATUS); + apple_dart_writel(dart, U32_MAX, v24 ? DART_T8110_V24_ERR_IRQ_ENABLE : + DART_T8110_ERR_IRQ_ENABLE); + for (i = 0; i < DIV_ROUND_UP(dart->num_streams, 32); i++) { + u32 sids = dart->num_streams >= 32 * (i + 1) ? U32_MAX : + GENMASK(dart->num_streams - 32 * i - 1, 0); + + apple_dart_writel(dart, sids, v24 ? DART_T8110_V24_ERR_SID_ENABLE(i) : + DART_T8110_ERR_SID_ENABLE(i)); + } + /* No lock on error: the driver acknowledges and carries on. */ + apple_dart_writel(dart, 0, DART_T8110_DIAG_LOCK); +} + +/* + * Apply the DART's own tunables. Client partition registers exist only where + * PARAMS1 says so; a write to one that does not exist is skipped. + */ +static void apple_dart_apply_own_tunables(struct apple_dart *dart) +{ + size_t i; + + for (i = 0; i < dart->tunables->sz; ++i) { + u32 offset = dart->tunables->values[i].offset; + u32 old_value, value; + + if (offset >= DART_T8110_CLIENT_PARTITION_BASE && + offset < DART_T8110_CLIENT_PARTITION_END && + !dart->client_partitions) + continue; + old_value = apple_dart_readl(dart, offset); + value = old_value & ~dart->tunables->values[i].mask; + value |= dart->tunables->values[i].value; + if (value != old_value) + apple_dart_writel(dart, value, offset); + } +} + static int apple_dart_hw_finish_reset(struct apple_dart *dart) { struct apple_dart_stream_map stream_map = { .dart = dart }; @@ -1795,6 +1882,9 @@ static int apple_dart_hw_finish_reset(struct apple_dart *dart) if (ret) dev_err(dart->dev, "failed to flush the TLBs after reset: %d\n", ret); + if (dart->full_errors) + apple_dart_t8110_enable_errors(dart); + return ret; } @@ -2012,10 +2102,18 @@ static int apple_dart_hw_reset(struct apple_dart *dart) bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); + /* Before the first stream is configured. */ + if (!dart->tunneled && dart->tunables) + apple_dart_apply_own_tunables(dart); + apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); apple_dart_hw_apply_remaps(dart); + /* Streams that bypass translation for good, outside any domain. */ + for_each_set_bit(i, dart->static_bypass, dart->num_streams) + apple_dart_writel(dart, DART_T8110_TCR_BYPASS_DART, DART_TCR(dart, i)); + /* enable all streams globally since TCR is used to control isolation */ for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) apple_dart_writel(dart, U32_MAX, @@ -3855,6 +3953,17 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) readl(dart->regs + DART_T8110_EXCEPTION(0)); } + /* With every error enabled, 2.4 latches the rest in its own status. */ + if (dart->full_errors && dart->version >= 0x0204) { + u32 status = readl(dart->regs + DART_T8110_V24_ERR_IRQ_STATUS); + + if (status) { + dev_err_ratelimited(dart->dev, "error status: 0x%08x\n", status); + writel(status, dart->regs + DART_T8110_V24_ERR_IRQ_STATUS); + ret = IRQ_HANDLED; + } + } + if (dart->version >= 0x0203) { block_error = readl(dart->regs + DART_T8110_BLOCK_ERROR); if (apple_dart_t8110_v2_block_pending(dart->version, block_error)) { @@ -4127,6 +4236,56 @@ static bool apple_dart_has_live_rtkit_client(struct device_node *np) return false; } +/* + * The setup a T8110 DART needs when nothing initialised it before Linux: + * apple,tunable (applied at every reset), apple,bypass-sids (streams kept in + * static bypass), apple,full-error-reporting and apple,ctc-flush. Log the + * identification registers of such a DART. + */ +static int apple_dart_parse_own_setup(struct apple_dart *dart, struct resource *res, + const u32 *params) +{ + struct device *dev = dart->dev; + struct device_node *np = dev->of_node; + int count, i; + u32 sid; + + dart->full_errors = of_property_read_bool(np, "apple,full-error-reporting"); + dart->ctc_flush = of_property_read_bool(np, "apple,ctc-flush") && + (params[1] & DART_T8110_PARAMS2_CTC); + dart->client_partitions = params[0] & DART_T8110_PARAMS1_CLIENT_PARTITIONS; + + if (of_property_present(np, "apple,tunable")) { + dart->tunables = devm_apple_tunable_parse(dev, np, "apple,tunable", res); + if (IS_ERR(dart->tunables)) + return dev_err_probe(dev, PTR_ERR(dart->tunables), + "failed to parse DART tunables\n"); + } + + count = of_property_count_u32_elems(np, "apple,bypass-sids"); + for (i = 0; i < count; i++) { + if (of_property_read_u32_index(np, "apple,bypass-sids", i, &sid) || + sid >= dart->num_streams) + return dev_err_probe(dev, -EINVAL, "invalid bypass stream\n"); + __set_bit(sid, dart->static_bypass); + } + + if (dart->tunables || count > 0 || dart->full_errors || + of_property_read_bool(np, "apple,ctc-flush")) + dev_info(dev, + "params %08x %08x %08x %08x, version %04x, %u streams, AS %u -> %u, protect %08x/%08x, diag lock %08x; %zu tunables%s, %d bypass streams, CTC flush %s\n", + params[0], params[1], params[2], params[3], dart->version, + dart->num_streams, dart->ias, dart->oas, + apple_dart_readl(dart, DART_T8110_PROTECT), + apple_dart_readl(dart, DART_T8110_PROTECT_LOCK), + apple_dart_readl(dart, DART_T8110_DIAG_LOCK), + dart->tunables ? dart->tunables->sz : 0, + dart->tunables && !dart->client_partitions ? + " (no client partitions)" : "", + max(count, 0), dart->ctc_flush ? "on" : "off"); + return 0; +} + static int apple_dart_probe(struct platform_device *pdev) { int ret, sid; @@ -4433,6 +4592,12 @@ static int apple_dart_probe(struct platform_device *pdev) } params_done: + if (!tunneled && dart->hw->type == DART_T8110) { + ret = apple_dart_parse_own_setup(dart, res, dart_params); + if (ret) + goto err_clk_disable; + } + dart->dma_min = 0; dart->dma_max = DMA_BIT_MASK(dart->ias); dart->dma_offset = 0; From 1102e101fc521998854f047da156d9089ae909ec Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Tue, 3 Nov 2026 22:35:02 -0500 Subject: [PATCH 226/392] arm64: dts: apple: j716c: Add the right port's USB4/Thunderbolt router Describe the USB4/Thunderbolt host router behind the right-hand USB-C port (ADT /arm-io/acio2, acio-cpu2 and dart-acio2). In t6050-max.dtsi: - the ATC2_CIO, ATC2_CIO_PCIE, ATC2_CIO_USB and ATC2_CIO_AUSS power domains; - the router CPU's mailbox and the router's LSTX pin function on the AOP GPIO controller; - the router itself, with its DART and NHI as children, so that both are populated only once the router runs: nothing is touched at boot; - the DART's stream 15 in bypass, full error reporting and the context cache flush. In the board: AUSS, kept as the boot loader leaves it, the CIO reconfig request (port 2 of the die's CIO reset), the AOP GPIO controller with only the LSTX pin exposed, the PHY's CIO80 RX_TOP tunables, the link from the PD controller to the router, and, until the boot loader fills them, this board's router, NHI and DART tunables and host DROM, from its ADT. The router node is named acio@: the boot loader disables a cio@ node unless its first register is the third register of the matching /arm-io/acioN, which on T6050 is not the router's rc block. Developed with Claude Code. Signed-off-by: Erik Bourget --- arch/arm64/boot/dts/apple/t6050-j716c.dts | 221 ++++++++++++++++++++++ arch/arm64/boot/dts/apple/t6050-max.dtsi | 178 +++++++++++++++++ 2 files changed, 399 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index 550aef6f8768f..abdf7ca85b592 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -342,3 +342,224 @@ }; }; }; + +/* + * ==== The right USB-C port's USB4/Thunderbolt router (ATC2) ==== + * + * Until the boot loader fills them, the router, NHI, PHY CIO and DART tunables and the host + * DROM are this board's, from its ADT (the DROM's router UUID from an earlier boot). + */ +/* ATC2_CIO_AUSS's other parent; kept as the boot loader leaves it */ +&ps_auss { + status = "okay"; +}; + +/* The CIO reconfig request, port 2 (clock gate CIO2_RECONFIG_V) */ +&cio_reconfig0 { + status = "okay"; +}; + +/* The PHY's CIO80 RX_TOP tunables (ADT tunable_LN0/LN1_RX_TOP_CIO_DFLT); the other three are empty */ +&atc2_phy { + apple,tunable-lane0-rx-top-cio = <0x35c 0xe00 0x0>; + apple,tunable-lane1-rx-top-cio = <0x35c 0xe00 0x0>; +}; + +/* Only the router's LSTX pin (81) of the AOP GPIO controller is exposed. */ +&pinctrl_aop { + gpio-reserved-ranges = <0 81>, <82 3>; + status = "okay"; +}; + +&hpm2 { + connector { + ports { + port@3 { + reg = <3>; + hpm2_usb4: endpoint { + remote-endpoint = <&acio2_tbt>; + }; + }; + }; + }; +}; + +&acio2 { + apple,tunable-rc = <0x0 0xfff0 0x4020>, + <0x2c 0x4 0x0>, + <0x4004 0x1ff01ff 0x1000100>, + <0x4008 0x801f 0x4>, + <0x4010 0x7f 0x40>, + <0x4014 0x801f 0x4>, + <0x401c 0x7f 0x40>, + <0x4024 0x801f 0x4>, + <0x402c 0x7f 0x40>, + <0x4030 0x801f 0x4>, + <0x4038 0xffffffff 0x1010101>, + <0x403c 0xffff 0x101>, + <0x4040 0x1 0x1>, + <0x4044 0xffffffff 0x1010101>, + <0x4048 0x1 0x1>, + <0x404c 0x801f 0x4>, + <0x8004 0x70007 0x40004>, + <0x8008 0x801f 0x4>, + <0x8010 0x70007 0x40004>, + <0x8018 0x801f 0x4>, + <0x8020 0x70007 0x40004>, + <0x8028 0x801f 0x4>, + <0x8030 0x801f 0x4>, + <0x8038 0x801f 0x4>, + <0x8040 0x801f 0x4>, + <0x8048 0x801f 0x4>, + <0x8050 0x801f 0x4>, + <0x8058 0x801f 0x4>, + <0x8060 0x801f 0x4>; + apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; + apple,tunable-router-ap-fw = <0x8 0xff2 0x482>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + acio2_tbt: endpoint { + remote-endpoint = <&hpm2_usb4>; + }; + }; + }; +}; + +&acio2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>, + <0x800 0xf003f 0x10000>, + <0x804 0xf003f 0x10002>, + <0x808 0xf003f 0x10004>, + <0x80c 0xf003f 0x20006>, + <0x810 0xf003f 0x2000a>, + <0x814 0xf003f 0x2000e>, + <0x818 0xf003f 0x20012>, + <0x81c 0xf003f 0x20016>, + <0x820 0xf003f 0x2001a>, + <0x824 0xf003f 0x2001e>, + <0x828 0xf003f 0x20022>, + <0x82c 0xf003f 0x20026>, + <0x830 0xf003f 0x2002a>, + <0x834 0xf003f 0x2002e>, + <0x838 0xf003f 0x20032>, + <0x83c 0xf003f 0x10036>; +}; + +&acio2_nhi { + apple,tunable-nhi = <0xf8004 0xff 0x80>, + <0xf8008 0x801f 0x4>, + <0xf8010 0x3f 0x20>, + <0xf8014 0x801f 0x4>, + <0xf801c 0x1f 0x10>, + <0xf8020 0x801f 0x4>, + <0xf8028 0xff 0x80>, + <0xf802c 0x801f 0x4>, + <0xf8034 0xff 0x80>, + <0xf8038 0x801f 0x4>, + <0xf8040 0x3f 0x20>, + <0xf8044 0x801f 0x4>, + <0xf804c 0x8007 0x1>, + <0xfc004 0x3 0x2>, + <0xfc008 0x801f 0x4>, + <0xfc010 0x7 0x4>, + <0xfc014 0x801f 0x4>, + <0xfc01c 0x7 0x4>, + <0xfc020 0x801f 0x4>, + <0xfc028 0x7 0x4>, + <0xfc02c 0x801f 0x4>, + <0xfc034 0x7 0x4>, + <0xfc038 0x801f 0x4>, + <0xfc040 0x3 0x2>, + <0xfc044 0x801f 0x4>, + <0xfc04c 0x8007 0x1>, + <0x4000 0x1000000f 0x10000002>, + <0x4004 0x1000000f 0x10000002>, + <0x4008 0x1000000f 0x10000002>, + <0x400c 0x1000000f 0x10000002>, + <0x4010 0x1000000f 0x10000002>, + <0x4014 0x1000000f 0x10000002>, + <0x4018 0x1000000f 0x10000002>, + <0x401c 0x1000000f 0x10000002>, + <0x4020 0x1000000f 0x10000002>, + <0x4024 0x1000000f 0x10000002>, + <0x4028 0x1000000f 0x10000002>, + <0x402c 0x1000000f 0x10000002>, + <0x4040 0x1000000f 0x10000002>, + <0x4044 0x1000000f 0x10000002>, + <0x4048 0x1000000f 0x10000002>, + <0x404c 0x1000000f 0x10000002>, + <0x4050 0x1000000f 0x10000002>, + <0x4054 0x1000000f 0x10000002>, + <0x4058 0x1000000f 0x10000002>, + <0x405c 0x1000000f 0x10000002>, + <0x4060 0x1000000f 0x10000002>, + <0x4064 0x1000000f 0x10000002>, + <0x4068 0x1000000f 0x10000002>, + <0x406c 0x1000000f 0x10000002>, + <0x4080 0x1000000f 0x10000002>, + <0x4084 0x1000000f 0x10000002>, + <0x4088 0x1000000f 0x10000002>, + <0x408c 0x1000000f 0x10000002>, + <0x4090 0x1000000f 0x10000002>, + <0x4094 0x1000000f 0x10000002>, + <0x4098 0x1000000f 0x10000002>, + <0x409c 0x1000000f 0x10000002>, + <0x40a0 0x1000000f 0x10000002>, + <0x40a4 0x1000000f 0x10000002>, + <0x40a8 0x1000000f 0x10000002>, + <0x40ac 0x1000000f 0x10000002>, + <0x40c0 0x1000000f 0x10000002>, + <0x40c4 0x1000000f 0x10000002>, + <0x40c8 0x1000000f 0x10000002>, + <0x40cc 0x1000000f 0x10000002>, + <0x40d0 0x1000000f 0x10000002>, + <0x40d4 0x1000000f 0x10000002>, + <0x40d8 0x1000000f 0x10000002>, + <0x40dc 0x1000000f 0x10000002>, + <0x40e0 0x1000000f 0x10000002>, + <0x40e4 0x1000000f 0x10000002>, + <0x40e8 0x1000000f 0x10000002>, + <0x40ec 0x1000000f 0x10000002>, + <0x4100 0x1000000f 0x10000002>, + <0x4104 0x1000000f 0x10000002>, + <0x4108 0x1000000f 0x10000002>, + <0x410c 0x1000000f 0x10000002>, + <0x4110 0x1000000f 0x10000002>, + <0x4114 0x1000000f 0x10000002>, + <0x4118 0x1000000f 0x10000002>, + <0x411c 0x1000000f 0x10000002>, + <0x4120 0x1000000f 0x10000002>, + <0x4124 0x1000000f 0x10000002>, + <0x4128 0x1000000f 0x10000002>, + <0x412c 0x1000000f 0x10000002>, + <0x4140 0x1000000f 0x10000002>, + <0x4144 0x1000000f 0x10000002>, + <0x4148 0x1000000f 0x10000002>, + <0x414c 0x1000000f 0x10000002>, + <0x4150 0x1000000f 0x10000002>, + <0x4154 0x1000000f 0x10000002>, + <0x4158 0x1000000f 0x10000002>, + <0x415c 0x1000000f 0x10000002>, + <0x4160 0x1000000f 0x10000002>, + <0x4164 0x1000000f 0x10000002>, + <0x4168 0x1000000f 0x10000002>, + <0x416c 0x1000000f 0x10000002>, + <0x43e4 0x7 0x7>; + apple,thunderbolt-drom = [ff 92 e6 6e ee 18 7c ac 05 90 19 c3 8c 00 5c 00 + 00 00 00 00 00 00 08 81 82 02 82 00 00 00 08 82 + 92 01 82 00 00 00 03 83 20 08 84 00 00 00 00 00 + 00 08 85 00 00 00 00 00 00 05 86 09 01 00 08 87 + 00 00 00 00 00 00 05 88 09 01 00 05 89 09 01 00 + 05 8a 09 01 00 0e 01 41 70 70 6c 65 2c 20 49 6e + 63 2e 00 06 02 69 4f 53 00 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index 55cf22a98e2a2..87bd77e4b4b99 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -129,3 +129,181 @@ }; #endif }; + +#ifdef T6050_J716C +/* + * ATC2's USB4/Thunderbolt router (ADT /arm-io/acio2, acio-cpu2 and dart-acio2), powered per + * connection: the PHY enters CIO mode, then the router's domains come on (ATC2_CIO, _PCIE, _USB, + * _AUSS), the CIO reset (reconfig request) clocks the CPU wrapper, the CPU runs from the image the + * boot loader left in its SRAM, and once its firmware is ready the DART and the NHI are populated. + * Nothing here is touched at boot. The router is named acio@: the boot loader disables a cio@ node + * unless its first register is the third register of the matching /arm-io/acioN, which on T6050 + * is not the router's rc block. The board gives the tunables and the host DROM until the boot + * loader fills them. + */ +&pmgr0 { + ps_atc2_cio: power-controller@2e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + label = "atc2_cio"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_common>; + }; + + ps_atc2_cio_pcie: power-controller@3f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + label = "atc2_cio_pcie"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc2_cio_usb: power-controller@400 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + label = "atc2_cio_usb"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc2_cio_auss: power-controller@408 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + label = "atc2_cio_auss"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_cio>, <&ps_auss>; + }; +}; + +/* LSTX: /arm-io/acio2 gpio-lstx, AOP GPIO pin 81 (0x51), function 1 for the router */ +&pinctrl_aop { + acio2_lstx_pins: acio2-lstx-pins { + pinmux = <0x10051>; /* APPLE_PINMUX(81, 1) */ + }; +}; + +&die0 { + /* /arm-io/acio-cpu2 reg[0]: the router CPU's mailbox, inside its wrapper */ + acio2_mbox: mailbox@381108000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x81108000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * /arm-io/acio2: rc (reg[1]), the RTKit data segment in its SRAM, the PCIe adapter, + * acio-cpu2 reg[1] (the CPU wrapper) and the PCIe router AP firmware block. + */ + acio2: acio@381ac0000 { + compatible = "apple,t6050-usb4-acio"; + reg = <0x3 0x81ac0000 0x0 0xc000>, + <0x3 0x81080000 0x0 0x26000>, + <0x3 0x81e44000 0x0 0x4000>, + <0x3 0x81100000 0x0 0x4000>, + <0x3 0x81c80000 0x0 0x4000>; + reg-names = "rc", "sram", "pcie-adapter", "cpu", "router-ap-fw"; + mboxes = <&acio2_mbox>; + resets = <&cio_reconfig0 2>; + power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, + <&ps_atc2_cio_usb>, <&ps_atc2_cio_auss>; + thunderbolt-switch; + pinctrl-0 = <&acio2_lstx_pins>; + pinctrl-names = "lstx"; + lstx-gpios = <&pinctrl_aop 81 GPIO_ACTIVE_HIGH>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x3 0x81000000 0x1000000>; + nonposted-mmio; + + /* + * /arm-io/dart-acio2, populated with the NHI once the router runs, so that it is + * first read with the router up. Nothing initialises it before Linux: its tunables + * (dart-tunables-instance-0), stream 15 (bypass-15) in bypass, every error + * reported and the context cache flushed. vm-base 0x100_0000_0000. + */ + acio2_dart: iommu@a80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0xa80000 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_cio>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; + apple,bypass-sids = <15>; + apple,full-error-reporting; + apple,ctc-flush; + }; + + /* Interrupts 0x66b.. are the TX rings and 0x677.. the RX rings */ + acio2_nhi: nhi@f00000 { + compatible = "apple,t6050-usb4-nhi"; + reg = <0xf00000 0x100000>; + reg-names = "nhi"; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "txring0", "txring1", "txring2", "txring3", "txring4", "txring5", + "txring6", "txring7", "txring8", "txring9", "txring10", "txring11", + "rxring0", "rxring1", "rxring2", "rxring3", "rxring4", "rxring5", + "rxring6", "rxring7", "rxring8", "rxring9", "rxring10", "rxring11", + "parity"; + iommus = <&acio2_dart 0>, + <&acio2_dart 1>, + <&acio2_dart 2>, + <&acio2_dart 3>, + <&acio2_dart 4>, + <&acio2_dart 5>, + <&acio2_dart 6>, + <&acio2_dart 7>, + <&acio2_dart 8>, + <&acio2_dart 9>, + <&acio2_dart 10>, + <&acio2_dart 11>, + <&acio2_dart 16>, + <&acio2_dart 17>, + <&acio2_dart 18>, + <&acio2_dart 19>, + <&acio2_dart 20>, + <&acio2_dart 21>, + <&acio2_dart 22>, + <&acio2_dart 23>, + <&acio2_dart 24>, + <&acio2_dart 25>, + <&acio2_dart 26>, + <&acio2_dart 27>; + }; + }; +}; +#endif From a0b688d65a3055183ed38a19376d8c060474e9c9 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 08:46:58 -0500 Subject: [PATCH 227/392] phy: apple: t6050: park the PIPE from a dummy producer on any clock At the first controller reset the USB3 PIPE mux is expected unset (0), on the dummy producer (2) or already parked (0x42). When nothing has reset the controller domain since boot, as on a boot without the SPTM, the mux reads 0x22: the dummy producer on another clock. The park then refused it and dwc3 failed to probe. The park clears the clock field before it selects the dummy producer and its clock, so start from any clock while the producer is the dummy one. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h index 812227cac03e9..f5962152444bf 100644 --- a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -23,7 +23,9 @@ apple_t6050_usb3_pipe_cold_dummy(const struct apple_t6050_usb_io *io) if (io->read(io->context, 44, 0x1c) & 1) return -1; mux = io->read(io->context, 44, 0x0c) & 0x73; - if (mux != 0 && mux != 2 && mux != 0x42) + /* Unset, or already on the dummy producer (1:0 = 2) with any clock: when nothing + * has reset the controller domain since boot, the mux can read 0x22. */ + if (mux != 0 && (mux & 3) != 2) return -1; if (mux == 0x42) return 0; From 178e80cc2f0c09f601919110709d604e6f744268 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 10:49:52 -0500 Subject: [PATCH 228/392] thunderbolt: Ignore DROM entries for ports the router does not implement The host DROM of the M5 Max USB4 router lists port 8, which the router reports as not implemented. Reading its port config fails with -ENODEV, which made the whole DROM unusable and left the host router without one, so the NHI failed to probe. Skip such entries, as is already done for entries past the last port. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/thunderbolt/eeprom.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c index 389de7df84f65..f2d56acb21a10 100644 --- a/drivers/thunderbolt/eeprom.c +++ b/drivers/thunderbolt/eeprom.c @@ -381,6 +381,12 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw, return 0; res = tb_port_read(port, &type, TB_CFG_PORT, 2, 1); + /* Listed, but not implemented by this router: skip it as well */ + if (res == -ENODEV) { + tb_sw_dbg(sw, "DROM entry for unimplemented port %u ignored\n", + header->index); + return 0; + } if (res) return res; type &= 0xffffff; From a1d6ef9718682a2a321798783e636c724a0cf913 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 12:17:36 -0500 Subject: [PATCH 229/392] phy: apple: t6050: restart the PHY firmware on its original data The PHY firmware is preloaded by the boot loader, data segment included, and keeps its state in that segment. Restarted after a stop, it ran on what its last run had left there and never finished calibrating the lanes: UC_TNBL_DONE did not come, and every second connection of a port failed, in USB as in CIO mode, until reboot. Save the data SRAM when the firmware is first started and put it back before every later start. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-dp.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c index e6941cf86c8ad..57d0587cdaf24 100644 --- a/drivers/phy/apple/atc-t6050-dp.c +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -97,6 +97,7 @@ struct t6050_dp { struct apple_rtkit *rtkit; void __iomem *fw_data; bool fw_running; + void *fw_data_saved; /* the firmware data SRAM as the first start found it */ }; static int atc_domain_set(struct atc_domain *pd, bool active) @@ -183,6 +184,20 @@ static int atc_firmware(struct t6050_dp *dp, bool start) if (dp->fw_running == start) return 0; if (start) { + /* + * The firmware is the boot loader's, preloaded with its data segment, and keeps + * its state there: restarted on the state its last run left, it never finishes + * calibrating the lanes (UC_TNBL_DONE). Give every start the data segment the + * first one found. The PHY's domains are on here, so the SRAM is accessible. + */ + if (!dp->fw_data_saved) { + dp->fw_data_saved = devm_kmalloc(dp->dev, 0x10000, GFP_KERNEL); + if (!dp->fw_data_saved) + return -ENOMEM; + memcpy_fromio(dp->fw_data_saved, dp->fw_data, 0x10000); + } else { + memcpy_toio(dp->fw_data, dp->fw_data_saved, 0x10000); + } /* Native coprocessor run, after its +0x40 firmware enable. */ writel(BIT(4), dp->banks[38] + 0x28); ret = apple_rtkit_wake(dp->rtkit); From 625229d3881c6a2319a524a1029ef5b3c884bcab Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 12:21:55 -0500 Subject: [PATCH 230/392] phy: apple: t6050: carry USB3 over a USB4 router's tunnel In CIO mode the lanes carry USB4/TBT, and USB3 reaches DWC3 through the router's USB3 adapter. DWC3 was left on its dummy PIPE producer, so a USB3 tunnel came up with nothing on the host side. When DWC3 asks for USB3 in CIO mode, move its PIPE mux to the tunnel producer (0x21: PIPE mode 1, clock 2) under the PIPE lock, as the older Apple PHYs do in USB4 mode. A lock that is not acknowledged is not fatal, as for native USB3. On power-off the PIPE goes back to the dummy producer while the CIO PHY is left to the router. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/phy/apple/atc-t6050-combo.h | 38 +++++++++++++++-- drivers/phy/apple/atc-t6050-dp.c | 1 + .../apple/usb-t6050/apple_t6050_usb3_pipe.h | 42 ++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/drivers/phy/apple/atc-t6050-combo.h b/drivers/phy/apple/atc-t6050-combo.h index c57779075ec05..185f71cd6e138 100644 --- a/drivers/phy/apple/atc-t6050-combo.h +++ b/drivers/phy/apple/atc-t6050-combo.h @@ -298,6 +298,34 @@ static int combo_usb2_reset(struct phy *phy) return ret ? -EIO : 0; } +/* A tunnelled PIPE leaves the CIO PHY to the router: nothing to quiesce on the PHY side. */ +static int combo_pipe_keep(void *ctx) +{ + return 0; +} + +/* + * In CIO mode the lanes carry USB4/TBT and USB3 reaches DWC3 through the router's USB3 adapter: + * move DWC3 from its dummy PIPE producer to the tunnel producer, as is done on the older PHYs. + */ +static int combo_pipe_tunnel(struct t6050_dp *dp) +{ + int ret; + + lockdep_assert_held(&dp->lock); + if (dp->pipe_active) + return dp->pipe_tunnel ? 0 : -EBUSY; + ret = apple_t6050_usb3_pipe_tunnel(&dp->usb_io); + if (ret) { + combo_pipe_fault(dp, "USB3 tunnel PIPE switch", ret); + return -EIO; + } + dp->pipe_active = true; + dp->pipe_tunnel = true; + dev_info(dp->dev, "USB3 PIPE on the USB4 tunnel\n"); + return 0; +} + static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) { struct t6050_dp *dp = phy_get_drvdata(phy); @@ -314,10 +342,8 @@ static int combo_usb3_mode(struct phy *phy, enum phy_mode mode, int submode) * uses its independent D+/D- PHY. Do not switch USB3 onto DP lanes. */ if (dp->combo_mode == COMBO_DP4 && dp->enabled) return 0; - /* In CIO the lanes carry USB4/TBT. DWC3 keeps its dummy PIPE until a - * USB3 tunnel hands PIPE to the router; native USB3 mode is 0 too. */ if (dp->combo_mode == COMBO_CIO && dp->cio_prepared) - return 0; + return combo_pipe_tunnel(dp); if (!dp->usb_prepared && !(dp->combo_mode == COMBO_DP2 && dp->enabled)) return -ENOLINK; if (dp->pipe_active) @@ -347,7 +373,10 @@ static int combo_usb3_poweroff(struct phy *phy) /* DWC3 calls power_off only after host/gadget shutdown and SUSPHY. * PIPE must select its dummy producer before the controller is reset. */ if (dp->pipe_active) { - ret = apple_t6050_usb3_pipe_stop(&dp->usb_io, combo_quiesce, dp); + ret = apple_t6050_usb3_pipe_stop(&dp->usb_io, + dp->pipe_tunnel ? combo_pipe_keep : combo_quiesce, dp); + if (!ret) + dp->pipe_tunnel = false; if (!ret) dp->pipe_active = false; } @@ -491,3 +520,4 @@ static int combo_setup(struct platform_device *pdev, struct t6050_dp *dp) } return 0; } + diff --git a/drivers/phy/apple/atc-t6050-dp.c b/drivers/phy/apple/atc-t6050-dp.c index 57d0587cdaf24..3eadc358589ff 100644 --- a/drivers/phy/apple/atc-t6050-dp.c +++ b/drivers/phy/apple/atc-t6050-dp.c @@ -73,6 +73,7 @@ struct atc_domain { enum combo_mode { COMBO_SAFE, COMBO_USB, COMBO_DP4, COMBO_DP2, COMBO_CIO }; struct t6050_dp { bool combo, usb_prepared, usb2_active, pipe_active, controller_active, cio_prepared; + bool pipe_tunnel; /* the active PIPE is a USB4 router's USB3 tunnel */ enum combo_mode combo_mode; enum typec_orientation orientation; struct phy *combo_phy[3]; diff --git a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h index f5962152444bf..30e462e416580 100644 --- a/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h +++ b/drivers/phy/apple/usb-t6050/apple_t6050_usb3_pipe.h @@ -96,6 +96,46 @@ apple_t6050_usb3_pipe_switch(const struct apple_t6050_usb_io *io) return result; } +/* + * Move a running DWC3 from the dummy producer (0x42) to a USB4 router's USB3 tunnel (0x21: PIPE mode 1, + * clock 2), once the tunnel is active. As for native USB3, the PIPE is locked only when DWC3 is out of + * reset, and a lock that is not acknowledged within 6 ms is not fatal. + * Returns 0, -2 if the PIPE is on another producer, or -7 if the mux did not read back the tunnel. + */ +static inline int +apple_t6050_usb3_pipe_tunnel(const struct apple_t6050_usb_io *io) +{ + unsigned int mux; + int locked; + + if (!io || !io->read || !io->write || !io->poll || !io->delay) + return -1; + mux = io->read(io->context, 44, 0x0c) & 0x73; + if (mux == 0x21) + return 0; + if (mux != 0x42) + return -2; + locked = io->read(io->context, 44, 0x1c) & 1; + + apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); + apple_t6050_usb_mask(io, 44, 0x00, 1, 1); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 1); + io->poll(io->context, 44, 0x14, 1, 1, 6000); + } + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0); + io->delay(io->context, 5); + apple_t6050_usb_mask(io, 44, 0x0c, 3, 1); + apple_t6050_usb_mask(io, 44, 0x0c, 0x70, 0x20); + apple_t6050_usb_mask(io, 44, 0x00, 1, 0); + apple_t6050_usb_mask(io, 44, 0x00, 4, 0); + if (locked) { + apple_t6050_usb_mask(io, 44, 0x10, 1, 0); + io->poll(io->context, 44, 0x14, 1, 0, 6000); + } + return (io->read(io->context, 44, 0x0c) & 0x73) == 0x21 ? 0 : -7; +} + /* The native close() call quiesces the ACIO PHY while PIPE is locked, * before selecting the dummy producer. The caller supplies that lifecycle * operation; merely changing MUX is not a complete disconnect sequence. @@ -110,7 +150,7 @@ apple_t6050_usb3_pipe_stop(const struct apple_t6050_usb_io *io, if (!io || !io->read || !io->write || !io->poll || !io->delay || !quiesce) return -1; mux = io->read(io->context, 44, 0x0c) & 0x73; - if (mux != 0x10 && mux != 0x42) + if (mux != 0x10 && mux != 0x21 && mux != 0x42) return -1; apple_t6050_usb_mask(io, 44, 0x04, 0x0e, 0); apple_t6050_usb_mask(io, 44, 0x00, 1, 1); From 37f3967b602a092e7016cdc0e293a5b1b62f19ac Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 18:08:07 -0500 Subject: [PATCH 231/392] dt-bindings: pci: apple,pcie: Add the T6050 PCIe-C tunnel The T6050 PCIe-C root port has the T600x register layout and is always initialized by the kernel. Developed with Claude Code. Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/pci/apple,pcie.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 64f0a2d43e9aa..36cf94442e064 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -75,6 +75,7 @@ properties: - const: apple,t8112-pciec - const: apple,t8103-pciec - const: apple,t6000-pciec + - const: apple,t6050-pciec - items: - const: apple,t8132-pciec - const: apple,t8122-pciec @@ -318,6 +319,7 @@ allOf: - apple,t8103-pciec - apple,t6000-pciec - apple,t6040-pciec + - apple,t6050-pciec then: properties: reg: From b1ffa4722f3d2a10872e1b7f1501a58af0de0e0d Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 12:54:13 -0500 Subject: [PATCH 232/392] PCI: apple: support the T6050 PCIe-C tunnel T6050 PCIe-C is the T8103 tunnel root port with a larger RID2SID table, a different MSI layout and no reset sequence of its own: - RID2SID has 256 entries, which run over the T8103 tunnel block: the posted request counters, TUNCTRL and TUNSTAT move up by 0x700, to 0x1080..0x108c. The release, assert and acknowledge paths use them. - The MSI doorbell takes its high word at 0x16c. MSIs go through the T602x map, indexed by the vector the endpoint sends: the bridge's slice of the window it shares with the internal host, vector n at AIC 1824 + n. Endpoints get base + i as their MSI data. - Nothing hands the port over, so it is always cold-initialized, as the T600x/T602x ports are with pcie_apple.tunnel_kernel_init: debug, fabric and port tunables, PERST and APPCLK, RUN, then the root port's tunables, before the router populates the DART. There is no port reset table to replay and no PMGR reset; only RID2SID is cleared. - The root port's ACK latency limit (0x700) and flow control update timer (0x73c) depend on the max payload size. Set them after every link-up and whenever the root port's DevCtl is written. The parity error interrupt is left alone. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 118 +++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a28781a22ed0d..c19f271e40d2b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -200,6 +200,21 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define RC_T6050_MISC_CONTROL_1 0x8bc #define RC_T6050_DBI_RO_WR_EN BIT(0) +/* + * T6050 PCIe-C: the 256-entry RID2SID table runs over the T8103 tunnel block, + * which moves up by 0x700 (posted request counters, TUNCTRL and TUNSTAT). + */ +#define PORT_T6050_TUNCTRL 0x1088 + +/* T6050 PCIe-C root port config space: the MPS-dependent link timers */ +#define PCIEC_RP_DEVCAP 0x074 +#define PCIEC_RP_DEVCTL 0x078 +#define PCIEC_RP_ACK_LATENCY 0x700 +#define PCIEC_RP_FC_TIMER 0x73c +#define PCIEC_RP_FC_TIMER_KEEP 0x6000ffff +#define PCIEC_RP_FC_TIMER_LIMIT GENMASK(28, 16) +#define PCIEC_RP_FC_TIMER_EN BIT(31) + /* * The doorbell address must be in the bottom 4GB because the base register * is only 32 bits wide. Exclude it from the IOVA range and share the same @@ -239,6 +254,8 @@ struct hw_info { * lane PIPE reset status, then release PORT_T8152_PIPE_RST. */ bool pipe_rst; + bool no_port_reset; + bool mps_timers; }; static const struct hw_info t8103_hw = { @@ -384,6 +401,32 @@ static const struct hw_info t6040_pciec_hw = { .s2idle_hold_domain = true, }; +/* + * T6050 PCIe-C keeps the t8103 doorbell, PERST and RID2SID registers, with + * the doorbell's high word at 0x16c and 256 RID2SID entries. MSIs go through + * the t602x map, indexed by the vector the endpoint sends: the bridge's slice + * of the shared window, not a local index. The port has no reset sequence of + * its own, and the root port's ACK latency and flow control timers follow + * the max payload size. + * + * Nothing hands the port over: apple,pciec-kernel-init brings it up with the + * T600x Intr2AXI sequence. + */ +static const struct hw_info t6050_pciec_hw = { + .port_msiaddr = PORT_MSIADDR, + .port_msiaddr_hi = PORT_MSIADDR + 4, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .port_tunctrl = PORT_T6050_TUNCTRL, + .max_rid2sid = 256, + .tunneled = true, + .tunnel_intr2axi = true, + .no_port_reset = true, + .msimap_by_vector = true, + .mps_timers = true, +}; + static const struct of_device_id apple_pcie_of_match[]; /** @@ -1076,6 +1119,10 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) { int i; + /* T6050 has no such sequence: only its RID2SID table is cleared. */ + if (port->pcie->hw->no_port_reset) + goto clear_rid2sid; + for (i = 0; i < ARRAY_SIZE(apple_pcie_tunnel_reset_regs); i++) { u32 value = apple_pcie_tunnel_reset_regs[i].value; @@ -1103,6 +1150,7 @@ static void apple_pcie_tunnel_reset_hardware(struct apple_pcie_port *port) apple_pcie_port_writel(port, value, apple_pcie_tunnel_reset_regs[i].offset); } +clear_rid2sid: for (i = 0; i < port->pcie->hw->max_rid2sid; i++) apple_pcie_rid2sid_write(port, i, 0); if (port->pcie->hw->t8132_reset) @@ -1175,8 +1223,8 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) /* * Bring up a port selected by apple,pciec-kernel-init that no firmware - * handoff left running: tunables and the reset table, then PERST and APPCLK, - * and only then the root-port config-space + * handoff left running: tunables and the reset table (T6050 has none), + * then PERST and APPCLK, and only then the root-port config-space * tunables. Stop before releasing the tunnel reset or starting link * training. */ @@ -1301,10 +1349,48 @@ static void apple_pcie_tunnel_restore_irq_hw(struct apple_pcie_port *port) */ #define APPLE_PCIE_TUNNEL_READY_MS 1000 +/* + * The T6050 PCIe-C root port's ACK latency limit (0x700, low half) and flow + * control update timer (0x73c, bits 28:16) are sized for the max payload + * size. Each word holds both, for MPS 128, 256 and 512. + */ +static const u32 apple_pcie_mps_timers[] = { + 0x003c003c, 0x005a005a, 0x006b006b, +}; + +static void apple_pcie_update_mps_timers(struct apple_pcie *pcie, u32 mps) +{ + void __iomem *rp = pcie->cfg->win; + u32 cap, word, val; + + cap = apple_pcie_tunnel_readl(rp, PCIEC_RP_DEVCAP) & PCI_EXP_DEVCAP_PAYLOAD; + if (mps > cap || mps >= ARRAY_SIZE(apple_pcie_mps_timers)) { + dev_warn(pcie->dev, "no link timers for a %u-byte MPS\n", 128 << mps); + return; + } + + word = apple_pcie_mps_timers[mps]; + writew_relaxed(lower_16_bits(word), rp + PCIEC_RP_ACK_LATENCY); + /* Complete the write, as apple_pcie_tunnel_writel() does. */ + mb(); + isb(); + val = apple_pcie_tunnel_readl(rp, PCIEC_RP_FC_TIMER); + val &= PCIEC_RP_FC_TIMER_KEEP; + val |= FIELD_PREP(PCIEC_RP_FC_TIMER_LIMIT, upper_16_bits(word)) | + PCIEC_RP_FC_TIMER_EN; + apple_pcie_tunnel_writel(rp, val, PCIEC_RP_FC_TIMER); +} + +/* Process context: the link of a tunnel came up. */ static void apple_pcie_tunnel_link_up(struct apple_pcie *pcie) { WRITE_ONCE(pcie->tunnel_ready_at, jiffies + msecs_to_jiffies(APPLE_PCIE_TUNNEL_READY_MS)); + if (pcie->hw->mps_timers) + apple_pcie_update_mps_timers(pcie, + FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, + apple_pcie_tunnel_readl(pcie->cfg->win, + PCIEC_RP_DEVCTL))); } /* Call before scanning the bus behind a tunnel; elapsed time counts. */ @@ -2778,6 +2864,23 @@ static int apple_msi_init(struct apple_pcie *pcie) goto invalid_slice; } + /* + * T6050's PCIe hosts share one MSI window, vector n at AIC 1824 + n. + * The internal host's bridges own vectors 0..127 and each PCIe-C + * bridge a 32-vector slice of the rest. + */ + if (pcie->hw == &t6050_pciec_hw) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + if (ret || pcie->msi_vector_base < 128 || pcie->msi_vector_base % 32 || + pcie->nvecs != 32 || + pcie->msi_vector_base + pcie->nvecs > FIELD_MAX(PORT_MSIMAP_TARGET) + 1 || + args.args_count != 4 || args.args[0] != 0 || args.args[1] != 0 || + args.args[2] != 1824 + pcie->msi_vector_base || + args.args[3] != IRQ_TYPE_EDGE_RISING || DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + /* * The controller-wide ADT count (512) is not a per-host allocation. * Internal PCIe owns vectors 0..31. Each PCIe-C bridge owns a distinct @@ -3294,11 +3397,17 @@ static int apple_pcie_config_write(struct pci_bus *bus, unsigned int devfn, int ret; ret = pci_generic_config_write(bus, devfn, where, size, val); - if (ret || !pcie || !pcie->kernel_init) + if (ret || !pcie) return ret; - /* Root-port prefetchable window. The port block has to be told too. */ if (bus->number != cfg->busr.start || devfn) return ret; + /* The root port's link timers follow its max payload size. */ + if (pcie->hw->mps_timers && where == PCIEC_RP_DEVCTL) + apple_pcie_update_mps_timers(pcie, + FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, val)); + if (!pcie->kernel_init) + return ret; + /* Root-port prefetchable window. The port block has to be told too. */ if (where < 0x24 || where >= 0x30) return ret; @@ -4830,6 +4939,7 @@ static const struct dev_pm_ops apple_pcie_pm_ops = { static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t6040-pciec", .data = &t6040_pciec_hw }, + { .compatible = "apple,t6050-pciec", .data = &t6050_pciec_hw }, { .compatible = "apple,t8132-pciec", .data = &t8132_pciec_hw }, { .compatible = "apple,t8122-pciec", .data = &t8122_pciec_hw }, { .compatible = "apple,t8112-pciec", .data = &t8112_pciec_hw }, From 79432bf9312e383e5f9e7688c26a5b6f93bd0b9d Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 12:54:13 -0500 Subject: [PATCH 233/392] thunderbolt: apple: recognize the T6050 PCIe-C host A router finds its PCIe-C host among the children of its apple,pcie-tunnel node by compatible: to tell whether the port gets initialized, to map its Intr2AXI bridge, to find its reset and to find the bound host once a tunnel is up. Accept apple,t6050-pciec there too. The T6050 port has no reset of its own, and the kernel cold-initializes it, which apple_pcie_tunnel_needs_cold_init() reports. Other SoCs see no change. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/thunderbolt/apple.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 0b0c25d5adb69..9cfd12a33dc06 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -935,7 +935,8 @@ static bool apple_cio_pciec_eligible(const struct device_node *np) { return of_device_is_compatible(np, "apple,t6040-pciec") || of_device_is_compatible(np, "apple,t8103-pciec") || - of_device_is_compatible(np, "apple,t6000-pciec"); + of_device_is_compatible(np, "apple,t6000-pciec") || + of_device_is_compatible(np, "apple,t6050-pciec"); } static bool From ec5485290aa55c484af18f623f2e30d98bb2c442 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 12:54:14 -0500 Subject: [PATCH 234/392] arm64: dts: apple: j716c: Add the right port's PCIe tunnel Describe the PCIe-C root port behind the right port's router and its DART, in a node the router points at with apple,pcie-tunnel and populates once it has enabled a PCIe tunnel, and the ATC2_PCIE power state they sit in, in t6050-max.dtsi. The router powers ATC2_PCIE with its other domains, so the port can be cold-initialized before its children are populated. Nothing initialises the DART, so like the router's it carries full error reporting and its two bypass streams, 253 and 255, on which the port issues its own writes, MSIs included; the board gives its tunables and the port's. Its container is not named usb4-pcie-tunnel-N, which would give it the fixed 64-stream topology of the older tunnel DARTs instead of the 256 streams it reports. The bridge's MSIs are vectors 192-223 of the window shared with the internal host. Developed with Claude Code. Signed-off-by: Erik Bourget --- arch/arm64/boot/dts/apple/t6050-j716c.dts | 58 ++++++++++ arch/arm64/boot/dts/apple/t6050-max.dtsi | 103 +++++++++++++++++- .../boot/dts/apple/t6050-peripherals.dtsi | 2 + 3 files changed, 162 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index abdf7ca85b592..55af2f818a954 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -563,3 +563,61 @@ 05 8a 09 01 00 0e 01 41 70 70 6c 65 2c 20 49 6e 63 2e 00 06 02 69 4f 53 00 00 00 00]; }; + +/* ==== The right USB-C port's PCIe tunnel (ATC2): this board's tunables ==== */ +&apciec2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>; +}; + +&apciec2 { + apple,tunable-debug = <0x1000 0x1 0x0>; + apple,tunable-rc = <0x78 0x7000 0x0>, + <0x718 0x7c000 0x0>, + <0x814 0xffffffff 0x2020f060>, + <0x8bc 0x1 0x0>; + apple,tunable-fabric = <0x4 0x3f003f 0x240024>, + <0x8 0x1 0x1>, + <0xc 0x1 0x1>, + <0x10 0x801f 0x4>, + <0x18 0x1ff 0x1e0>, + <0x1c 0x1 0x1>, + <0x20 0x801f 0x4>, + <0x28 0x1f 0x10>, + <0x2c 0x801f 0x4>, + <0x34 0x3f 0x20>, + <0x38 0x801f 0x4>, + <0x40 0x3f 0x20>, + <0x44 0x801f 0x4>, + <0x4c 0x10001 0x10001>, + <0x50 0x1 0x1>, + <0x54 0x1 0x1>, + <0x58 0x801f 0x4>, + <0x60 0x7f007f 0x450045>, + <0x64 0x801f 0x4>, + <0x6c 0x1f001f 0x100010>, + <0x70 0xf1 0x1>, + <0x74 0x801f 0x4>, + <0x7c 0x801f 0x4>, + <0x84 0x1 0x1>, + <0x88 0x1 0x1>, + <0x8c 0x801f 0x4>, + <0x94 0x1 0x1>, + <0x98 0x801f 0x4>, + <0xa0 0x1 0x1>, + <0xa4 0x801f 0x4>, + <0xac 0x1 0x1>, + <0xb0 0x1 0x1>, + <0xb4 0x801f 0x4>, + <0xbc 0x801f 0x4>, + <0xc4 0x801f 0x4>; +}; + +&apciec2_port { + apple,tunable = <0x140 0x1 0x1>, + <0x2a0 0x7e 0xa>; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index 87bd77e4b4b99..c04b66519953c 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -214,9 +214,11 @@ reg-names = "rc", "sram", "pcie-adapter", "cpu", "router-ap-fw"; mboxes = <&acio2_mbox>; resets = <&cio_reconfig0 2>; + /* ATC2_PCIE last: the PCIe tunnel root port, set up before it is populated */ power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, - <&ps_atc2_cio_usb>, <&ps_atc2_cio_auss>; + <&ps_atc2_cio_usb>, <&ps_atc2_cio_auss>, <&ps_atc2_pcie>; thunderbolt-switch; + apple,pcie-tunnel = <&acio2_pcie_tunnel>; pinctrl-0 = <&acio2_lstx_pins>; pinctrl-names = "lstx"; lstx-gpios = <&pinctrl_aop 81 GPIO_ACTIVE_HIGH>; @@ -306,4 +308,103 @@ }; }; }; + +/* + * ATC2's PCIe tunnel: /arm-io/apciec2 and dart-apciec2, the PCIe-C root port behind the router's + * PCIe down adapter, and its DART. The router populates both once it has enabled a PCIe tunnel, + * after the kernel has cold-initialized the port: no boot loader sets it up, and it has no reset + * line of its own. Both sit in ATC2_PCIE, which the router powers with its own domains and the + * host keeps on once it is bound. + * + * The container is not a usb4-pcie-tunnel-N node, whose DART would get the fixed 64-stream + * topology of the earlier tunnel DARTs: this one is read like any T8110 DART. Nothing + * initialises it before Linux, so like the router's it carries every error reported and its + * bypass streams, 253 and 255, which carry the port's own writes (MSIs); the board gives its + * tunables, and the port's. + * + * MSIs use the bridge's 32-vector slice (192-223) of the window shared with the internal host. + */ +&pmgr0 { + /* With the ATC2 USB always-on clock gate, which has no state register of its own */ + ps_atc2_pcie: power-controller@2f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + label = "atc2_pcie"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc2_common>, <&ps_atc2_usb_aon>; + }; +}; + +/ { + acio2_pcie_tunnel: pcie-tunnel-2 { + #address-cells = <2>; + #size-cells = <2>; + ranges; + nonposted-mmio; + + apciec2_dart: iommu@384000000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x84000000 0x0 0x20000>; + interrupts = ; + power-domains = <&ps_atc2_pcie>; + #iommu-cells = <1>; + apple,bypass-sids = <253 255>; + apple,full-error-reporting; + apple,ctc-flush; + }; + + apciec2: pcie@1c70000000 { + compatible = "apple,t6050-pciec"; + device_type = "pci"; + reg = <0x1c 0x70000000 0x0 0x10000000>, + <0x3 0x84034000 0x0 0x4000>, + <0x3 0x84020000 0x0 0x8000>, + <0x3 0x84038000 0x0 0x4000>, + <0x3 0x84060000 0x0 0x4000>, + <0x3 0x8402c000 0x0 0x4000>, + <0x3 0x84028000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "fabric", "debug", "ltssm", "intr2axi"; + /* The port's: link events and INTx */ + interrupts = ; + interrupt-names = "port0"; + power-domains = <&ps_atc2_pcie>; + /* No boot loader hands the port over: the kernel cold-initializes it. */ + apple,pciec-kernel-init; + + msi-controller; + msi-parent = <&apciec2>; + msi-ranges = <&aic AIC_IRQ 0 2016 IRQ_TYPE_EDGE_RISING 32>; + apple,msi-vector-base = <192>; + + iommu-map = <0x0 &apciec2_dart 0x0 0x40>; + iommu-map-mask = <0x3f>; + linux,pci-domain = <3>; + bus-range = <0x0 0x7f>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x10 0x0 0x10 0x0 0x2 0x0>, + <0x02000000 0x0 0x00100000 0x12 0x00100000 0x0 0x3ff00000>, + <0x42000000 0x0 0x40000000 0x12 0x40000000 0x0 0x40000000>; + + apciec2_port: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + external-facing; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &apciec2_port 0 0 0 0>, + <0 0 0 2 &apciec2_port 0 0 0 1>, + <0 0 0 3 &apciec2_port 0 0 0 2>, + <0 0 0 4 &apciec2_port 0 0 0 3>; + }; + }; + }; +}; #endif diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index ae450252f2acb..05526b53d8bfb 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -1469,6 +1469,7 @@ status = "disabled"; }; +#ifndef T6050_J716C apciec2_dart: iommu@384000000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x3 0x84000000 0x0 0x20000>; @@ -1476,6 +1477,7 @@ #iommu-cells = <1>; status = "disabled"; }; +#endif apciec2_piodma: dma-controller@384068000 { compatible = "apple,t6050-pciec-piodma", "apple,pciec-piodma"; From 5a99d43ab4c88986cb30bec2946333c6e023f5c6 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 19:49:04 -0500 Subject: [PATCH 235/392] mux: apple-display-crossbar: Accept busy T6050 connection status T6050 can report a busy crossbar connection after selecting a valid output. Log that status while retaining the selected route, so an active USB4 DisplayPort tunnel can complete its display setup. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/mux/apple-display-crossbar.c | 86 ++++++++++++++++------------ 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 223d7b813dc15..f45d296eebef4 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -108,6 +108,8 @@ struct apple_dpxbar_hw { bool no_tunable; /* power domain held only while an output is selected */ bool runtime_pm; + /* T8142 layout: a connection status still busy after its poll is only logged */ + bool status_log_only; }; struct apple_dpxbar { @@ -901,9 +903,33 @@ static const u8 t8142_dest_shift[T8142_MUX_MAX] = { 8, 0, 4, 12, 16 }; static const u8 t8142_select_shift[T8142_MUX_MAX] = { 8, 0, 4, 24, 0 }; static const u8 t8142_select_shift2[T8142_MUX_MAX] = { 20, 12, 16, 28, 4 }; +/* + * Wait up to 1 ms for @mask to clear in status word @reg. Where the hardware + * only reports these bits (T6050), one still set is logged and the change + * goes on; elsewhere the crossbar is taken as faulted. + */ +static int t8142_settle(struct apple_dpxbar *xbar, unsigned int reg, u32 mask, + unsigned int index, const char *what) +{ + u32 value; + int ret; + + ret = readl_poll_timeout_atomic(xbar->regs + reg, value, !(value & mask), 1, 1000); + if (!ret) + return 0; + if (xbar->hw->status_log_only) { + dev_warn_ratelimited(xbar->dev, + "output %u %s: status +%#x = %#x, %#x still set; going on\n", + index, what, reg, value, mask); + return 0; + } + xbar->faulted = true; + return ret; +} + static int t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) { - u32 src, dst, clock, value; + u32 src, dst, clock; int ret; if (!xbar->enabled[index]) @@ -917,24 +943,13 @@ static int t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) dpxbar_clear32(xbar, 0x0c + 4 * clock, src); dpxbar_clear32(xbar, 0x24 + 4 * clock, dst); udelay(1); - ret = readl_poll_timeout_atomic(xbar->regs + 0x800, value, - !(value & src), 1, 1000); - if (ret) { - xbar->faulted = true; - return ret; - } - ret = readl_poll_timeout_atomic(xbar->regs + 0x808 + 4 * clock, value, - !(value & src), 1, 1000); - if (ret) { - xbar->faulted = true; - return ret; - } - ret = readl_poll_timeout_atomic(xbar->regs + 0x81c + 4 * clock, value, - !(value & dst), 1, 1000); - if (ret) { - xbar->faulted = true; + ret = t8142_settle(xbar, 0x800, src, index, "link down"); + if (!ret) + ret = t8142_settle(xbar, 0x808 + 4 * clock, src, index, "link down"); + if (!ret) + ret = t8142_settle(xbar, 0x81c + 4 * clock, dst, index, "link down"); + if (ret) return ret; - } dpxbar_clear32(xbar, 0x08, src); dpxbar_clear32(xbar, 0x20, 7 << (3 * xbar->selected_dispext[index])); dpxbar_clear32(xbar, 0x38, 7 << t8142_dest_shift[index]); @@ -948,7 +963,7 @@ static int t8142_link_down(struct apple_dpxbar *xbar, unsigned int index) static int t8142_link_up(struct apple_dpxbar *xbar, unsigned int index) { int state = xbar->selected_dispext[index]; - u32 clock, select, src, dst, value; + u32 clock, select, src, dst; int ret; if (state < 0) @@ -973,24 +988,13 @@ static int t8142_link_up(struct apple_dpxbar *xbar, unsigned int index) dpxbar_clear32(xbar, 0x1c, src); dpxbar_clear32(xbar, 0x34, dst); udelay(1); - ret = readl_poll_timeout_atomic(xbar->regs + 0x804, value, - !(value & src), 1, 1000); - if (ret) { - xbar->faulted = true; - return ret; - } - ret = readl_poll_timeout_atomic(xbar->regs + 0x818, value, - !(value & src), 1, 1000); - if (ret) { - xbar->faulted = true; - return ret; - } - ret = readl_poll_timeout_atomic(xbar->regs + 0x82c, value, - !(value & dst), 1, 1000); - if (ret) { - xbar->faulted = true; + ret = t8142_settle(xbar, 0x804, src, index, "link up"); + if (!ret) + ret = t8142_settle(xbar, 0x818, src, index, "link up"); + if (!ret) + ret = t8142_settle(xbar, 0x82c, dst, index, "link up"); + if (ret) return ret; - } dpxbar_set32(xbar, 0x08, src); dpxbar_mask32(xbar, 0x20, 7 << (3 * state), select << (3 * state)); dpxbar_mask32(xbar, 0x38, 7 << t8142_dest_shift[index], @@ -1466,8 +1470,16 @@ static const struct apple_dpxbar_hw apple_dpxbar_hw_t8142 = { .ops = &apple_dpxbar_t8142_ops, }; +/* T6050: the T8142 layout; a status that stays busy is logged after settling. */ +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6050 = { + .n_mux = T8142_MUX_MAX, + .n_ufp = 8, + .ops = &apple_dpxbar_t8142_ops, + .status_log_only = true, +}; + static const struct of_device_id apple_dpxbar_ids[] = { - { .compatible = "apple,t6050-display-crossbar", .data = &apple_dpxbar_hw_t8142 }, + { .compatible = "apple,t6050-display-crossbar", .data = &apple_dpxbar_hw_t6050 }, { .compatible = "apple,t8142-display-crossbar", .data = &apple_dpxbar_hw_t8142 }, { .compatible = "apple,t8103-display-crossbar", From 0c906ebf668b7e481c62342a116524b6eedb02a8 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 20:05:40 -0500 Subject: [PATCH 236/392] usb: typec: tipd: Restore J716c dock USB across boot Reinitialize an inherited J716c boot-port USB2 connection before its first connect notification, after suppliers and the Type-C port exist. Limit the operation to the board property and normal host operation; exclude the debug bridge modes. Allow SN201202x up to five seconds to wake so a connected dock can restore its USB hub after boot. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../bindings/usb/apple,sn201202x.yaml | 7 ++ drivers/usb/typec/tipd/core.c | 68 ++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index c2d2eb1068ea8..282d26ae0c6d2 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -44,6 +44,13 @@ properties: connector: $ref: /schemas/connector/usb-connector.yaml# + apple,reset-inherited-usb2-connection: + type: boolean + description: + The boot USB port retains a USB2 repeater connection that cannot be + recovered by resetting the host PHY. Renegotiate a normal application + connection before starting the host controllers. Debug routes are preserved. + required: - compatible - reg diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 74c820dd0be24..5d157996b214a 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2017,7 +2017,14 @@ static int cd321x_switch_power_state(struct tps6598x *tps, u8 target_state) if (state == target_state) return 0; - ret = tps6598x_exec_cmd(tps, "SSPS", sizeof(u8), &target_state, 0, NULL); + /* SN201202x can still be negotiating an attached dock while waking. */ + if (target_state == TPS_SYSTEM_POWER_STATE_S0 && + device_is_compatible(tps->dev, "apple,sn201202x")) + ret = tps6598x_exec_cmd_tmo(tps, "SSPS", sizeof(target_state), + &target_state, 0, NULL, 5000, 0); + else + ret = tps6598x_exec_cmd(tps, "SSPS", sizeof(target_state), + &target_state, 0, NULL); if (ret) return ret; @@ -2905,6 +2912,61 @@ static int cd321x_remove(struct tps6598x *tps) return 0; } +static int cd321x_reset_inherited_connection(struct tps6598x *tps, u32 *status) +{ + /* J716c's port-controller key is the reversed target identifier J716. */ + static const u8 key[] = { '6', '1', '7', 'J' }; + static const u8 locked[4]; + u8 delay = 5, result; + unsigned long timeout; + int ret, lock_ret, mode; + + if (!device_property_read_bool(tps->dev, "apple,reset-inherited-usb2-connection") || + !of_machine_is_compatible("apple,j716c") || + !device_is_compatible(tps->dev, "apple,sn201202x") || + tps->dbma_at_probe || !(*status & TPS_STATUS_PLUG_PRESENT) || + !TPS_STATUS_TO_TYPEC_DATAROLE(*status)) + return 0; + mode = tps6598x_check_mode(tps); + if (mode != TPS_MODE_APP && mode != TPS_MODE_APP1) + return 0; + + /* All suppliers are available, but no USB or router session has started. */ + ret = tps6598x_exec_cmd(tps, "LOCK", sizeof(key), key, sizeof(result), &result); + if (ret) + return ret; + if (result) { + ret = -EIO; + goto relock; + } + + /* Nonzero DISC delay restores normal CC detection automatically. */ + ret = tps6598x_exec_cmd(tps, "DISC", sizeof(delay), &delay, 0, NULL); + if (!ret) { + msleep(5100); + timeout = jiffies + msecs_to_jiffies(2000); + do { + mode = tps6598x_check_mode(tps); + if (mode == TPS_MODE_APP || mode == TPS_MODE_APP1) + break; + msleep(20); + } while (time_before(jiffies, timeout)); + if (mode != TPS_MODE_APP && mode != TPS_MODE_APP1) + ret = mode < 0 ? mode : -ETIMEDOUT; + } +relock: + /* CC/PD negotiation may still be finishing when application mode returns. */ + lock_ret = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(locked), locked, 0, NULL, + 5000, 0); + if (ret || lock_ret) + return ret ?: lock_ret; + if (!tps6598x_read_status(tps, status)) + return -EIO; + tps->status = *status; + dev_info(tps->dev, "reinitialized inherited USB2 connection\n"); + return 0; +} + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -3017,6 +3079,10 @@ int tipd_init(struct tps6598x *tps) } } + ret = cd321x_reset_inherited_connection(tps, &status); + if (ret) + goto err_disconnect; + if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; if (!tps->data->read_power_status(tps)) From 371a42c0dfda5def5478b79634971ec14688a551 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 9 Nov 2026 23:01:09 +1100 Subject: [PATCH 237/392] selftests: apple-dcp: Stub the inherited-connection reset in the tipd probe test tipd_init() now calls cd321x_reset_inherited_connection() before it reads the plug state. The probe-transaction model extracts tipd_init() with device_property_read_bool() returning false, for which the real function returns 0 without touching the controller, so stub it that way. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index b41f7e2f80689..631237b2ac7b3 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -397,6 +397,7 @@ def probe(root): static int register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)t;(void)f;return register_error; } static bool read_data_status(struct tps6598x *t) { (void)t;return true; } static int connect(struct tps6598x *t,u32 status) { (void)t;(void)status;return connect_error; } +static int cd321x_reset_inherited_connection(struct tps6598x *t,u32 *s) { (void)t;(void)s;return 0; } """ helpers += '\n#define of_machine_is_compatible(name) (!strcmp((name), \"apple,t8140\"))\n#define BIT_ULL(n) (1ULL << (n))\n#define SN201202X_INT_DP_SID_UPDATE BIT_ULL(37)\n' helpers += block(core, 'static bool cd321x_uses_dp_sid(') + '\n' + block(core, 'static u64 cd321x_irq_mask(') + '\n' From b4d99c2f65664836da862c9bfae497d22e38e0b9 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 22:04:50 -0500 Subject: [PATCH 238/392] usb: typec: tipd: Wait for the inherited disconnect task An attached J716c dock can leave DISC running beyond the default one-second command timeout. Probe then attempts LOCK while DISC still owns the command register, receives EBUSY, and abandons the port even though a later reprobe works. Allow five seconds for DISC to acknowledge, matching the surrounding startup operations. Keep the automatic CC restore delay and the post-command application-mode check unchanged. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/usb/typec/tipd/core.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 5d157996b214a..a6eb842552f1e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2940,8 +2940,13 @@ static int cd321x_reset_inherited_connection(struct tps6598x *tps, u32 *status) goto relock; } - /* Nonzero DISC delay restores normal CC detection automatically. */ - ret = tps6598x_exec_cmd(tps, "DISC", sizeof(delay), &delay, 0, NULL); + /* + * Nonzero DISC delay restores normal CC detection automatically. The + * task can take more than a second to acknowledge at boot; wait for it + * before issuing LOCK, which a still-busy controller would reject. + */ + ret = tps6598x_exec_cmd_tmo(tps, "DISC", sizeof(delay), &delay, 0, NULL, + 5000, 0); if (!ret) { msleep(5100); timeout = jiffies + msecs_to_jiffies(2000); From a442efa4477ad2f39086f53c854839cc3e0a09fb Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 22:32:16 -0500 Subject: [PATCH 239/392] usb: typec: tipd: preserve dry result during LDCM startup An empty SN201202x port can take 1.6 seconds to acknowledge LDCM at startup. Allow five seconds so a command still running is not treated as a failed mitigation change. Removing mitigation can start a new measurement after a valid dry snapshot. Preserve that result while the new measurement is in progress, as the existing entry path does for a published result. Re-enabling mitigation immediately can hold the detector in the measuring state and permanently suppress a later dock attachment. Keep failures and liquid results mitigated. Exercise the dry-to-measuring transition from every previous state and the failed re-read path in KUnit. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/usb/typec/tipd/core.c | 5 +- drivers/usb/typec/tipd/sn201202x-pm-test.c | 53 ++++++++++++++++++++++ drivers/usb/typec/tipd/spmi.c | 18 +++++++- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index a6eb842552f1e..25325dde72a8c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -3078,8 +3078,9 @@ int tipd_init(struct tps6598x *tps) if (tps->contaminant_ops) { ret = tps->contaminant_ops->update(tps); if (ret) { - dev_warn(tps->dev, - "connector contaminant state unavailable: %d\n", ret); + if (ret != -EINPROGRESS) + dev_warn(tps->dev, + "connector contaminant state unavailable: %d\n", ret); ret = 0; } } diff --git a/drivers/usb/typec/tipd/sn201202x-pm-test.c b/drivers/usb/typec/tipd/sn201202x-pm-test.c index 68e36f6d951c8..ae55166c33179 100644 --- a/drivers/usb/typec/tipd/sn201202x-pm-test.c +++ b/drivers/usb/typec/tipd/sn201202x-pm-test.c @@ -826,6 +826,57 @@ static void sn201202x_ldcm_dry_order_test(struct kunit *test) sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); } +static void sn201202x_ldcm_dry_then_busy_test(struct kunit *test) +{ + static const enum typec_contaminant last[] = { + TYPEC_CONTAMINANT_UNKNOWN, + TYPEC_CONTAMINANT_NONE, + TYPEC_CONTAMINANT_LIQUID, + }; + const enum sn201202x_ldcm_test_action actions[] = { + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_OFF, + LDCM_TEST_READ, + LDCM_TEST_PUBLISH_NONE, + }; + unsigned int i; + + /* Removing mitigation can start a measurement after a fresh dry result. */ + for (i = 0; i < ARRAY_SIZE(last); i++) { + struct sn201202x_ldcm_test t = { + .reads = { TYPEC_CONTAMINANT_NONE, TYPEC_CONTAMINANT_UNKNOWN }, + }; + + KUNIT_EXPECT_EQ(test, sn201202x_ldcm_update_state(&t, + &sn201202x_ldcm_test_ops, + last[i]), + -EINPROGRESS); + sn201202x_ldcm_test_actions(test, &t, actions, + ARRAY_SIZE(actions)); + } +} + +static void sn201202x_ldcm_dry_then_read_failure_test(struct kunit *test) +{ + const enum sn201202x_ldcm_test_action actions[] = { + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_OFF, + LDCM_TEST_READ, + LDCM_TEST_MITIGATE_ON, + LDCM_TEST_PUBLISH_UNKNOWN, + }; + struct sn201202x_ldcm_test t = { + .reads = { TYPEC_CONTAMINANT_NONE, TYPEC_CONTAMINANT_UNKNOWN }, + .read_rets = { 0, -EIO }, + }; + + /* A failed second read must still mitigate, even after a dry result. */ + KUNIT_EXPECT_EQ(test, sn201202x_ldcm_update_state(&t, + &sn201202x_ldcm_test_ops, + TYPEC_CONTAMINANT_UNKNOWN), -EIO); + sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); +} + static void sn201202x_ldcm_dry_race_test(struct kunit *test) { const enum sn201202x_ldcm_test_action actions[] = { @@ -945,6 +996,8 @@ static struct kunit_case sn201202x_pm_cases[] = { KUNIT_CASE(sn201202x_ldcm_read_failure_order_test), KUNIT_CASE(sn201202x_ldcm_mitigation_failure_order_test), KUNIT_CASE(sn201202x_ldcm_dry_order_test), + KUNIT_CASE(sn201202x_ldcm_dry_then_busy_test), + KUNIT_CASE(sn201202x_ldcm_dry_then_read_failure_test), KUNIT_CASE(sn201202x_ldcm_dry_race_test), KUNIT_CASE(sn201202x_ldcm_dry_race_mitigation_failure_test), KUNIT_CASE(sn201202x_ldcm_busy_keeps_result_test), diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 32c3778871013..0ea062474dc0f 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -300,9 +300,13 @@ static int sn201202x_ldcm_set_mitigation(void *ctx, bool enable) if (sn->ldcm_mitigation_known && sn->ldcm_mitigated == enable) return 0; - /* Successful task completion is the controller's mitigation ACK. */ + /* + * Successful task completion is the controller's mitigation ACK. + * Startup commands can take more than a second on SN201202x. Wait for + * completion before a retry can request the opposite mitigation state. + */ ret = tps6598x_exec_cmd_tmo(tps, "LDCM", sizeof(command), &command, - 0, NULL, 1000, 0); + 0, NULL, 5000, 0); sn->ldcm_mitigation_known = !ret; if (!ret) sn->ldcm_mitigated = enable; @@ -354,6 +358,16 @@ sn201202x_ldcm_update_state(void *ctx, const struct sn201202x_ldcm_ops *ops, /* Do not report dry if liquid appeared while mitigation was removed. */ read_ret = ops->read(ctx, &contaminant); + if (!read_ret && contaminant == TYPEC_CONTAMINANT_UNKNOWN) { + /* + * Removing mitigation can start another measurement. The dry + * snapshot above is still the latest result; preserve it just + * as we preserve a previously published result at entry. Forcing + * mitigation back on here can prevent the measurement finishing. + */ + ops->publish(ctx, TYPEC_CONTAMINANT_NONE); + return -EINPROGRESS; + } if (read_ret || contaminant != TYPEC_CONTAMINANT_NONE) { ret = ops->set_mitigation(ctx, true); if (ret) { From 0471ea2c525c8e6d041479c60593e60f0ad90158 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 22:46:40 -0500 Subject: [PATCH 240/392] usb: typec: tipd: disable LDCM work during removal Reset recovery can invalidate the detector after tipd_remove cancels its work but before cd321x_remove drains resume work. Disable delayed work permanently at removal so this late producer cannot restart it after driver resources are released. Exercise the production stop and invalidate callbacks in KUnit. The regression test runs the callback after cancellation on the old implementation and passes when work is disabled. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/usb/typec/tipd/sn201202x-pm-test.c | 50 ++++++++++++++++++++++ drivers/usb/typec/tipd/spmi.c | 6 ++- drivers/usb/typec/tipd/tps6598x.h | 1 + 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/sn201202x-pm-test.c b/drivers/usb/typec/tipd/sn201202x-pm-test.c index ae55166c33179..7fab562ea81d8 100644 --- a/drivers/usb/typec/tipd/sn201202x-pm-test.c +++ b/drivers/usb/typec/tipd/sn201202x-pm-test.c @@ -982,6 +982,55 @@ static void sn201202x_ldcm_read_failure_with_result_test(struct kunit *test) sn201202x_ldcm_test_actions(test, &t, actions, ARRAY_SIZE(actions)); } +struct sn201202x_ldcm_stop_test { + struct sn201202x sn; + atomic_t runs; +}; + +static void sn201202x_ldcm_test_work(struct work_struct *work) +{ + struct sn201202x *sn = container_of(to_delayed_work(work), + struct sn201202x, ldcm_work); + struct sn201202x_ldcm_stop_test *t = container_of(sn, + struct sn201202x_ldcm_stop_test, sn); + + atomic_inc(&t->runs); +} + +static void sn201202x_ldcm_test_cancel(void *data) +{ + cancel_delayed_work_sync(data); +} + +static void sn201202x_ldcm_stop_requeue_test(struct kunit *test) +{ + struct sn201202x_ldcm_stop_test *t; + struct tps6598x *tps; + int ret; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t); + tps = &t->sn.cd.tps; + mutex_init(&tps->lock); + atomic_set(&t->runs, 0); + INIT_DELAYED_WORK(&t->sn.ldcm_work, sn201202x_ldcm_test_work); + ret = kunit_add_action_or_reset(test, sn201202x_ldcm_test_cancel, &t->sn.ldcm_work); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* Removal must cancel a pending retry and prevent later producers. */ + mod_delayed_work(system_freezable_wq, &t->sn.ldcm_work, 60 * HZ); + sn201202x_contaminant_ops.stop(tps); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->sn.ldcm_work)); + + /* Reset recovery can still invalidate the detector before it is drained. */ + mutex_lock(&tps->lock); + sn201202x_contaminant_ops.invalidate(tps); + mutex_unlock(&tps->lock); + flush_delayed_work(&t->sn.ldcm_work); + KUNIT_EXPECT_EQ(test, atomic_read(&t->runs), 0); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->sn.ldcm_work)); +} + static struct kunit_case sn201202x_pm_cases[] = { KUNIT_CASE(sn201202x_sleep_cycle_test), KUNIT_CASE(sn201202x_failed_wakeup_test), @@ -1003,6 +1052,7 @@ static struct kunit_case sn201202x_pm_cases[] = { KUNIT_CASE(sn201202x_ldcm_busy_keeps_result_test), KUNIT_CASE(sn201202x_ldcm_busy_without_result_test), KUNIT_CASE(sn201202x_ldcm_read_failure_with_result_test), + KUNIT_CASE(sn201202x_ldcm_stop_requeue_test), KUNIT_CASE(sn201202x_idle_sleep_test), KUNIT_CASE(sn201202x_atomic_access_test), KUNIT_CASE(sn201202x_idle_sleep_retry_test), diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index 0ea062474dc0f..30d472c3d83a4 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -467,16 +467,18 @@ static void sn201202x_ldcm_invalidate(struct tps6598x *tps) static void sn201202x_ldcm_stop(struct tps6598x *tps) { - cancel_delayed_work_sync(&tps_to_sn(tps)->ldcm_work); + /* Reset recovery is drained later and may still try to queue an update. */ + disable_delayed_work_sync(&tps_to_sn(tps)->ldcm_work); } -static const struct tipd_contaminant_ops sn201202x_contaminant_ops = { +VISIBLE_IF_KUNIT const struct tipd_contaminant_ops sn201202x_contaminant_ops = { .update = sn201202x_ldcm_update, .refresh = sn201202x_ldcm_refresh, .event = sn201202x_ldcm_event, .invalidate = sn201202x_ldcm_invalidate, .stop = sn201202x_ldcm_stop, }; +EXPORT_SYMBOL_IF_KUNIT(sn201202x_contaminant_ops); static struct regmap *__devm_regmap_init_sn201202x(struct spmi_device *sdev, const struct regmap_config *config, diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c39feab87c20f..20b6b73436a29 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -564,6 +564,7 @@ int sn201202x_runtime_resume(struct device *dev); void sn201202x_idle_sleep_start(struct tps6598x *tps); void sn201202x_idle_sleep_stop(struct tps6598x *tps); extern const struct regmap_bus regmap_sn201202x; +extern const struct tipd_contaminant_ops sn201202x_contaminant_ops; u32 sn201202x_sleep_links(const struct cd321x_sleep_state *st); enum typec_contaminant sn201202x_ldcm_decode(const u8 status[SN201202X_LDCM_STATUS_LEN]); From 10ada134e51a575b25f3cd9c45fea27841edab3f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 07:06:54 -0500 Subject: [PATCH 241/392] pmdomain: apple: pmc: Add T6050 The boot loader leaves every PMC rail of the T6050 with no voter agent enabled, so nothing raises the DRAM controller (DCS) or the SoC fabric above state 0. On the M5 Pro a CPU memcpy then caps at about 42 GB/s for 1 to 12 threads and dependent DRAM loads take 310 ns. T6050 uses the T8140 voter layout with five agents (PMP, CLPC, GFX, AP and the second die's PMP1, from the ADT's voter-agent) and the pending word after each floor word that T8132 has. Its rails are DCS, SOC, DISP and DISP2 (voter-rail-data); the PMP's dvfs-domain lists DCS states 0-6 and fabric states 0-3. As on T6040, agents are enabled on DCS and SOC only. Tested on J714s: the PMC enables the CLPC and AP agents on DCS and SOC (enable words 0xa). With the AP's floor swept by hand, 12-thread memcpy reads 46, 81, 108, 149, 159, 167 and 171 GB/s at DCS 0-6, and 177 GB/s at DCS 6 with fabric 2 or 3; DRAM latency falls from 324 to 135 ns. The layout follows Jeff Queisser's J775D (M5 Ultra) series. Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/pmc.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 3765e9a775ff7..e7189b4a9590a 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1385,7 +1385,29 @@ static const struct apple_pmc_hw apple_pmc_hw_t8152 = { .ap_agent = 3, }; +/* + * T6050 (M5 Pro/Max): the T8140 layout with five agents (PMP, CLPC, GFX, AP + * and the second die's PMP1, ADT voter-agent) and the T8132 pending word + * after each floor word. Rails DCS, SOC, DISP and DISP2; the PMP's + * dvfs-domain lists DCS states 0-6 and fabric states 0-3. The boot loader + * enables no agent on any rail. Agents are enabled on DCS and SOC only, as + * on T6040. + */ +static const struct apple_pmc_hw apple_pmc_hw_t6050 = { + .agent_stride = 0x4000, + .agents = 5, + .rails = 4, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 6, + .agent_rails = GENMASK(1, 0), + .ap_agent = 3, + .has_pending = true, +}; + static const struct of_device_id apple_pmc_of_match[] = { + { .compatible = "apple,t6050-pmc", .data = &apple_pmc_hw_t6050 }, { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, { .compatible = "apple,t8142-pmc", .data = &apple_pmc_hw_t8142 }, From 68e0bebd3dbf60c7c6cc7a7745cdcc7fccd11b96 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 07:06:55 -0500 Subject: [PATCH 242/392] arm64: dts: apple: t6050: Link the GPU to the CLPC The GPU driver reports its work in flight to the CLPC named by apple,clpc, which then holds the DRAM controller and fabric floors up while there is some. Without the link, GPU work gets no floor and bandwidth-bound work runs at about a tenth of macOS's rate. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-peripherals.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index eef667d647d3d..f04009130a73c 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -3139,6 +3139,8 @@ , ; power-domains = <&ps_gfx>; + /* Raises the DRAM and fabric floors while GPU work is in flight */ + apple,clpc = <&clpc>; status = "disabled"; }; From 0e3325d9a65b6ae2f942d15998c657368e2b96dc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 07:06:55 -0500 Subject: [PATCH 243/392] soc: apple: clpc: Add T6050 T6050 has DCS states 0-6 on rail 0 and fabric states 0-3 on rail 1. As on T6040, a cluster requests a state after three busy samples in a row (12 ms), and the GPU driver reports its work in flight, which gets the top of the map while there is some. Idle releases the floor. Tested on J714s (M5 Pro), with the T6050 cpufreq profile loaded: - CPU: a busy cluster at its top state holds DCS 6 / fabric 3; idle clusters at 300 MHz hold nothing. 12-thread memcpy reads 252 GB/s and DRAM latency is 109 ns, against 42 GB/s and 310 ns with neither the floor nor cpufreq (177 GB/s and 135 ns with the floor alone). - GPU (gpubench against macOS 27.2): bandwidth-bound compute 3165 -> 486 us per submission (9.5x -> 1.46x macOS, 42 -> 276 GB/s), buffer copies 3165 -> 486 us (7.0x -> 1.07x), light compute 124 -> 54 us (38 us with cpufreq); ALU and fill stay at macOS's rate. - Holding DCS 6 / fabric 3 at idle costs 1.4 W of system power, which the release avoids. Signed-off-by: Diljit Singh --- drivers/soc/apple/clpc.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 6dfd277463a13..1c590aa538f23 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -265,6 +265,7 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) if (!np) return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; if (!of_device_is_compatible(np, "apple,t6041-clpc") && + !of_device_is_compatible(np, "apple,t6050-clpc") && !of_device_is_compatible(np, "apple,t8132-clpc") && !of_device_is_compatible(np, "apple,t8140-clpc") && !of_device_is_compatible(np, "apple,t8142-clpc")) { @@ -1669,7 +1670,27 @@ static const struct apple_clpc_hw apple_clpc_hw_t8152 = { }, }; +/* + * T6050 (M5 Pro/Max): DCS states 0-6 on rail 0, fabric states 0-3 on rail 1. + * The boot loader enables no agent on either rail, so without this floor + * the DRAM controller and the fabric stay at state 0: CPU memcpy caps at + * about 34 GB/s on the M5 Pro. A busy cluster in the upper half of its range + * asks for the top states; three busy samples in a row (12 ms) are needed, + * as on T6040. The GPU driver reports its work in flight and gets the top + * of the map while it has some. + */ +static const struct apple_clpc_hw apple_clpc_hw_t6050 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 6 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 3 }, + }, + .sustain = 3, + .gpu_input = true, +}; + static const struct of_device_id apple_clpc_of_match[] = { + { .compatible = "apple,t6050-clpc", .data = &apple_clpc_hw_t6050 }, { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, { .compatible = "apple,t8142-clpc", .data = &apple_clpc_hw_t8142 }, From ed7f7d392e89355d8636eab35cbdb9372822b9a1 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:17:02 +0100 Subject: [PATCH 244/392] iommu: apple-dart: generalize the external display power gating An adopted J713 external display DART can be unpowered across sleep, so its registers are used only with its power confirmed: apple_dart_dispext_get() counts a use, an invalidation waits for the uses and marks the DART unconfirmed until a state check after power-up. The internal display DARTs can lose power the same way once their DCP powers its domains down. Give the gate its own flag, power_gated, set for the external display DARTs as before, and rename its state and helpers from dispext_* to pm_*. The external display DART's adoption, frontends, fault handling and state check stay as they are. No functional change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/iommu/apple-dart-dcp-shared.h | 12 ++-- drivers/iommu/apple-dart-dcp.h | 75 ++++++++++++------------ drivers/iommu/apple-dart.c | 82 +++++++++++++++------------ 3 files changed, 89 insertions(+), 80 deletions(-) diff --git a/drivers/iommu/apple-dart-dcp-shared.h b/drivers/iommu/apple-dart-dcp-shared.h index f597edd6b6128..2396ac5e1d33c 100644 --- a/drivers/iommu/apple-dart-dcp-shared.h +++ b/drivers/iommu/apple-dart-dcp-shared.h @@ -378,7 +378,7 @@ int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid) if (apple_dart_dispext_retired(pio->dart)) return -EIO; /* Its power not confirmed since sleep: usable later, not now. */ - return apple_dart_dispext_unconfirmed(pio->dart) ? -EAGAIN : 0; + return apple_dart_pm_unconfirmed(pio->dart) ? -EAGAIN : 0; } EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_stream_ok); @@ -807,7 +807,7 @@ static int __apple_dart_dcp_shared_map(struct device *dev, struct device *piodma /* * J713 external DCP: the external display DART operations above, each - * inside one use of the DART's registers (apple_dart_dispext_get()): with + * inside one use of the DART's registers (apple_dart_pm_get()): with * its power not confirmed since sleep, -EAGAIN before any step, nothing * published, nothing retired. A sleep waits for a use under way to end, so * no step of it (its flushes included) runs unconfirmed. @@ -824,12 +824,12 @@ static struct apple_dart *apple_dart_dcp_dispext_gate(struct device *piodma) struct apple_dart *__gd = apple_dart_dcp_dispext_gate(piodma); \ typeof(call) __ret; \ \ - if (__gd && !apple_dart_dispext_get(__gd)) { \ + if (__gd && !apple_dart_pm_get(__gd)) { \ __ret = (typeof(call))-EAGAIN; \ } else { \ __ret = (call); \ if (__gd) \ - apple_dart_dispext_put(__gd); \ + apple_dart_pm_put(__gd); \ } \ __ret; \ }) @@ -863,11 +863,11 @@ void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u struct apple_dart *dart = apple_dart_dcp_dispext_gate(piodma); void *ret; - if (dart && !apple_dart_dispext_get(dart)) + if (dart && !apple_dart_pm_get(dart)) return ERR_PTR(-EAGAIN); ret = __apple_dart_dcp_dispext_scanout_alloc(piodma, size, dva); if (dart) - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_scanout_alloc); diff --git a/drivers/iommu/apple-dart-dcp.h b/drivers/iommu/apple-dart-dcp.h index 6e74f56c5132c..b65115f08d6ec 100644 --- a/drivers/iommu/apple-dart-dcp.h +++ b/drivers/iommu/apple-dart-dcp.h @@ -68,65 +68,66 @@ struct apple_dart_dcp { } while (0) /* - * J713 external display DART: its power not confirmed since sleep (from - * suspend until its firmware powers it up again): no register of it may be - * read or written, and nothing is retired for that. + * A power-gated display DART (see struct apple_dart): its power not + * confirmed since it may have gone off (from sleep, or from a power-down by + * its DCP, until it is confirmed on again): no register of it may be read or + * written, and nothing is retired for that. */ -static inline bool apple_dart_dispext_unconfirmed(const struct apple_dart *dart) +static inline bool apple_dart_pm_unconfirmed(const struct apple_dart *dart) { - return dart->dispext && READ_ONCE(dart->dispext_unverified); + return dart->power_gated && READ_ONCE(dart->pm_unverified); } /* - * A use of an external display DART's registers, taken at the start of an + * A use of a power-gated display DART's registers, taken at the start of an * operation and dropped at its end, and only with its power confirmed: - * false (the caller returns -EAGAIN, nothing retired) from sleep until its - * next confirmed power-up. Inside it the operation's steps, flushes included, - * run unchecked: an invalidation (sleep) waits for it to end. Any other - * DART: always true, no count. + * false (the caller returns -EAGAIN, nothing retired) from the moment it may + * go off until its next confirmed power-up. Inside it the operation's steps, + * flushes included, run unchecked: an invalidation waits for it to end. Any + * other DART: always true, no count. */ -static inline bool apple_dart_dispext_get(struct apple_dart *dart) +static inline bool apple_dart_pm_get(struct apple_dart *dart) { unsigned long flags; bool ok; - if (!dart->dispext) + if (!dart->power_gated) return true; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - ok = !dart->dispext_unverified; + spin_lock_irqsave(&dart->pm_gate_lock, flags); + ok = !dart->pm_unverified; if (ok) - dart->dispext_users++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + dart->pm_users++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); return ok; } -static inline void apple_dart_dispext_put(struct apple_dart *dart) +static inline void apple_dart_pm_put(struct apple_dart *dart) { unsigned long flags; - if (!dart->dispext) + if (!dart->power_gated) return; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - if (!--dart->dispext_users) - wake_up_all(&dart->dispext_wq); - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + if (!--dart->pm_users) + wake_up_all(&dart->pm_wq); + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); } /* - * Sleep: no power confirmed from now on, a new generation (a check - * still under way confirms nothing), and every use in flight waited for. - * @suspend: asleep until the matching resume (no check runs then). + * Sleep or a power-down: no power confirmed from now on, a new generation (a + * check still under way confirms nothing), and every use in flight waited + * for. @suspend: asleep until the matching resume (no check runs then). */ -static inline void apple_dart_dispext_invalidate(struct apple_dart *dart, bool suspend) +static inline void apple_dart_pm_invalidate(struct apple_dart *dart, bool suspend) { unsigned long flags; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - dart->dispext_unverified = true; - dart->dispext_sleeping = suspend; - dart->dispext_gen++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); - wait_event(dart->dispext_wq, !READ_ONCE(dart->dispext_users)); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + dart->pm_unverified = true; + dart->pm_sleeping = suspend; + dart->pm_gen++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); + wait_event(dart->pm_wq, !READ_ONCE(dart->pm_users)); } /* Every borrowed stream of @dart, for one reason. */ @@ -207,7 +208,7 @@ static bool apple_dart_dcp_registers(struct apple_dart_dcp *dcp) struct apple_dart *dart = dcp->dart; u32 protect, ttbr, tcr, streams; - /* An external display DART's callers hold a use of it (apple_dart_dispext_get()). */ + /* An external display DART's callers hold a use of it (apple_dart_pm_get()). */ if (READ_ONCE(dcp->failed)) return false; protect = readl(dart->regs + DART_T8110_PROTECT); @@ -643,7 +644,7 @@ static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova * An external display DART's registers only inside a use of them (its * power confirmed, a sleep waits for it): none, no lookup. */ - if (dcp->failed || !apple_dart_dispext_get(dcp->dart)) + if (dcp->failed || !apple_dart_pm_get(dcp->dart)) goto out; used = true; if (!apple_dart_dcp_registers(dcp)) @@ -675,7 +676,7 @@ static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova out: spin_unlock_irqrestore(&dcp->lock, flags); if (used) - apple_dart_dispext_put(dcp->dart); + apple_dart_pm_put(dcp->dart); return phys; } @@ -1375,13 +1376,13 @@ static size_t apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, * has no generic DMA user (its host mappings are the private window and * scanout APIs, which never come through here). */ - if (!apple_dart_dispext_get(dcp->dart)) { + if (!apple_dart_pm_get(dcp->dart)) { dev_err_ratelimited(dcp->dart->dev, "SID%u unmap at %llx while unconfirmed since sleep: refused\n", dcp->sid, iova); return 0; } unmapped = __apple_dart_dcp_unmap(dcp, iova, size, count, gather); - apple_dart_dispext_put(dcp->dart); + apple_dart_pm_put(dcp->dart); return unmapped; } diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..ed6a33e187fc0 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -320,20 +320,27 @@ struct apple_dart { /* A J713 external display DART, borrowed from its external DCP */ bool dispext; /* - * J713 external display DART: its power confirmation across - * sleep, under dispext_pm_lock (a spinlock: only these fields, never + * A display DART whose power can go away under its borrowed streams: + * an external display DART across sleep (its firmware powers it), or an + * internal one powered down by its DCP. Its registers are used only + * with its power confirmed (apple_dart_pm_get()). + */ + bool power_gated; + /* + * Power-gated display DART: its power confirmation across + * sleep, under pm_gate_lock (a spinlock: only these fields, never * across a register access): unverified from suspend until a state * check at a D572(1) of the current generation, which every suspend * and resume moves on; sleeping between suspend and resume; and the - * register uses in flight (apple_dart_dispext_get()), which an - * invalidation waits for (dispext_wq). + * register uses in flight (apple_dart_pm_get()), which an + * invalidation waits for (pm_wq). */ - spinlock_t dispext_pm_lock; - bool dispext_unverified; - bool dispext_sleeping; - u64 dispext_gen; - unsigned int dispext_users; - wait_queue_head_t dispext_wq; + spinlock_t pm_gate_lock; /* the pm_* fields below */ + bool pm_unverified; + bool pm_sleeping; + u64 pm_gen; + unsigned int pm_users; + wait_queue_head_t pm_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; @@ -1576,10 +1583,10 @@ static int apple_dart_dcp_invalidate_tlb_gated(struct apple_dart_stream_map *str { int ret; - if (!apple_dart_dispext_get(stream_map->dart)) + if (!apple_dart_pm_get(stream_map->dart)) return -EAGAIN; ret = apple_dart_dcp_invalidate_tlb(stream_map); - apple_dart_dispext_put(stream_map->dart); + apple_dart_pm_put(stream_map->dart); return ret; } @@ -2036,10 +2043,10 @@ static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) struct apple_dart *dart = domain->dcp->dart; int ret; - if (!apple_dart_dispext_get(dart)) + if (!apple_dart_pm_get(dart)) return -EAGAIN; ret = __apple_dart_dcp_domain_flush(domain); - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } @@ -2785,11 +2792,11 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) int ret = -EINVAL; /* An external display DART's registers only inside a use. */ - if (borrowed && !apple_dart_dispext_get(borrowed->dart)) { + if (borrowed && !apple_dart_pm_get(borrowed->dart)) { ret = -EAGAIN; } else if (borrowed) { ret = apple_dart_dcp_validate(borrowed, dev); - apple_dart_dispext_put(borrowed->dart); + apple_dart_pm_put(borrowed->dart); } if (ret) return ERR_PTR(ret); @@ -3054,7 +3061,7 @@ static int apple_dart_of_xlate(struct device *dev, #if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) /* * The check, inside a use of the DART's registers (on an external display - * DART, apple_dart_dispext_get() by the caller). + * DART, apple_dart_pm_get() by the caller). */ static int __apple_dart_dcp_check(struct device *dev) { @@ -3126,10 +3133,10 @@ int apple_dart_dcp_check(struct device *dev) if (!domain || domain->owner != &apple_dart_iommu_ops || !to_dart_domain(domain)->dcp) return __apple_dart_dcp_check(dev); dart = to_dart_domain(domain)->dcp->dart; - if (!apple_dart_dispext_get(dart)) + if (!apple_dart_pm_get(dart)) return -EAGAIN; ret = __apple_dart_dcp_check(dev); - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } EXPORT_SYMBOL_GPL(apple_dart_dcp_check); @@ -4261,8 +4268,9 @@ static int apple_dart_probe(struct platform_device *pdev) spin_lock_init(&dart->managed_lock); dart->preserve_display = j713_display || j713_dcpext || j713_dispext; dart->dispext = j713_dispext; - spin_lock_init(&dart->dispext_pm_lock); - init_waitqueue_head(&dart->dispext_wq); + dart->power_gated = j713_dispext; + spin_lock_init(&dart->pm_gate_lock); + init_waitqueue_head(&dart->pm_wq); if (j713_dcpext) dart->dcp_instance = &dcp_adt_external[res->start == 0x48a340000ULL]; else if (j713_dispext) @@ -5061,7 +5069,7 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; if (apple_dart_has_borrowed(dart)) - apple_dart_dispext_invalidate(dart, true); + apple_dart_pm_invalidate(dart, true); return 0; } /* @@ -5183,13 +5191,13 @@ static __maybe_unused int apple_dart_resume(struct device *dev) /* * An adopted external display DART may be off now, its display with it, * and nothing confirms otherwise: no register of it is read or written - * (apple_dart_dispext_unconfirmed()) until its firmware powers it up + * (apple_dart_pm_unconfirmed()) until its firmware powers it up * again; the state check then (apple_dart_dcp_dispext_state()) runs * first: kept, it goes on; lost, it is retired, saying so. */ if (dart->dispext) { if (apple_dart_has_borrowed(dart)) { - apple_dart_dispext_invalidate(dart, false); + apple_dart_pm_invalidate(dart, false); dev_dbg(dev, "external display DART unconfirmed after sleep\n"); } return 0; @@ -5290,16 +5298,16 @@ static int apple_dart_dispext_verify(struct apple_dart *dart, const char *when) * not while asleep (between suspend and resume nothing confirms its * power), and an invalidation meanwhile waits for it to end. */ - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - if (dart->dispext_sleeping) { - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + if (dart->pm_sleeping) { + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); dev_dbg(dart->dev, "external display DART %s during sleep: not checked\n", when); return -EAGAIN; } - gen = dart->dispext_gen; - was = dart->dispext_unverified; - dart->dispext_users++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + gen = dart->pm_gen; + was = dart->pm_unverified; + dart->pm_users++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); protect = readl(dart->regs + DART_T8110_PROTECT); lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); @@ -5319,14 +5327,14 @@ static int apple_dart_dispext_verify(struct apple_dart *dart, const char *when) kept &= (protect & lock & DART_T8110_PROTECT_TTBR_TCR) == (dart->dispext_protect & dart->dispext_lock & DART_T8110_PROTECT_TTBR_TCR); - spin_lock_irqsave(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); /* Only a check of the current generation confirms: a sleep since voids it. */ - current_gen = gen == dart->dispext_gen; + current_gen = gen == dart->pm_gen; if (kept && current_gen) - dart->dispext_unverified = false; - if (!--dart->dispext_users) - wake_up_all(&dart->dispext_wq); - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + dart->pm_unverified = false; + if (!--dart->pm_users) + wake_up_all(&dart->pm_wq); + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); dev_dbg(dart->dev, "external display DART %s: protect %08x lock %08x (adopted %08x/%08x): state %s%s%s\n", when, protect, lock, dart->dispext_protect, dart->dispext_lock, From e794d107fcdc9ec1149883816be5a9ba870e7dd6 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:41:57 +0100 Subject: [PATCH 245/392] dt-bindings: iommu: apple,dart: describe the clock protection slice On T8132 a power state register of a display power domain is guarded by a "clock protection" slice in the PS_PROT register bank. While the slice is enabled, a write that powers the domain's CPU down faults. The native DART power-down disables the DART's streams, flushes it and then clears the slice; power-up sets it again. Add apple,clock-protection to the DART binding: a phandle to the PS_PROT bank and the DART's slice in it. The bank is shared by several DARTs, so describe it as a syscon and add its compatible to the syscon binding. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- .../devicetree/bindings/iommu/apple,dart.yaml | 13 +++++++++++++ Documentation/devicetree/bindings/mfd/syscon.yaml | 1 + 2 files changed, 14 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..73a3e1764fcd8 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -132,6 +132,19 @@ properties: handoff. Linux retains occupied root slots and excludes their entire IOVA spans from allocation. This does not assert immutable descendants. + apple,clock-protection: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: PS_PROT (clock protection) register bank + - description: Index of this DART's slice in the bank + description: > + The clock protection slice that guards a power state register of the + power domain of this DART. Its consumer may power that domain down: the + DART's streams are then disabled and flushed, and the slice disabled, + before the domain powers off, and the slice is enabled again when it + powers up. Only the J713 internal display DARTs use it. + apple,tunable: $ref: /schemas/types.yaml#/definitions/uint32-matrix items: diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml index 945a168b613db..4c6bfa4986ea0 100644 --- a/Documentation/devicetree/bindings/mfd/syscon.yaml +++ b/Documentation/devicetree/bindings/mfd/syscon.yaml @@ -46,6 +46,7 @@ properties: - apm,xgene-mcb - apm,xgene-rb - apm,xgene-scu + - apple,t8132-ps-prot - atmel,sama5d2-sfrbu - atmel,sama5d3-nfc-io - atmel,sama5d3-sfrbu From d584152cb4b50d9b7f4346f74c773b6a247b405c Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:41:57 +0100 Subject: [PATCH 246/392] iommu: apple-dart: power down the J713 internal display DARTs The J713 internal display DARTs are borrowed from the running DCP and stay powered: the DCP keeps the display domains on. To let the DCP power the domains down while the display is off, add apple_dart_dcp_power_off() and apple_dart_dcp_power_on() for a DART that has a clock protection slice (apple,clock-protection). Power-down, with the DCP CPU stopped and the DART's masters idle, marks the DART's power unconfirmed (apple_dart_pm_get() refuses from then on and the uses in flight are waited for) and gates its commands. Then, as the native DART power-down does, it requires no fault, waits up to 1 ms for the TLB command and outstanding transactions to drain, disables the enabled streams, flushes with the power-down TLB command (V2 | STT) and clears the clock protection slice. It saves the translations, their lock and the stream enables first. apple_dart_dcp_power_check() says, changing nothing, whether a power-down can run, so that the DCP can ask before it starts. Power-up restores the slice, requires the streams still disabled and the translations and their lock as saved (a DART that lost them is not rebuilt), flushes, enables the saved streams and confirms the power. A failure in either direction retires the DART's borrowed streams, as other display DART failures do; nothing is retried. While the DART is down its generic DMA streams, the display's SID0 and the DCP's SID23, keep working on their page tables: unmap, the flush after a map and translation use the tables in memory and no register, and the power-up flushes before it enables the streams. The DART's powered-down state changes only with the locks of all its borrowed streams held, so the power-up's flush and its confirmed power are one step for those updates. System resume leaves a powered-down DART alone; its gated commands already keep suspend and the fault handler off its registers. KUnit: a full cycle against RAM registers and a RAM-backed PS_PROT bank, a DART that comes back without its translations, refusals for a locked bank, a stream that stays enabled, a transaction that does not drain and a pending fault, and a SID0 stream whose unmap, flush and translation use only its page tables while the DART is down. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/iommu/Kconfig | 1 + drivers/iommu/apple-dart-dcp.h | 59 +++++- drivers/iommu/apple-dart-test.c | 194 ++++++++++++++++++++ drivers/iommu/apple-dart.c | 316 +++++++++++++++++++++++++++++++- include/linux/soc/apple/dart.h | 12 ++ 5 files changed, 577 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 03a3c6d3759e1..dd2dd23043b54 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -319,6 +319,7 @@ config APPLE_DART config APPLE_DART_J713_DCP bool "Retained J713 display DART support" depends on APPLE_DART=y && ARM64_16K_PAGES + select MFD_SYSCON help Adopt the qualified running J713 internal display DARTs. Their firmware-visible allocations, page tables and noncoherent CPU diff --git a/drivers/iommu/apple-dart-dcp.h b/drivers/iommu/apple-dart-dcp.h index b65115f08d6ec..49ed48c2f3de6 100644 --- a/drivers/iommu/apple-dart-dcp.h +++ b/drivers/iommu/apple-dart-dcp.h @@ -130,6 +130,36 @@ static inline void apple_dart_pm_invalidate(struct apple_dart *dart, bool suspen wait_event(dart->pm_wq, !READ_ONCE(dart->pm_users)); } +/* + * The locks of every borrowed stream of @dart, in a fixed order (SID23, then + * SID0, then SID16). An internal display DART's host_off changes only with + * all of them held, so under any one of them it is stable: while it is set, + * the DART's DCP has it powered down (or powering down), and its generic DMA + * streams (SID0, SID23) update their page tables in memory only, with no + * register used; its power-up flushes every translation, with these locks + * held, before it enables the streams again. + */ +static void apple_dart_host_lock(struct apple_dart *dart, unsigned long *flags) +{ + struct apple_dart_dcp *streams[] = { dart->dcp, dart->frontend[0], dart->frontend[1] }; + unsigned int held = 0; + + local_irq_save(*flags); + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i]) + spin_lock_nested(&streams[i]->lock, held++); +} + +static void apple_dart_host_unlock(struct apple_dart *dart, unsigned long flags) +{ + struct apple_dart_dcp *streams[] = { dart->frontend[1], dart->frontend[0], dart->dcp }; + + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i]) + spin_unlock(&streams[i]->lock); + local_irq_restore(flags); +} + /* Every borrowed stream of @dart, for one reason. */ #define apple_dart_fail_borrowed(dart, fmt, ...) \ do { \ @@ -640,6 +670,18 @@ static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); goto out; } + /* + * Powered down by its DCP (stable under this lock): no register, so + * only the host-owned leaves, which expected[] mirrors. + */ + if (READ_ONCE(dcp->dart->host_off)) { + if (!test_bit(root, dcp->foreign) && dcp->expected[root]) { + pte = dcp->expected[root][leaf]; + if (pte) + phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); + } + goto out; + } /* * An external display DART's registers only inside a use of them (its * power confirmed, a sleep waits for it): none, no lookup. @@ -1367,6 +1409,14 @@ static size_t apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, { size_t unmapped; + /* + * An internal display DART that its DCP powers down: no use of its + * registers is needed. Under the stream's lock __apple_dart_dcp_unmap() + * sees whether the DART is powered (apple_dart_host_lock()), and while it + * is not, only updates the page tables. + */ + if (dcp->dart->ps_prot) + return __apple_dart_dcp_unmap(dcp, iova, size, count, gather); /* * On an external display DART, only inside a use of its registers. * Returning 0 (nothing unmapped) is not safe for general DMA teardown: @@ -1420,8 +1470,13 @@ static size_t __apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, while (cleared < count && dcp->table[root][leaf + cleared]) WRITE_ONCE(dcp->table[root][leaf + cleared++], 0); dma_wmb(); - /* Strict SID0: invalidate before the IOVA or its pages can be reused. */ - if (cleared && dcp->sid == 0 && apple_dart_dcp_flush(dcp)) { + /* + * Strict SID0: invalidate before the IOVA or its pages can be reused. + * Not while its DCP has the DART powered down: nothing is cached then, + * and the power-up flushes before it enables the streams. + */ + if (cleared && dcp->sid == 0 && !READ_ONCE(dcp->dart->host_off) && + apple_dart_dcp_flush(dcp)) { /* A stale translation may survive: republish the leaves and keep the pages. */ for (size_t i = 0; i < cleared; i++) WRITE_ONCE(dcp->table[root][leaf + i], dcp->expected[root][leaf + i]); diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 5f512e18bfd50..b3c95c3067e45 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -963,6 +963,194 @@ static void apple_dart_test_strict_engines(struct kunit *test) "%s", lazy[i]); } +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +#define APPLE_DART_TEST_PS_SLICE 42 + +static void apple_dart_test_regmap_exit(void *map) +{ + regmap_exit(map); +} + +/* An internal display DART with one live stream and a PS_PROT bank in RAM. */ +static struct apple_dart *apple_dart_test_host_dart(struct kunit *test) +{ + static const struct regmap_config cfg = { + .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = APPLE_PS_PROT_SLICE(APPLE_PS_PROT_SLICES - 1), + }; + static atomic_t instance = ATOMIC_INIT(0); + struct apple_dart *dart; + char name[40]; + void *bank; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + /* Device names must be unique: a test may set up several. */ + snprintf(name, sizeof(name), "apple-dart-host-power-%d", atomic_inc_return(&instance)); + dart->dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + /* Up to the exception registers at 0x4000, which a 0x0203 DART reads. */ + dart->regs = (__force void __iomem *)kunit_kzalloc(test, APPLE_DART_TEST_REGS_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 2; + dart->pgsize = SZ_16K; + dart->version = 0x0203; + dart->power_gated = true; + spin_lock_init(&dart->lock); + spin_lock_init(&dart->pm_gate_lock); + init_waitqueue_head(&dart->pm_wq); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + bank = kunit_kzalloc(test, APPLE_PS_PROT_SLICE(APPLE_PS_PROT_SLICES), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bank); + dart->ps_prot = regmap_init_mmio(NULL, (__force void __iomem *)bank, &cfg); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->ps_prot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, apple_dart_test_regmap_exit, + dart->ps_prot), 0); + dart->ps_prot_slice = APPLE_DART_TEST_PS_SLICE; + regmap_write(dart->ps_prot, APPLE_PS_PROT_ENABLE, 1); + regmap_write(dart->ps_prot, APPLE_PS_PROT_SLICE(APPLE_DART_TEST_PS_SLICE), 0x12); + + writel(1 << 16, dart->regs + DART_PARAMS2); /* one port */ + writel(DART_T8110_TCR_TRANSLATE_ENABLE, dart->regs + DART_TCR(dart, 1)); + writel(dart->hw->ttbr_valid | 0x1234, dart->regs + DART_TTBR(dart, 1, 0)); + writel(DART_T8110_PROTECT_TTBR_TCR, dart->regs + DART_T8110_PROTECT); + writel(DART_T8110_PROTECT_TTBR_TCR, dart->regs + DART_T8110_PROTECT_LOCK); + return dart; +} + +static u32 apple_dart_test_slice(struct apple_dart *dart) +{ + u32 value = 0; + + regmap_read(dart->ps_prot, APPLE_PS_PROT_SLICE(dart->ps_prot_slice), &value); + return value; +} + +static void apple_dart_test_host_power_cycle(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->commands_gated)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_POWERDOWN); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x2U); + /* Its registers are not used until power-up. */ + KUNIT_EXPECT_FALSE(test, apple_dart_pm_get(dart)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EALREADY); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_on(dart), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->commands_gated)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_V2); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + KUNIT_EXPECT_TRUE(test, apple_dart_pm_get(dart)); + apple_dart_pm_put(dart); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_on(dart), -EALREADY); +} + +static void apple_dart_test_host_power_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + /* Power came back with a reset DART: not rebuilt, not used. */ + writel(0, dart->regs + DART_TTBR(dart, 1, 0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_on(dart), -EIO); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_FALSE(test, apple_dart_pm_get(dart)); +} + +static void apple_dart_test_host_power_refused(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + regmap_write(dart->ps_prot, APPLE_PS_PROT_LOCK, 1); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EPERM); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + + dart = apple_dart_test_host_dart(test); + /* A stream that stays enabled: the flush and the slice are left alone. */ + writel(BIT(1), dart->regs + DART_T8110_ENABLE_STREAMS); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EIO); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_DISABLE_STREAMS), BIT(1)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + + /* An outstanding transaction that does not drain in time. */ + dart = apple_dart_test_host_dart(test); + writel(1, dart->regs + DART_T8110_OUTSTANDING(0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + + /* A pending fault is not waited for. */ + dart = apple_dart_test_host_dart(test); + writel(1, dart->regs + DART_T8110_EXCEPTION(0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EIO); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); +} + +/* + * Generic SID0 DMA while the DCP has the DART powered down: translations, + * flushes and unmaps use the page tables only, and the power-up flushes. + */ +static void apple_dart_test_host_power_tables(struct kunit *test) +{ + const u64 iova = DCP_DART_ALIAS + 3 * DCP_DART_SLICE + 5 * DCP_DART_PAGE; + const unsigned int root = (iova >> 25) & 2047, leaf = (iova >> 14) & 2047; + const u64 pte = (0x12340000ULL >> 4) | 1 | (0xfffULL << 40); + struct apple_dart *dart = apple_dart_test_host_dart(test); + struct iommu_iotlb_gather gather; + struct apple_dart_domain *domain; + struct apple_dart_dcp *dcp; + + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, domain); + dcp->dart = dart; + spin_lock_init(&dcp->lock); + dcp->root = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + dcp->table[root] = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + dcp->expected[root] = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp->root); + KUNIT_ASSERT_NOT_NULL(test, dcp->table[root]); + KUNIT_ASSERT_NOT_NULL(test, dcp->expected[root]); + dcp->root[root] = 0x5670001; + dcp->entry[root] = dcp->root[root]; + dcp->table[root][leaf] = pte; + dcp->expected[root][leaf] = pte; + dart->frontend[0] = dcp; + domain->dcp = dcp; + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + writel(0, dart->regs + DART_T8110_TLB_CMD); /* Any later command shows here. */ + + KUNIT_EXPECT_EQ(test, apple_dart_dcp_translate(dcp, iova + 0x123), + dcp_dart_pte_phys(pte) | 0x123); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_domain_flush(domain), 0); + iommu_iotlb_gather_init(&gather); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_unmap(dcp, iova, DCP_DART_PAGE, 1, &gather), + (size_t)DCP_DART_PAGE); + KUNIT_EXPECT_EQ(test, dcp->table[root][leaf], 0ULL); + KUNIT_EXPECT_EQ(test, dcp->expected[root][leaf], 0ULL); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_translate(dcp, iova), (phys_addr_t)0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->failed)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_on(dart), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_V2); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->host_off)); +} +#endif + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_strict_engines), KUNIT_CASE(apple_dart_test_iommu_specifiers), @@ -989,6 +1177,12 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_reset_retry), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) + KUNIT_CASE(apple_dart_test_host_power_cycle), + KUNIT_CASE(apple_dart_test_host_power_lost), + KUNIT_CASE(apple_dart_test_host_power_refused), + KUNIT_CASE(apple_dart_test_host_power_tables), +#endif {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ed6a33e187fc0..60c14b261c358 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -190,6 +192,27 @@ #define DART_T8110_ENABLE_STREAMS 0xc00 #define DART_T8110_DISABLE_STREAMS 0xc20 +/* Per-port outstanding transactions; the port counts are in PARAMS2. */ +#define DART_T8110_OUTSTANDING(port) (0x230 + 4 * (port)) +#define DART_T8110_PARAMS2_READ_PORTS GENMASK(18, 16) +#define DART_T8110_PARAMS2_WRITE_PORTS GENMASK(22, 20) +/* How long the power-down and power-up wait for transactions to drain. */ +#define DART_T8110_IDLE_TIMEOUT_US 1000 +/* TLB command of the power-down path: flush everything, also stream table state. */ +#define DART_T8110_TLB_CMD_POWERDOWN (DART_T8110_TLB_CMD_V2 | DART_T8110_TLB_CMD_STT) + +/* + * PS_PROT ("clock protection") bank: one slice per protected DART. An + * enabled slice (bit 4) guards a power-state register of the DART's power + * domain; the native DART power-down clears bit 4 after its streams are + * disabled and flushed, and power-up sets it again. + */ +#define APPLE_PS_PROT_ENABLE 0x20 +#define APPLE_PS_PROT_LOCK 0x24 +#define APPLE_PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) +#define APPLE_PS_PROT_SLICES 64 +#define APPLE_PS_PROT_SLICE_ON BIT(4) + #define DART_T8110_TCR 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_ENABLE BIT(7) @@ -326,6 +349,17 @@ struct apple_dart { * with its power confirmed (apple_dart_pm_get()). */ bool power_gated; + /* + * Internal display DART: powered down by its DCP (from + * apple_dart_dcp_power_off() until apple_dart_dcp_power_on(); changed + * only under apple_dart_host_lock()), with what the power-up checks + * and restores. + */ + bool host_off; + u32 off_streams, off_protect, off_lock; + /* Its clock-protection slice in the PS_PROT bank, if it has one. */ + struct regmap *ps_prot; + u32 ps_prot_slice, ps_prot_saved; /* * Power-gated display DART: its power confirmation across * sleep, under pm_gate_lock (a spinlock: only these fields, never @@ -2036,15 +2070,23 @@ static int __apple_dart_dcp_domain_flush(struct apple_dart_domain *domain); /* * Borrowed DCP domains have no io-pgtable; their flush ignores the range. * On an external display DART only with its power confirmed: -EAGAIN - * (nothing retired) from sleep until its next confirmed power-up. + * (nothing retired) from sleep until its next confirmed power-up. An + * internal display DART that its DCP has powered down needs none: nothing + * is cached, and its power-up flushes every update made before it (they + * hold the stream's lock, which the power-up takes; apple_dart_host_lock()). + * A power-up that completed meanwhile confirmed the power in the same step. */ static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) { struct apple_dart *dart = domain->dcp->dart; int ret; - if (!apple_dart_pm_get(dart)) - return -EAGAIN; + if (!apple_dart_pm_get(dart)) { + if (READ_ONCE(dart->host_off)) + return 0; + if (!dart->ps_prot || !apple_dart_pm_get(dart)) + return -EAGAIN; + } ret = __apple_dart_dcp_domain_flush(domain); apple_dart_pm_put(dart); return ret; @@ -4269,6 +4311,18 @@ static int apple_dart_probe(struct platform_device *pdev) dart->preserve_display = j713_display || j713_dcpext || j713_dispext; dart->dispext = j713_dispext; dart->power_gated = j713_dispext; + if (j713_display && of_property_present(dev->of_node, "apple,clock-protection")) { + /* Its DCP can power it down: see apple_dart_dcp_power_off(). */ + dart->ps_prot = syscon_regmap_lookup_by_phandle_args(dev->of_node, + "apple,clock-protection", + 1, &dart->ps_prot_slice); + if (IS_ERR(dart->ps_prot)) + return dev_err_probe(dev, PTR_ERR(dart->ps_prot), + "clock-protection bank\n"); + if (dart->ps_prot_slice >= APPLE_PS_PROT_SLICES) + return -EINVAL; + dart->power_gated = true; + } spin_lock_init(&dart->pm_gate_lock); init_waitqueue_head(&dart->pm_wq); if (j713_dcpext) @@ -5188,6 +5242,9 @@ static __maybe_unused int apple_dart_resume(struct device *dev) /* No reset or restore: see apple_dart_suspend(). */ if (dart->cold_ane) return 0; + /* Powered down by its DCP, which restores it at power-up. */ + if (READ_ONCE(dart->host_off)) + return 0; /* * An adopted external display DART may be off now, its display with it, * and nothing confirms otherwise: no register of it is read or written @@ -5278,6 +5335,259 @@ static const struct of_device_id apple_dart_of_match[] = { MODULE_DEVICE_TABLE(of, apple_dart_of_match); #if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +/* The internal display DART of a DCP consumer (its firmware or display stream). */ +static struct apple_dart *apple_dart_dcp_consumer(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dd; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return NULL; + dd = to_dart_domain(domain); + if (!dd->dcp || dd->dcp->owner != dev || dd->dcp->dart->dispext || + !dd->dcp->dart->ps_prot) + return NULL; + return dd->dcp->dart; +} + +static int apple_dart_tlb_command(struct apple_dart *dart, u32 command) +{ + u32 status; + + writel(command, dart->regs + DART_T8110_TLB_CMD); + return readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); +} + +/* + * Nothing in flight: no fault (-EIO), and no TLB command or outstanding + * transaction (-EBUSY, which can drain). + */ +static int apple_dart_idle(struct apple_dart *dart) +{ + u32 params = readl(dart->regs + DART_PARAMS2); + unsigned int ports = max(FIELD_GET(DART_T8110_PARAMS2_READ_PORTS, params), + FIELD_GET(DART_T8110_PARAMS2_WRITE_PORTS, params)); + + if (!ports || ports > 4 || apple_dart_fault_pending(dart)) + return -EIO; + if (readl(dart->regs + DART_T8110_TLB_CMD) & DART_T8110_TLB_CMD_BUSY) + return -EBUSY; + for (unsigned int port = 0; port < ports; port++) + if (readl(dart->regs + DART_T8110_OUTSTANDING(port))) + return -EBUSY; + return 0; +} + +/* Idle, giving transactions and a command just issued time to drain. */ +static int apple_dart_wait_idle(struct apple_dart *dart) +{ + int ret, idle; + + ret = read_poll_timeout_atomic(apple_dart_idle, idle, idle != -EBUSY, 1, + DART_T8110_IDLE_TIMEOUT_US, false, dart); + return ret ?: idle; +} + +/* Its PS_PROT bank enabled and not locked: its slice can be switched. */ +static int apple_dart_ps_prot_usable(struct apple_dart *dart) +{ + u32 enable, lock; + int ret; + + ret = regmap_read(dart->ps_prot, APPLE_PS_PROT_ENABLE, &enable) ?: + regmap_read(dart->ps_prot, APPLE_PS_PROT_LOCK, &lock); + if (!ret && (!(enable & BIT(0)) || (lock & BIT(0)))) + ret = -EPERM; + return ret; +} + +static int apple_dart_ps_prot_set(struct apple_dart *dart, u32 value) +{ + unsigned int reg = APPLE_PS_PROT_SLICE(dart->ps_prot_slice), now; + int ret; + + ret = regmap_write(dart->ps_prot, reg, value); + if (!ret) + ret = regmap_read(dart->ps_prot, reg, &now); + return ret ?: now == value ? 0 : -EIO; +} + +/* + * Whether apple_dart_dcp_power_off() can run for @dev's DART, changing + * nothing: its DCP asks before the first step of its power-down. + */ +int apple_dart_dcp_power_check(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + struct apple_dart_dcp *streams[3]; + + if (!dart) + return -ENODEV; + if (READ_ONCE(dart->host_off)) + return -EALREADY; + streams[0] = dart->dcp; + streams[1] = dart->frontend[0]; + streams[2] = dart->frontend[1]; + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i] && READ_ONCE(streams[i]->failed)) + return -EIO; + return apple_dart_ps_prot_usable(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_check); + +/* + * J713 internal display DART, before its DCP powers the display domains + * down (its DCP CPU stopped, the DART's masters idle): as the native DART + * power-down does, disable its enabled streams, flush with the power-down + * TLB command and clear its clock-protection slice. From here until + * apple_dart_dcp_power_on() its registers are not used: its commands are + * gated, its generic DMA streams update their page tables in memory only + * (apple_dart_host_lock()) and system sleep leaves it alone. On failure its + * borrowed streams are retired. + * @dev is a consumer: the DCP (firmware DART) or its PIODMA (display DART). + */ +static int apple_dart_host_power_off(struct apple_dart *dart) +{ + unsigned long flags; + u32 enabled; + const char *why; + int ret; + + if (READ_ONCE(dart->host_off)) + return -EALREADY; + /* + * Page-table updates stay in memory from now on; then no use of its + * registers either, and the uses in flight are waited for. + */ + apple_dart_host_lock(dart, &flags); + WRITE_ONCE(dart->host_off, true); + apple_dart_host_unlock(dart, flags); + apple_dart_pm_invalidate(dart, false); + apple_dart_quiesce_commands(dart->dev); + + why = "not idle"; + ret = apple_dart_wait_idle(dart); + if (ret) + goto fail; + why = "clock protection"; + ret = apple_dart_ps_prot_usable(dart) ?: + regmap_read(dart->ps_prot, APPLE_PS_PROT_SLICE(dart->ps_prot_slice), + &dart->ps_prot_saved); + if (ret) + goto fail; + + /* What power-up checks: the translations and their lock, and the streams. */ + apple_dart_save_state(dart); + dart->off_protect = readl(dart->regs + DART_T8110_PROTECT); + dart->off_lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); + dart->off_streams = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + + why = "stream disable"; + writel(dart->off_streams, dart->regs + DART_T8110_DISABLE_STREAMS); + enabled = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + ret = enabled ? -EIO : 0; + if (ret) + goto fail; + why = "power-down flush"; + ret = apple_dart_tlb_command(dart, DART_T8110_TLB_CMD_POWERDOWN) ?: + apple_dart_wait_idle(dart); + if (ret) + goto fail; + if (dart->ps_prot_saved & APPLE_PS_PROT_SLICE_ON) { + why = "clock protection off"; + ret = apple_dart_ps_prot_set(dart, dart->ps_prot_saved & ~APPLE_PS_PROT_SLICE_ON); + if (ret) + goto fail; + } + dev_dbg(dart->dev, "powered down: streams %08x, clock protection %08x\n", + dart->off_streams, dart->ps_prot_saved); + return 0; +fail: + apple_dart_fail_borrowed(dart, "power-down: %s (%d)", why, ret); + return ret; +} + +int apple_dart_dcp_power_off(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + + return dart ? apple_dart_host_power_off(dart) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_off); + +/* + * After its DCP powered the display domains up again, before the DCP CPU + * runs: restore the clock-protection slice, check that the DART kept its + * translations and their lock (it is not rebuilt), flush, and enable the + * streams that were enabled at power-down. Its registers are then usable + * again. On failure its borrowed streams are retired. + */ +static int apple_dart_host_power_on(struct apple_dart *dart) +{ + unsigned long flags; + const char *why; + int ret; + + if (!READ_ONCE(dart->host_off)) + return -EALREADY; + why = "clock protection on"; + ret = apple_dart_ps_prot_set(dart, dart->ps_prot_saved); + if (ret) + goto fail; + why = "state lost"; + ret = -EIO; + if (readl(dart->regs + DART_T8110_ENABLE_STREAMS) || + readl(dart->regs + DART_T8110_PROTECT) != dart->off_protect || + readl(dart->regs + DART_T8110_PROTECT_LOCK) != dart->off_lock || + !apple_dart_state_retained(dart)) + goto fail; + why = "commands"; + ret = apple_dart_resume_commands(dart->dev); + if (ret) + goto fail; + + /* + * The flush, the streams and the confirmed power are one step for the + * streams' page-table updates (apple_dart_host_lock()): the flush covers + * every update made while the DART was down, and every later update + * finds it powered. + */ + apple_dart_host_lock(dart, &flags); + why = "flush"; + ret = apple_dart_wait_idle(dart) ?: + apple_dart_tlb_command(dart, DART_T8110_TLB_CMD_V2); + if (!ret) { + why = "stream enable"; + writel(dart->off_streams, dart->regs + DART_T8110_ENABLE_STREAMS); + if (readl(dart->regs + DART_T8110_ENABLE_STREAMS) != dart->off_streams) + ret = -EIO; + } + if (!ret) { + WRITE_ONCE(dart->host_off, false); + spin_lock(&dart->pm_gate_lock); + dart->pm_unverified = false; + spin_unlock(&dart->pm_gate_lock); + } + apple_dart_host_unlock(dart, flags); + if (ret) + goto fail; + dev_dbg(dart->dev, "powered up: state kept, streams %08x\n", dart->off_streams); + return 0; +fail: + apple_dart_fail_borrowed(dart, "power-up: %s (%d)", why, ret); + return ret; +} + +int apple_dart_dcp_power_on(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + + return dart ? apple_dart_host_power_on(dart) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_on); + /* * J713 external DCP: whether an adopted external display DART kept its * state, its power on (the caller's to know): the TTBR/TCR lock (PROTECT diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index ce81d6ec5bcd3..10d05a7cc042b 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -38,6 +38,15 @@ int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid); void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u64 *dva); /* An adopted external display DART's state after its power-up; reads only. */ int apple_dart_dcp_dispext_state(struct device_node *consumer); +/* + * Internal display DART of a DCP consumer (the DCP or its PIODMA), around a + * power-down of the display domains by the DCP: streams off, flushed and + * clock protection cleared before; state checked and streams back after. + * The check says, changing nothing, whether the power-off can run now. + */ +int apple_dart_dcp_power_check(struct device *dev); +int apple_dart_dcp_power_off(struct device *dev); +int apple_dart_dcp_power_on(struct device *dev); /* * Remappable host windows in SID0 of an external display DART: one @@ -56,6 +65,9 @@ void *apple_dart_dcp_scanout_alloc(struct device *dev, size_t size, dma_addr_t * #else static inline int apple_dart_dcp_check(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_check(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_off(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_on(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_dcp_check_buffer(struct device *dev, dma_addr_t iova, size_t size) { return -EOPNOTSUPP; From 41e83511551d6a18b36c93bd91eeaf59fbf2f4e8 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:50:08 +0100 Subject: [PATCH 247/392] dt-bindings: power: apple,pmgr-pwrstate: consumer-switched states The J713 display coprocessor's CPU, frontend and system power states are firmware-owned: the OS keeps them on and never touches their registers. macOS powers them off while the display is asleep, after stopping the coprocessor, and on again before restarting it, with a retention workaround on the CPU state. Add apple,consumer-switched for a firmware-owned state that its consumer may power off and on through the PMGR driver, and apple,retention-workaround for the PMGR registers to set while it is off: triplets of offset, mask and value. A chain of such states powers off child first and on parent first. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- .../bindings/power/apple,pmgr-pwrstate.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d9cf0b9129926..8281b883bd2ac 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -28,6 +28,19 @@ allOf: apple,force-disable: false apple,force-reset: false apple,externally-clocked: false + apple,consumer-switched: false + - if: + required: + - apple,consumer-switched + then: + required: + - apple,firmware-owned + - if: + required: + - apple,retention-workaround + then: + required: + - apple,consumer-switched description: | Apple SoCs include PMGR blocks responsible for power management, @@ -132,6 +145,32 @@ properties: the power-state register. type: boolean + apple,consumer-switched: + description: + With apple,firmware-owned, the consumer of this domain may power it + off and on again around stopping its coprocessor, through the PMGR + driver, which then writes the state register. The OS otherwise treats + the domain as firmware-owned and always on. A chain of such domains + (this one and its consumer-switched parents) is powered off child first + and on parent first. + type: boolean + + apple,retention-workaround: + description: + PMGR registers to set while this consumer-switched domain is off, and + to restore afterwards, as triplets of the offset in the PMGR register + block, a mask and the masked value while the domain is off. They are + set after automatic power management of the domain is disabled and + before it powers off, and restored after it powers on and before + automatic power management is enabled again. + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: register offset + - description: mask + - description: value while the domain is off + maxItems: 4 + apple,inherited-on: description: | On T8132, the boot loader hands over an active domain whose state From ca61e0808c44088f4d17b2c6ea4af685fa4b0303 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:50:08 +0100 Subject: [PATCH 248/392] pmdomain: apple: pmgr-pwrstate: let a consumer switch firmware-owned states Add apple_pmgr_ps_consumer_off() and apple_pmgr_ps_consumer_on() for the chain of apple,consumer-switched power states that starts at a consumer's power domain: off child first, on parent first. genpd still sees the states as firmware-owned and always on; the consumer has stopped its coprocessor before power-off and restarts it after power-on. Each state follows the native sequence. Power-off requires the state fully on and not in reset, disables automatic power management while staying on (when it was enabled), sets the apple,retention-workaround registers and then power gates, clearing the history bits. Power-on powers on with automatic power management disabled, gives the retention registers' workaround bits back the values they had (no other bit: power-off changed none) and then automatic power management as it was. Every write is read back or waited for; a failure leaves the state failed until reboot, and nothing is retried. A state that is not fully on is refused without being touched. apple_pmgr_ps_consumer_check() says, changing nothing, whether the chain can be powered off at all: consumer-switched, not failed and not off. Whether a state is fully on is known only at power-off: under automatic power management the J713 DCP CPU state reads level 0 while its CPU idles. KUnit: the J713 DCP CPU state reproduces the native power-off and power-on writes as captured on macOS, and keeps retention bits outside the workaround that change while it is off; a state without automatic power management or retention keeps its other bits; and a state that is not on, one whose actual state does not follow and a retention register that does not take its value are refused or failed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/pmdomain/apple/pmgr-pwrstate-test.c | 191 +++++++++++++ drivers/pmdomain/apple/pmgr-pwrstate.c | 283 ++++++++++++++++++++ include/linux/soc/apple/pmgr-pwrstate.h | 14 + 3 files changed, 488 insertions(+) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate-test.c b/drivers/pmdomain/apple/pmgr-pwrstate-test.c index 67698ccfc74aa..66fb065523dee 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate-test.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate-test.c @@ -357,6 +357,193 @@ static void pmgr_ps_power_on_no_reset_test(struct kunit *test) KUNIT_EXPECT_EQ(test, r->reset_writes, 0U); } +/* + * Consumer-switched states: a state word at offset 0 whose actual state + * follows the target (the history bits are set by a power-gate and cleared + * by writing them), retention registers as plain words, and a write log. + */ +struct pmgr_ps_test_bank { + u32 word[0x30000 / 4]; + bool stuck; /* the actual state no longer follows the target */ + bool ignore_retention; /* retention writes are dropped */ + unsigned int nr_log; + struct { u32 reg, val; } log[16]; +}; + +static int pmgr_ps_test_bank_read(void *ctx, unsigned int reg, unsigned int *val) +{ + struct pmgr_ps_test_bank *b = ctx; + + *val = b->word[reg / 4]; + return 0; +} + +static int pmgr_ps_test_bank_write(void *ctx, unsigned int reg, unsigned int val) +{ + struct pmgr_ps_test_bank *b = ctx; + u32 old = b->word[reg / 4], target, cleared; + + if (b->nr_log < ARRAY_SIZE(b->log)) { + b->log[b->nr_log].reg = reg; + b->log[b->nr_log++].val = val; + } + if (reg) { + if (!b->ignore_retention) + b->word[reg / 4] = val; + return 0; + } + target = FIELD_GET(APPLE_PMGR_PS_TARGET, val); + cleared = val & APPLE_PMGR_FLAGS; /* written as 1: cleared */ + val &= ~(APPLE_PMGR_PS_ACTUAL | APPLE_PMGR_FLAGS); + val |= old & APPLE_PMGR_FLAGS & ~cleared; + val |= FIELD_PREP(APPLE_PMGR_PS_ACTUAL, + b->stuck ? FIELD_GET(APPLE_PMGR_PS_ACTUAL, old) : target); + if (!b->stuck && target == APPLE_PMGR_PS_PWRGATE) + val |= APPLE_PMGR_FLAGS; + b->word[0] = val; + return 0; +} + +static const struct regmap_config pmgr_ps_test_bank_regmap = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0x30000 - 4, + .reg_read = pmgr_ps_test_bank_read, + .reg_write = pmgr_ps_test_bank_write, + .fast_io = true, +}; + +/* The J713 DCP CPU state: AUTO on, with its two retention workaround words. */ +static struct apple_pmgr_ps *pmgr_ps_test_consumer(struct kunit *test, u32 word, + struct pmgr_ps_test_bank **bank) +{ + static atomic_t instance = ATOMIC_INIT(0); + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps; + struct device *dev; + char name[32]; + + b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, b); + b->word[0] = word; + b->word[0x2504c / 4] = 0; + b->word[0x25070 / 4] = 1; + snprintf(name, sizeof(name), "apple-pmgr-consumer-%d", atomic_inc_return(&instance)); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + ps = kunit_kzalloc(test, sizeof(*ps), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ps); + ps->dev = dev; + ps->genpd.name = "disp-cpu"; + ps->firmware_owned = true; + ps->consumer_switched = true; + ps->nr_retention = 2; + ps->retention[0].offset = 0x2504c; + ps->retention[0].mask = 3; + ps->retention[0].value = 3; + ps->retention[1].offset = 0x25070; + ps->retention[1].mask = 1; + ps->retention[1].value = 0; + ps->regmap = devm_regmap_init(dev, NULL, b, &pmgr_ps_test_bank_regmap); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ps->regmap); + *bank = b; + return ps; +} + +static void pmgr_ps_test_expect_log(struct kunit *test, struct pmgr_ps_test_bank *b, + const u32 (*want)[2], unsigned int n) +{ + KUNIT_ASSERT_EQ(test, b->nr_log, n); + for (unsigned int i = 0; i < n; i++) { + KUNIT_EXPECT_EQ_MSG(test, b->log[i].reg, want[i][0], "write %u", i); + KUNIT_EXPECT_EQ_MSG(test, b->log[i].val, want[i][1], "write %u", i); + } + b->nr_log = 0; +} + +/* The native J713 DCP CPU writes, as captured on macOS, in order. */ +static void pmgr_ps_consumer_native_test(struct kunit *test) +{ + static const u32 off[][2] = { + { 0, 0x0f0000ff }, { 0x2504c, 3 }, { 0x25070, 0 }, { 0, 0x0f0003f0 }, + }; + static const u32 on[][2] = { + { 0, 0x0f00000f }, { 0x2504c, 0 }, { 0x25070, 1 }, { 0, 0x1f00000f }, + }; + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + pmgr_ps_test_expect_log(test, b, off, ARRAY_SIZE(off)); + KUNIT_EXPECT_EQ(test, b->word[0] & (APPLE_PMGR_PS_TARGET | APPLE_PMGR_PS_ACTUAL), 0U); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EALREADY); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + pmgr_ps_test_expect_log(test, b, on, ARRAY_SIZE(on)); + KUNIT_EXPECT_EQ(test, b->word[0] & ~APPLE_PMGR_FLAGS, 0x1f0000ffU); + KUNIT_EXPECT_EQ(test, b->word[0x2504c / 4], 0U); + KUNIT_EXPECT_EQ(test, b->word[0x25070 / 4], 1U); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EALREADY); +} + +/* Retention bits outside the workaround's mask that change while off are kept. */ +static void pmgr_ps_consumer_retention_mask_test(struct kunit *test) +{ + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + b->word[0x2504c / 4] |= 0x100; + b->word[0x25070 / 4] |= 0x10; + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + KUNIT_EXPECT_EQ(test, b->word[0x2504c / 4], 0x100U); + KUNIT_EXPECT_EQ(test, b->word[0x25070 / 4], 0x11U); +} + +/* FE and SYS: no AUTO and no retention; other bits are kept. */ +static void pmgr_ps_consumer_plain_test(struct kunit *test) +{ + static const u32 off[][2] = { { 0, 0x23f0 } }; + static const u32 on[][2] = { { 0, 0x200f } }; + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x20ff, &b); + + ps->nr_retention = 0; + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + pmgr_ps_test_expect_log(test, b, off, ARRAY_SIZE(off)); + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + pmgr_ps_test_expect_log(test, b, on, ARRAY_SIZE(on)); +} + +static void pmgr_ps_consumer_refused_test(struct kunit *test) +{ + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps; + + /* Not fully on: left alone, and not failed. */ + ps = pmgr_ps_test_consumer(test, 0x1f000044, &b); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EBUSY); + KUNIT_EXPECT_EQ(test, b->nr_log, 0U); + KUNIT_EXPECT_FALSE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EALREADY); + + /* The actual state does not follow: failed, and stays failed. */ + ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + b->stuck = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EIO); + + /* A retention register that does not take the value: failed before power-off. */ + ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + b->ignore_retention = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EIO); + KUNIT_EXPECT_TRUE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, b->word[0]), + APPLE_PMGR_PS_ACTIVE); +} + static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_test), KUNIT_CASE(pmgr_ps_power_off_undo_test), @@ -372,6 +559,10 @@ static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_latch_sleep_test), KUNIT_CASE(pmgr_ps_power_on_latch_sticky_test), KUNIT_CASE(pmgr_ps_power_on_no_reset_test), + KUNIT_CASE(pmgr_ps_consumer_native_test), + KUNIT_CASE(pmgr_ps_consumer_retention_mask_test), + KUNIT_CASE(pmgr_ps_consumer_plain_test), + KUNIT_CASE(pmgr_ps_consumer_refused_test), {} }; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 688842390a6a2..d6b0cc28b6c2c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -41,6 +41,14 @@ #define APPLE_PMGR_PS_PWRGATE 0x0 #define APPLE_PMGR_PS_SET_TIMEOUT 100 +#define APPLE_PMGR_PS_CONSUMER_TIMEOUT 1000 +#define APPLE_PMGR_PS_MAX_RETENTION 4 +#define APPLE_PMGR_PS_MAX_CHAIN 4 + +#define APPLE_PMGR_PS_LEVELS (APPLE_PMGR_PS_TARGET | APPLE_PMGR_PS_ACTUAL) +/* Target and actual state both on (or both power gated). */ +#define APPLE_PMGR_PS_ALL_ON (FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_ACTIVE) | \ + FIELD_PREP(APPLE_PMGR_PS_ACTUAL, APPLE_PMGR_PS_ACTIVE)) #define APPLE_PMGR_PS_PARENT_TIMEOUT 10000 #define APPLE_PMGR_RESET_TIME 1 #define APPLE_PMGR_T8132_RESET_TIME 10 @@ -67,6 +75,20 @@ struct apple_pmgr_ps { bool strict_state; bool wait_parent; u32 reset_time_us; + /* + * apple,consumer-switched: firmware-owned, but powered off and on by its + * consumer (apple_pmgr_ps_consumer_off()). The state word before + * power-off, and the retention workaround registers with their values + * before power-off. A failed transition leaves it failed until reboot. + */ + bool consumer_switched; + bool consumer_off; + bool consumer_failed; + u32 consumer_saved; + unsigned int nr_retention; + struct { + u32 offset, mask, value, saved; + } retention[APPLE_PMGR_PS_MAX_RETENTION]; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -649,6 +671,28 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) * vote and never read, reset or change the hardware state. */ ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); + ps->consumer_switched = of_property_read_bool(node, "apple,consumer-switched"); + if (ps->consumer_switched) { + int n = of_property_count_u32_elems(node, "apple,retention-workaround"); + + if (!ps->firmware_owned || + (n > 0 && (n % 3 || n / 3 > APPLE_PMGR_PS_MAX_RETENTION))) + return dev_err_probe(dev, -EINVAL, "bad consumer-switched policy\n"); + for (int i = 0; n > 0 && i < n / 3; i++) { + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i, + &ps->retention[i].offset); + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i + 1, + &ps->retention[i].mask); + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i + 2, + &ps->retention[i].value); + if (ps->retention[i].value & ~ps->retention[i].mask) + return dev_err_probe(dev, -EINVAL, "bad retention workaround\n"); + } + ps->nr_retention = n > 0 ? n / 3 : 0; + } else if (of_property_present(node, "apple,retention-workaround")) { + return dev_err_probe(dev, -EINVAL, + "retention workaround needs a consumer switch\n"); + } ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); /* * Only T8132 hands over inherited-on domains with this contract. Older @@ -794,6 +838,245 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return ret; } +static struct platform_driver apple_pmgr_ps_driver; + +/* Wait for the masked state word to read @want; @cur gets the last value. */ +static int apple_pmgr_ps_wait(struct apple_pmgr_ps *ps, u32 mask, u32 want, u32 *cur) +{ + return regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + (*cur & mask) == want, 1, + APPLE_PMGR_PS_CONSUMER_TIMEOUT); +} + +/* Write a retention register and read it back. */ +static int apple_pmgr_ps_retention_set(struct apple_pmgr_ps *ps, unsigned int i, u32 value) +{ + u32 now; + int ret; + + ret = regmap_write(ps->regmap, ps->retention[i].offset, value); + if (!ret) + ret = regmap_read(ps->regmap, ps->retention[i].offset, &now); + return ret ?: now == value ? 0 : -EIO; +} + +/* Fully on (target and actual), and not in reset, busy or disabled. */ +static bool apple_pmgr_ps_consumer_ready(u32 reg) +{ + return (reg & APPLE_PMGR_PS_LEVELS) == APPLE_PMGR_PS_ALL_ON && + !(reg & (APPLE_PMGR_RESET | APPLE_PMGR_PS_RESET | APPLE_PMGR_BUSY | + APPLE_PMGR_DEV_DISABLE)); +} + +/* + * The native power-off of a consumer-switched state: from fully on, disable + * automatic power management while staying on, set the retention workaround, + * then power off (clearing the history bits) and wait for it. + */ +static int apple_pmgr_ps_consumer_state_off(struct apple_pmgr_ps *ps) +{ + u32 reg, cur; + int ret; + + if (ps->consumer_failed) + return -EIO; + if (ps->consumer_off) + return -EALREADY; + ret = regmap_read(ps->regmap, ps->offset, ®); + if (ret) + goto fail; + if (!apple_pmgr_ps_consumer_ready(reg)) + return -EBUSY; /* Not touched: not a failure of this state. */ + for (unsigned int i = 0; i < ps->nr_retention; i++) { + ret = regmap_read(ps->regmap, ps->retention[i].offset, &ps->retention[i].saved); + if (ret) + goto fail; + } + ps->consumer_saved = reg; + reg &= ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + if (ps->consumer_saved & APPLE_PMGR_AUTO_ENABLE) { + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_PS_LEVELS, + APPLE_PMGR_PS_ALL_ON, &cur); + if (ret) + goto fail; + } + for (unsigned int i = 0; i < ps->nr_retention; i++) { + ret = apple_pmgr_ps_retention_set(ps, i, (ps->retention[i].saved & + ~ps->retention[i].mask) | + ps->retention[i].value); + if (ret) + goto fail; + } + ps->consumer_off = true; + reg = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_FLAGS | + FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_PWRGATE); + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_PS_LEVELS | APPLE_PMGR_AUTO_ENABLE, 0, + &cur); + if (ret) + goto fail; + dev_dbg(ps->dev, "PS %s: consumer power-off: %#x -> %#x\n", ps->genpd.name, + ps->consumer_saved, cur); + return 0; +fail: + ps->consumer_failed = true; + dev_err(ps->dev, "PS %s: consumer power-off failed: %d\n", ps->genpd.name, ret); + return ret; +} + +/* + * The native power-on: power on with automatic power management disabled, + * give the retention registers' workaround bits back the values they had + * (only those: power-off changed no other), then automatic power management + * as it was. + */ +static int apple_pmgr_ps_consumer_state_on(struct apple_pmgr_ps *ps) +{ + u32 reg, cur, now; + int ret; + + if (ps->consumer_failed) + return -EIO; + if (!ps->consumer_off) + return -EALREADY; + reg = (ps->consumer_saved & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | + APPLE_PMGR_BUSY | APPLE_PMGR_PS_ACTUAL)) | + FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_ACTIVE); + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_PS_LEVELS, APPLE_PMGR_PS_ALL_ON, &cur); + if (ret) + goto fail; + for (unsigned int i = 0; i < ps->nr_retention; i++) { + u32 mask = ps->retention[i].mask; + + ret = regmap_read(ps->regmap, ps->retention[i].offset, &now); + if (!ret) + ret = apple_pmgr_ps_retention_set(ps, i, (now & ~mask) | + (ps->retention[i].saved & mask)); + if (ret) + goto fail; + } + if (ps->consumer_saved & APPLE_PMGR_AUTO_ENABLE) { + ret = regmap_write(ps->regmap, ps->offset, reg | APPLE_PMGR_AUTO_ENABLE); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_AUTO_ENABLE, APPLE_PMGR_AUTO_ENABLE, + &cur); + if (ret) + goto fail; + } + ps->consumer_off = false; + dev_dbg(ps->dev, "PS %s: consumer power-on: %#x\n", ps->genpd.name, cur); + return 0; +fail: + ps->consumer_failed = true; + dev_err(ps->dev, "PS %s: consumer power-on failed: %d\n", ps->genpd.name, ret); + return ret; +} + +/* The consumer-switched states from @consumer's domain up, child first. */ +static int apple_pmgr_ps_consumer_chain(struct device *consumer, + struct apple_pmgr_ps **chain) +{ + struct device_node *np = of_parse_phandle(consumer->of_node, "power-domains", 0); + int n = 0; + + while (np) { + struct platform_device *pdev = of_find_device_by_node(np); + struct apple_pmgr_ps *ps = NULL; + struct device_node *parent; + + if (pdev && pdev->dev.driver == &apple_pmgr_ps_driver.driver) + ps = platform_get_drvdata(pdev); + if (pdev) + put_device(&pdev->dev); + if (!ps || !ps->consumer_switched || n == APPLE_PMGR_PS_MAX_CHAIN) { + of_node_put(np); + break; + } + chain[n++] = ps; + parent = of_parse_phandle(np, "power-domains", 0); + of_node_put(np); + np = parent; + } + return n ?: -ENODEV; +} + +static int apple_pmgr_ps_consumer_switch(struct apple_pmgr_ps *ps, bool on) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = on ? apple_pmgr_ps_consumer_state_on(ps) : apple_pmgr_ps_consumer_state_off(ps); + spin_unlock_irqrestore(&ps->genpd.slock, flags); + return ret; +} + +/* + * Whether apple_pmgr_ps_consumer_off() can run for @consumer, changing + * nothing: the whole chain consumer-switched, not failed and not off. Not + * whether each state is fully on: under automatic power management the + * J713 DCP CPU state reads level 0 while its CPU idles, and is fully on + * again once the CPU has been stopped, which the power-off then requires. + */ +int apple_pmgr_ps_consumer_check(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret = 0; + + if (n < 0) + return n; + for (int i = 0; i < n && !ret; i++) { + struct apple_pmgr_ps *ps = chain[i]; + unsigned long flags; + + spin_lock_irqsave(&ps->genpd.slock, flags); + if (ps->consumer_failed) + ret = -EIO; + else if (ps->consumer_off) + ret = -EALREADY; + spin_unlock_irqrestore(&ps->genpd.slock, flags); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_check); + +int apple_pmgr_ps_consumer_off(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret; + + if (n < 0) + return n; + for (int i = 0; i < n; i++) { + ret = apple_pmgr_ps_consumer_switch(chain[i], false); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_off); + +int apple_pmgr_ps_consumer_on(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret; + + if (n < 0) + return n; + for (int i = n - 1; i >= 0; i--) { + ret = apple_pmgr_ps_consumer_switch(chain[i], true); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_on); + static const struct of_device_id apple_pmgr_ps_of_match[] = { { .compatible = "apple,t8103-pmgr-pwrstate" }, { .compatible = "apple,pmgr-pwrstate" }, diff --git a/include/linux/soc/apple/pmgr-pwrstate.h b/include/linux/soc/apple/pmgr-pwrstate.h index 63311cfd7d979..e09a7ee3e031c 100644 --- a/include/linux/soc/apple/pmgr-pwrstate.h +++ b/include/linux/soc/apple/pmgr-pwrstate.h @@ -4,12 +4,26 @@ #include +struct device; struct device_node; #if IS_ENABLED(CONFIG_APPLE_PMGR_PWRSTATE) int apple_pmgr_ps_cycle(struct device_node *np); +/* + * Power off (child first) or on (parent first) the chain of + * apple,consumer-switched power states that starts at @consumer's power + * domain. The consumer has stopped its coprocessor before power-off and + * starts it again after power-on. The check says, changing nothing, whether + * the power-off can run now. + */ +int apple_pmgr_ps_consumer_check(struct device *consumer); +int apple_pmgr_ps_consumer_off(struct device *consumer); +int apple_pmgr_ps_consumer_on(struct device *consumer); #else static inline int apple_pmgr_ps_cycle(struct device_node *np) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_check(struct device *consumer) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_off(struct device *consumer) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_on(struct device *consumer) { return -EOPNOTSUPP; } #endif #endif From 71f705417c1be929d9d787655f316d50eb8078a4 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:56:12 +0100 Subject: [PATCH 249/392] dt-bindings: power: apple,pmc: consumer-switched interface enables A display pipe's BWR and boost interface enables are described as a power domain between the pipe's SYS and FE power states. The J713 internal display's power states are firmware-owned and on from boot, while the boot loader leaves only its boost enable set, so its enables cannot be such a domain; macOS clears them before powering the pipe down and sets them again after powering it up. Add apple,consumer-switched for an interface node that is not a power domain: the display coprocessor that refers to it clears its bits before it powers the pipe down and restores them after it powers the pipe up. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- .../devicetree/bindings/power/apple,pmc.yaml | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 85b54bc769bfa..24f77b80d6495 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -214,12 +214,28 @@ patternProperties: maximum: 31 description: Bit of the pipe in the boost interface enable word. + apple,consumer-switched: + type: boolean + description: | + Not a power domain. The pipe's power is firmware-owned, and the + display coprocessor that refers to this node (apple,pmc-interface) + clears the bits before it powers the pipe down and gives them back + the values they had after it powers the pipe up again. + required: - - '#power-domain-cells' - - power-domains - apple,bwr-bit - apple,boost-bit + oneOf: + - required: + - '#power-domain-cells' + - power-domains + - required: + - apple,consumer-switched + properties: + '#power-domain-cells': false + power-domains: false + '^performance-controller@[0-9a-f]+$': type: object description: The host performance controller's voter agent. From 7d88b7239ebb269ee1cea176f3344a30960c299c Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 17:56:12 +0100 Subject: [PATCH 250/392] pmdomain: apple: pmc: let a consumer switch a pipe's interface enables An interface node with apple,consumer-switched keeps its BWR and boost bits reserved but is not a genpd domain. Add apple_pmc_interface_consumer_off() and apple_pmc_interface_consumer_on() for the consumer that names it in apple,pmc-interface: off saves both bits and clears the boost, then the BWR enable, as a pipe's power-down does; on gives both back the values they had. If the BWR enable does not take the change on the way off, the boost enable gets its old value back. Nothing is retried, and the bits of other pipes are left alone. As for the other interface domains, the bits return to their values at probe when the PMC goes away. apple_pmc_interface_consumer_check() says, changing nothing, whether the bits can be cleared now. KUnit: the internal display's bits with the boost enable only (as Linux finds them) and with both set (as macOS keeps them), next to dispext0's boost bit, and a BWR enable that does not clear. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/pmdomain/apple/pmc-test.c | 60 ++++++++++++++ drivers/pmdomain/apple/pmc.c | 125 +++++++++++++++++++++++++++++- include/linux/soc/apple/pmc.h | 23 ++++++ 3 files changed, 207 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index d563a178dc0e8..4e7f19d47317a 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -355,6 +355,64 @@ static void apple_pmc_test_interface_stuck(struct kunit *test) test->priv = NULL; } +/* + * The internal display's consumer-switched interface (BWR and boost bit 0): + * off clears its bits only, on gives back the values they had, whether the + * boot loader left both set or only the boost enable. + */ +static void apple_pmc_test_interface_consumer(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 0, .boost_bit = 0, + .consumer_switched = true, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + + pmc->hw = hw; + /* As Linux finds them: the boost enable only, next to dispext0's. */ + writel(BIT(0) | BIT(1), pmc->rails + hw->boost_enable); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), -EALREADY); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(1)); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), -EALREADY); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); + + /* As macOS keeps them while the display is on: both set. */ + writel(BIT(0), pmc->rails + hw->bwr_enable); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(1)); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), BIT(0)); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); +} + +/* A BWR enable that does not clear: the boost enable goes back, still on. */ +static void apple_pmc_test_interface_consumer_stuck(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 0, .boost_bit = 0, + .consumer_switched = true, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + struct apple_pmc_test_stuck stuck = { .offset = hw->bwr_enable, .value = BIT(0) }; + + pmc->hw = hw; + writel(BIT(0), pmc->rails + hw->boost_enable); + test->priv = &stuck; + kunit_activate_static_stub(test, apple_pmc_interface_read, apple_pmc_test_stuck_read); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), -EIO); + KUNIT_EXPECT_FALSE(test, dom.consumer_off); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->boost_enable), BIT(0)); + kunit_deactivate_static_stub(test, apple_pmc_interface_read); + test->priv = NULL; +} + static void apple_pmc_test_genpd_remove(void *genpd) { pm_genpd_remove(genpd); @@ -497,6 +555,8 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_floor_pending_timeout), KUNIT_CASE(apple_pmc_test_interface), KUNIT_CASE(apple_pmc_test_interface_stuck), + KUNIT_CASE(apple_pmc_test_interface_consumer), + KUNIT_CASE(apple_pmc_test_interface_consumer_stuck), KUNIT_CASE(apple_pmc_test_interface_genpd), KUNIT_CASE(apple_pmc_test_interface_boot_state), {} diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 3765e9a775ff7..40392c9d46603 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -152,6 +152,10 @@ struct apple_pmc { * @interface: the domain is a display pipe's BWR and boost interface enables * @bwr_bit: with @interface, the bit in the BWR interface enable word * @boost_bit: with @interface, the bit in the boost interface enable word + * @consumer_switched: with @interface, not a genpd domain: its consumer clears + * and restores its bits (apple_pmc_interface_consumer_off()) + * @consumer_off: with @consumer_switched, cleared by its consumer, and the + * bits it had then (@consumer_bwr, @consumer_boost); protected by pmc->lock */ struct apple_pmc_domain { struct generic_pm_domain genpd; @@ -168,6 +172,10 @@ struct apple_pmc_domain { bool interface; unsigned int bwr_bit; unsigned int boost_bit; + bool consumer_switched; + bool consumer_off; + bool consumer_bwr; + bool consumer_boost; bool provider; }; @@ -766,6 +774,107 @@ static int apple_pmc_interface_set(struct apple_pmc_domain *dom, bool on) return ret; } +/* + * A consumer-switched interface: off clears the boost, then the BWR enable, + * as a pipe's power-down does, after saving both; on gives both back what + * they had then. If the BWR enable does not take the change on the way off, + * the boost enable gets its old value back. Nothing is retried. + */ +static int apple_pmc_interface_consumer_set(struct apple_pmc_domain *dom, bool on) +{ + struct apple_pmc *pmc = dom->pmc; + const struct apple_pmc_hw *hw = pmc->hw; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&pmc->lock, flags); + if (on != dom->consumer_off) { + ret = -EALREADY; + } else if (on) { + ret = apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, + dom->consumer_boost); + if (!ret) + ret = apple_pmc_interface_update(pmc, hw->bwr_enable, dom->bwr_bit, + dom->consumer_bwr); + if (!ret) + dom->consumer_off = false; + } else { + dom->consumer_bwr = apple_pmc_interface_read(pmc, hw->bwr_enable) & + BIT(dom->bwr_bit); + dom->consumer_boost = apple_pmc_interface_read(pmc, hw->boost_enable) & + BIT(dom->boost_bit); + ret = apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, false); + if (!ret) { + ret = apple_pmc_interface_update(pmc, hw->bwr_enable, dom->bwr_bit, false); + if (ret) + apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, + dom->consumer_boost); + } + if (!ret) + dom->consumer_off = true; + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static struct platform_driver apple_pmc_driver; + +/* The consumer-switched interface named by @consumer's apple,pmc-interface. */ +static struct apple_pmc_domain *apple_pmc_consumer_interface(struct device *consumer) +{ + struct device_node *np = of_parse_phandle(consumer->of_node, "apple,pmc-interface", 0); + struct apple_pmc_domain *dom, *found = NULL; + struct platform_device *pdev; + struct apple_pmc *pmc = NULL; + + if (!np) + return NULL; + pdev = of_find_device_by_node(np->parent); + if (pdev && pdev->dev.driver == &apple_pmc_driver.driver) + pmc = platform_get_drvdata(pdev); + if (pmc) + list_for_each_entry(dom, &pmc->domains, list) + if (dom->np == np && dom->consumer_switched) + found = dom; + if (pdev) + put_device(&pdev->dev); + of_node_put(np); + return found; +} + +/* Whether apple_pmc_interface_consumer_off() can run for @consumer now, changing nothing. */ +int apple_pmc_interface_consumer_check(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + unsigned long flags; + int ret; + + if (!dom) + return -ENODEV; + raw_spin_lock_irqsave(&dom->pmc->lock, flags); + ret = dom->consumer_off ? -EALREADY : 0; + raw_spin_unlock_irqrestore(&dom->pmc->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_check); + +int apple_pmc_interface_consumer_off(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + + return dom ? apple_pmc_interface_consumer_set(dom, false) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_off); + +int apple_pmc_interface_consumer_on(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + + return dom ? apple_pmc_interface_consumer_set(dom, true) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_on); + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); @@ -794,6 +903,8 @@ static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) static void apple_pmc_domain_free(struct apple_pmc_domain *dom) { + if (dom->consumer_switched) + goto free; if (dom->provider) of_genpd_del_provider(dom->np); if (pm_genpd_remove(&dom->genpd)) { @@ -802,6 +913,7 @@ static void apple_pmc_domain_free(struct apple_pmc_domain *dom) dom->genpd.name); return; } +free: of_node_put(dom->np); kfree(dom->genpd.name); kfree(dom); @@ -1011,6 +1123,15 @@ static int apple_pmc_interface_add(struct apple_pmc *pmc, struct device_node *np return -ENOMEM; } + /* Its consumer clears and restores its bits: no genpd domain. */ + dom->consumer_switched = of_property_read_bool(np, "apple,consumer-switched"); + if (dom->consumer_switched) { + list_add_tail(&dom->list, &pmc->domains); + pmc->bwr_bits |= BIT(bwr); + pmc->boost_bits |= BIT(boost); + return 0; + } + ret = apple_pmc_interface_init(pmc, dom); if (ret) { of_node_put(dom->np); @@ -1280,7 +1401,9 @@ static int apple_pmc_probe(struct platform_device *pdev) } for_each_available_child_of_node(np, child) { - if (!of_property_present(child, "#power-domain-cells")) + /* A consumer-switched interface is no power domain, but still ours. */ + if (!of_property_present(child, "#power-domain-cells") && + !of_property_read_bool(child, "apple,consumer-switched")) continue; if (of_property_present(child, "apple,rail")) ret = apple_pmc_floor_add(pmc, child); diff --git a/include/linux/soc/apple/pmc.h b/include/linux/soc/apple/pmc.h index be391b83d9752..da0635015f52d 100644 --- a/include/linux/soc/apple/pmc.h +++ b/include/linux/soc/apple/pmc.h @@ -26,7 +26,30 @@ bool apple_pmc_voter_enabled(struct device *pmc, unsigned int agent, u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent); int apple_pmc_floor_update(struct device *pmc, unsigned int agent, u32 mask, u32 value); +/* + * Clear, and later restore, the display pipe interface enables named by + * @consumer's apple,pmc-interface (an apple,consumer-switched interface). + * The check says, changing nothing, whether they can be cleared now. + */ +int apple_pmc_interface_consumer_check(struct device *consumer); +int apple_pmc_interface_consumer_off(struct device *consumer); +int apple_pmc_interface_consumer_on(struct device *consumer); #else +static inline int apple_pmc_interface_consumer_check(struct device *consumer) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmc_interface_consumer_off(struct device *consumer) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmc_interface_consumer_on(struct device *consumer) +{ + return -EOPNOTSUPP; +} + static inline unsigned int apple_pmc_rail_count(struct device *pmc) { return 0; From 6fde3b9bb23974eb2b86e723aa74edbbb0fedf24 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:02:08 +0100 Subject: [PATCH 251/392] soc: apple: pmp-display: let the J713 DCP release its DISP vote The J713 DCP takes its PMP DISP vote (power command 0x0f, logical device 8, state 1) when it activates, and the vote is held until reboot. macOS releases it after stopping the DCP and before powering the display domains down, and takes it again before powering them up. Add apple_pmp_j713_display_put(): the same power command with state 0, sent through the PMP owner's serialized request path (which consumes the exact reply), after which apple_pmp_j713_display_get() takes the vote again. The Rust owner forwards the release word to apple_pmp_set_device_power(), which already handles state 0. One request is in flight at a time, as before. A release that was not transmitted leaves the vote held; one whose outcome is unknown leaves it neither held nor available again, so the display cannot come back up on a vote that may be gone. Other accepted votes are kept. apple_pmp_j713_display_check_release() says, changing nothing, whether the vote can be released now and taken again later: held, nothing in flight, and admission open, as the later get is a new request and a failed untagged request closes admission. The DCP asks before it powers down. Host test: release and retake, a release with nothing held, one while another request is pending, one that was not sent and one without a reply, and the release check, also with admission closed by another request. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/soc/apple/pmp-display.c | 132 +++++++++++++++++- drivers/soc/apple/pmp.rs | 2 + include/linux/soc/apple/pmp-display.h | 9 +- .../drivers/gpu/apple-dcp/m4-error-paths.py | 45 +++++- 4 files changed, 178 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/pmp-display.c b/drivers/soc/apple/pmp-display.c index fed0b13bdfe83..b32e155ec09f0 100644 --- a/drivers/soc/apple/pmp-display.c +++ b/drivers/soc/apple/pmp-display.c @@ -24,6 +24,8 @@ /* AGX: command 0x0f, logical device 5, state 1. The reply carries no tag. */ #define PMP_AGX_REQUEST 0x0020f50100000000ULL #define PMP_DISP_REQUEST 0x0020f80100000000ULL +/* DISP, state 0: released while its DCP has powered the display domains down. */ +#define PMP_DISP_RELEASE 0x0020f80000000000ULL #define PMP_POWER_REPLY 0x0021000000000000ULL #define PMP_ENDPOINT 0x20 @@ -41,6 +43,8 @@ struct pmp_power_vote { /* A failed PMP owner invalidates every accepted vote as well. */ int provider_error; enum pmp_power_kind pending; + /* The pending request releases its vote rather than taking it. */ + bool releasing; bool held[PMP_POWER_COUNT]; }; @@ -48,7 +52,11 @@ static void pmp_vote_fail(struct pmp_power_vote *s, int error) { if (!s->error) s->error = error < 0 ? error : -EIO; + /* A release with an unknown outcome no longer counts as held. */ + if (s->pending && s->releasing) + s->held[s->pending] = false; s->pending = PMP_POWER_NONE; + s->releasing = false; } /* The Rust owner latches every ambiguous transaction as a provider failure. @@ -57,10 +65,12 @@ static void pmp_vote_fail(struct pmp_power_vote *s, int error) */ static void pmp_vote_send_failed(struct pmp_power_vote *s) { - if (s->provider_error) + if (s->provider_error) { pmp_vote_fail(s, s->provider_error); - else + } else { s->pending = PMP_POWER_NONE; + s->releasing = false; + } } /* One means send the request, zero means the same retained vote is held. */ @@ -80,6 +90,52 @@ static int pmp_vote_begin(struct pmp_power_vote *s, enum pmp_power_kind kind) return 1; } +/* One means send the release, zero means the vote is not held. */ +static int pmp_vote_begin_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (kind <= PMP_POWER_NONE || kind >= PMP_POWER_COUNT) + return -EINVAL; + if (s->provider_error) + return s->provider_error; + if (!s->held[kind]) + return 0; + if (s->error) + return s->error; + if (s->pending) + return -EBUSY; + s->pending = kind; + s->releasing = true; + return 1; +} + +/* + * Whether a vote can be released now and taken again later, changing + * nothing: held, no request in flight, and admission open (a failed untagged + * request closes it, and the later get is a new request). + */ +static int pmp_vote_check_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (kind <= PMP_POWER_NONE || kind >= PMP_POWER_COUNT) + return -EINVAL; + if (s->provider_error) + return s->provider_error; + if (!s->held[kind]) + return -ENODEV; + if (s->error) + return s->error; + return s->pending ? -EBUSY : 0; +} + +/* Released, or the outcome is unknown and admission is closed. */ +static int pmp_vote_finish_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (s->provider_error) + return s->provider_error; + if (s->pending == kind && s->releasing) + pmp_vote_fail(s, -ETIMEDOUT); + return s->held[kind] || s->error ? s->error ?: -EIO : 0; +} + /* Accepted votes are retained even if a different untagged request fails. */ static int pmp_vote_check(struct pmp_power_vote *s, enum pmp_power_kind kind) { @@ -110,8 +166,9 @@ static void pmp_vote_reply(struct pmp_power_vote *s, u8 endpoint, u64 word) pmp_vote_fail(s, -EPROTO); return; } - s->held[s->pending] = true; + s->held[s->pending] = !s->releasing; s->pending = PMP_POWER_NONE; + s->releasing = false; } static inline bool j713_disp_table(const void *data, size_t size) @@ -338,7 +395,7 @@ static int pmp_power_prepare(struct device *consumer, enum pmp_power_kind kind) return ret; } -static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind) +static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind, bool release) { struct apple_pmp_display *p = pmp_power_consumer(consumer, kind); unsigned long flags; @@ -347,7 +404,7 @@ static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind) if (IS_ERR(p)) return PTR_ERR(p); spin_lock_irqsave(&p->lock, flags); - ret = pmp_vote_check(&p->vote, kind); + ret = release ? pmp_vote_check_release(&p->vote, kind) : pmp_vote_check(&p->vote, kind); spin_unlock_irqrestore(&p->lock, flags); return ret; } @@ -405,6 +462,55 @@ static int pmp_power_get(struct device *consumer, enum pmp_power_kind kind) return ret; } +/* + * Release the DISP vote while the consumer has powered the display domains + * down; apple_pmp_j713_display_get() takes it again before power-up. An + * ambiguous release closes admission: the vote is then neither held nor + * taken again. + */ +static int pmp_power_put(struct device *consumer, enum pmp_power_kind kind) +{ + struct apple_pmp_display *p = pmp_power_consumer(consumer, kind); + unsigned long flags; + int ret; + + if (IS_ERR(p)) + return PTR_ERR(p); + if (kind != PMP_POWER_DISP) + return -EINVAL; + device_lock(p->dev); + mutex_lock(&p->request); + if (p->dev->driver != p->driver || !dev_get_drvdata(p->dev)) { + ret = -ENODEV; + apple_pmp_display_fail(p, ret); + goto out; + } + spin_lock_irqsave(&p->lock, flags); + reinit_completion(&p->reply); + ret = pmp_vote_begin_release(&p->vote, kind); + spin_unlock_irqrestore(&p->lock, flags); + if (ret <= 0) + goto out; + ret = p->send(p->cookie, PMP_DISP_RELEASE); + if (ret) { + spin_lock_irqsave(&p->lock, flags); + pmp_vote_send_failed(&p->vote); + spin_unlock_irqrestore(&p->lock, flags); + goto out; + } + spin_lock_irqsave(&p->lock, flags); + pmp_vote_reply(&p->vote, PMP_ENDPOINT, PMP_POWER_REPLY); + ret = pmp_vote_finish_release(&p->vote, kind); + spin_unlock_irqrestore(&p->lock, flags); + if (!ret) + dev_dbg(p->dev, "Retained DISP vote released: request=%016llx\n", + PMP_DISP_RELEASE); +out: + mutex_unlock(&p->request); + device_unlock(p->dev); + return ret; +} + /* Probe creates the retained supplier links; get/check never change owners. */ int apple_pmp_j713_display_prepare(struct device *consumer) { @@ -420,13 +526,19 @@ EXPORT_SYMBOL_GPL(apple_pmp_j713_gpu_prepare); int apple_pmp_j713_display_check(struct device *consumer) { - return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP) : -ENODEV; + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP, false) : -ENODEV; } EXPORT_SYMBOL_GPL(apple_pmp_j713_display_check); +int apple_pmp_j713_display_check_release(struct device *consumer) +{ + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP, true) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmp_j713_display_check_release); + int apple_pmp_j713_gpu_check(struct device *consumer) { - return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_AGX) : -ENODEV; + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_AGX, false) : -ENODEV; } EXPORT_SYMBOL_GPL(apple_pmp_j713_gpu_check); @@ -436,6 +548,12 @@ int apple_pmp_j713_display_get(struct device *consumer) } EXPORT_SYMBOL_GPL(apple_pmp_j713_display_get); +int apple_pmp_j713_display_put(struct device *consumer) +{ + return apple_pmp_j713_board() ? pmp_power_put(consumer, PMP_POWER_DISP) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmp_j713_display_put); + int apple_pmp_j713_gpu_get(struct device *consumer) { return apple_pmp_j713_board() ? pmp_power_get(consumer, PMP_POWER_AGX) : -ENODEV; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 2212509efd82c..39cc8821e0ba4 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -487,6 +487,8 @@ unsafe extern "C" fn display_send(cookie: *const kernel::ffi::c_void, word: u64) match word { 0x0020f50100000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) }, 0x0020f80100000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 8, 1) }, + // DISP released while its DCP has powered the display domains down. + 0x0020f80000000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 8, 0) }, _ => EINVAL.to_errno(), } } diff --git a/include/linux/soc/apple/pmp-display.h b/include/linux/soc/apple/pmp-display.h index 7db16206d4e8a..e65effb337aa8 100644 --- a/include/linux/soc/apple/pmp-display.h +++ b/include/linux/soc/apple/pmp-display.h @@ -31,12 +31,17 @@ void apple_pmp_display_fail(struct apple_pmp_display *display, int error); struct apple_pmp_display *apple_pmp_display_bound_owner(struct device *dev); /* J713 votes share the existing PMP owner's untagged request stream. - * Prepare is probe-only; accepted DISP/AGX votes and supplier links persist - * until reboot. + * Prepare is probe-only; supplier links persist until reboot, and so do + * accepted votes, except that the DCP releases its DISP vote while it has + * powered the display domains down (put) and takes it again (get). Before + * that it asks, changing nothing, whether the vote can be released now and + * taken again later (check_release). */ int apple_pmp_j713_display_prepare(struct device *consumer); int apple_pmp_j713_display_get(struct device *consumer); +int apple_pmp_j713_display_put(struct device *consumer); int apple_pmp_j713_display_check(struct device *consumer); +int apple_pmp_j713_display_check_release(struct device *consumer); int apple_pmp_j713_gpu_prepare(struct device *consumer); int apple_pmp_j713_gpu_get(struct device *consumer); int apple_pmp_j713_gpu_check(struct device *consumer); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py index e57b10a738947..027eec2ecbc1a 100755 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py @@ -59,7 +59,8 @@ def main(): votes += definition(pmp, 'enum pmp_power_kind {') + ';\n' votes += definition(pmp, 'struct pmp_power_vote {') + ';\n' for function in ('static void pmp_vote_fail(', 'static void pmp_vote_send_failed(', - 'static int pmp_vote_begin(', + 'static int pmp_vote_begin(', 'static int pmp_vote_begin_release(', + 'static int pmp_vote_check_release(', 'static int pmp_vote_finish_release(', 'static int pmp_vote_check(', 'static int pmp_vote_finish(', 'static void pmp_vote_reply('): votes += definition(pmp, function) + '\n' @@ -111,6 +112,48 @@ def main(): pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); assert(pmp_vote_check(&s, PMP_POWER_DISP) == 0); assert(pmp_vote_begin(&s, PMP_POWER_AGX) == -EPROTO); + + /* The DCP releases DISP while its domains are down and takes it again. */ + s = (struct pmp_power_vote){0}; + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ENODEV); /* not held */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 0); /* not held */ + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ENODEV); /* still pending */ + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == 0); + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_finish_release(&s, PMP_POWER_DISP) == 0); + assert(!s.held[PMP_POWER_DISP] && !s.releasing && !s.error); + assert(pmp_vote_check(&s, PMP_POWER_DISP) == -ENODEV); + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_finish(&s, PMP_POWER_DISP) == 0); + /* One request at a time. */ + assert(pmp_vote_begin(&s, PMP_POWER_AGX) == 1); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -EBUSY); + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == -EBUSY); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(s.held[PMP_POWER_AGX] && s.held[PMP_POWER_DISP]); + /* Not transmitted: the vote is still held. */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + pmp_vote_send_failed(&s); + assert(s.held[PMP_POWER_DISP] && !s.releasing && !s.error); + /* No reply: neither held nor taken again; AGX stays held. */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + assert(pmp_vote_finish_release(&s, PMP_POWER_DISP) == -ETIMEDOUT); + assert(!s.held[PMP_POWER_DISP] && !s.releasing); + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == -ETIMEDOUT); + assert(pmp_vote_check(&s, PMP_POWER_AGX) == 0); + + /* Admission closed by another request: DISP stays held, but is not released. */ + s = (struct pmp_power_vote){0}; + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_begin(&s, PMP_POWER_AGX) == 1); + assert(pmp_vote_finish(&s, PMP_POWER_AGX) == -ETIMEDOUT); + assert(pmp_vote_check(&s, PMP_POWER_DISP) == 0); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ETIMEDOUT); return 0; } ''' From c94d8ff4cf90d79156b925cecacc5002946d7059 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:37:47 +0100 Subject: [PATCH 252/392] soc: apple: rtkit: wake a co-processor with its retained session apple_rtkit_idle() leaves a co-processor's firmware, endpoints and shared buffers in place, but waking it again goes through INIT and starts the system endpoints anew. The J713 DCP is to stop its CPU after the idle handshake, power its domains down and start the CPU again later. Its firmware then says HELLO by itself, and macOS answers the endpoint map with the endpoints that are still running, without INIT and without starting any endpoint. Add apple_rtkit_wake_retained(). From the idle state, with RX held by the caller since the idle, it restores the mailbox (apple_mbox_restore(), as its power domain may have been off), resumes RX and lets the caller start the CPU. While it runs, the endpoint map is answered with the endpoints started in this session, which RTKit now records, and nothing is started again; a started endpoint missing from the map fails the wake. A co-processor that first reports QUIESCED gets one ON request, any other IOP state before ON fails the wake. The usual boot wait then requires IOP ON and requests AP ON. Nothing is allocated, freed or reinitialized. A signal to the caller does not interrupt the wake's waits: the CPU is running by then, and nothing would undo that. An unexpected IOP state ends whichever wait the wake is in, the map or IOP ON after it. A last map part cannot echo endpoint 32 * base, as bit 0 of the reply is also MORE: such a wake fails. The DCP's map has two parts, with its endpoint 0 in the first. Also accept management message 0xc, which some co-processors send once idle after the IOP IDLE request, instead of warning about it. apple_mbox_restore() sets the mailbox-level enables of a started mailbox again, as apple_mbox_start() does. Host test: the retained endpoint map replies for the DCP's map, a missing started endpoint, a last part that cannot be echoed, QUIESCED before ON, an unexpected state before and after the map, the unchanged acknowledgment outside a retained wake, and which waits a signal can interrupt. The OSLog test's RTKit stub gains the started endpoint bitmap. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/soc/apple/mailbox.c | 60 +++++--- drivers/soc/apple/rtkit-internal.h | 9 ++ drivers/soc/apple/rtkit.c | 147 +++++++++++++++++++- include/linux/soc/apple/mailbox.h | 1 + include/linux/soc/apple/rtkit.h | 8 ++ tools/asahi/rtkit-oslog-tests.py | 2 +- tools/asahi/rtkit-retained-tests.py | 203 ++++++++++++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 8 files changed, 410 insertions(+), 21 deletions(-) create mode 100755 tools/asahi/rtkit-retained-tests.py diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index c13c03f9d92ab..21ae3a51f9966 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -957,25 +957,10 @@ static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) cancel_delayed_work_sync(&priv->rx_poll_work); } -int apple_mbox_start(struct apple_mbox *mbox) +/* The mailbox-level enables, which the mailbox's power domain does not keep. */ +static void apple_mbox_enable_hw(struct apple_mbox *mbox) { u32 control; - int ret = 0; - - mutex_lock(&mbox->lifecycle_lock); - if (mbox->removing) { - ret = -ENODEV; - goto out_unlock; - } - if (mbox->active) - goto out_unlock; - - ret = pm_runtime_resume_and_get(mbox->dev); - if (ret) { - dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); - apple_mbox_clear_resume_error(mbox); - goto out_unlock; - } if (mbox->hw->ap_initializes_mailboxes) { /* The primary T8140 outbox requires its enable bit before use. */ @@ -996,6 +981,28 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->hw->irq_bit_send_empty, mbox->regs + mbox->hw->irq_enable); } +} + +int apple_mbox_start(struct apple_mbox *mbox) +{ + int ret = 0; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + if (mbox->active) + goto out_unlock; + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) { + dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } + + apple_mbox_enable_hw(mbox); if (apple_mbox_has_rx_poller(mbox)) { WRITE_ONCE(mbox->active, true); @@ -1036,6 +1043,25 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * The mailbox's power domain was off while it stayed started (its consumer + * powered its co-processor down and up again around it): set the + * mailbox-level enables again before the co-processor runs. + */ +int apple_mbox_restore(struct apple_mbox *mbox) +{ + int ret = 0; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing || !mbox->active) + ret = -ENODEV; + else + apple_mbox_enable_hw(mbox); + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL(apple_mbox_restore); + void apple_mbox_hold_rx(struct apple_mbox *mbox) { disable_irq(mbox->irq_recv_not_empty); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 2ad4d96bec768..577f68fad529c 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -65,6 +65,15 @@ struct apple_rtkit { struct work_struct sync_ack_work; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + /* Endpoints started in this session; kept by a retained wake. */ + DECLARE_BITMAP(started, APPLE_RTKIT_MAX_ENDPOINTS); + /* + * apple_rtkit_wake_retained() in progress: the endpoint map is answered + * with the started endpoints and nothing is started again; a QUIESCED + * report gets one ON request. + */ + bool retained_wake; + bool retained_on_sent; struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem coreanalytics_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3b821415d54d8..b6b27a713442a 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -45,6 +45,8 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, + /* Sent by some co-processors once idle after an IOP IDLE request. */ + APPLE_RTKIT_MGMT_IDLE_NOTIFY = 0xc, }; #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0) @@ -229,6 +231,55 @@ static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg) complete_all(&rtk->epmap_completion); } +/* + * Endpoint map of a retained wake: the endpoints and their buffers are still + * set up, so answer with the started endpoints of this part of the map, as + * the native retained wake does, and start nothing. A started endpoint that + * the firmware no longer offers fails the wake. + */ +static void apple_rtkit_management_rx_epmap_retained(struct apple_rtkit *rtk, u64 msg, + u32 base, unsigned long bitmap) +{ + unsigned long started = 0; + u64 reply; + int i; + + for (i = 0; i < 32 && 32 * base + i < APPLE_RTKIT_MAX_ENDPOINTS; i++) + if (test_bit(32 * base + i, rtk->started)) + started |= BIT(i); + if (started & ~bitmap) { + dev_err(rtk->dev, "RTKit: retained endpoints %#lx missing from map %#lx (base %u)\n", + started & ~bitmap, bitmap, base); + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + return; + } + /* + * Bit 0 of a reply is also MORE: a last part cannot echo endpoint + * 32 * base there. The DCP's map has two parts, and its endpoint 0 + * (management) is in the first. + */ + if ((msg & APPLE_RTKIT_MGMT_EPMAP_LAST) && (started & APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE)) { + dev_err(rtk->dev, "RTKit: retained endpoint %u cannot be echoed in the last map part\n", + 32 * base); + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + return; + } + + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base) | started; + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; + else + reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); + + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } +} + static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) { int i, ep, ret; @@ -261,6 +312,11 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + if (rtk->retained_wake) { + apple_rtkit_management_rx_epmap_retained(rtk, msg, base, bitmap); + return; + } + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; @@ -330,6 +386,28 @@ static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk, rtk->iop_power_state, new_state); rtk->iop_power_state = new_state; + /* + * A retained wake: a co-processor may report QUIESCED when it starts + * running and wait for an ON request before its HELLO. Ask once, and + * wait for ON. + */ + if (rtk->retained_wake && new_state != APPLE_RTKIT_PWR_STATE_ON) { + if (new_state == APPLE_RTKIT_PWR_STATE_QUIESCED && !rtk->retained_on_sent) { + rtk->retained_on_sent = true; + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, + APPLE_RTKIT_PWR_STATE_ON)); + return; + } + dev_err(rtk->dev, "RTKit: unexpected IOP power state 0x%x on retained wake\n", + new_state); + /* Whichever wait the wake is in: the map, or IOP ON after it. */ + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + complete_all(&rtk->iop_pwr_ack_completion); + return; + } + complete_all(&rtk->iop_pwr_ack_completion); } @@ -362,6 +440,9 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK: apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; + case APPLE_RTKIT_MGMT_IDLE_NOTIFY: + dev_dbg(rtk->dev, "RTKit: co-processor idle\n"); + break; default: dev_warn( rtk->dev, @@ -1245,6 +1326,8 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) endpoint); ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); } + if (!ret) + set_bit(endpoint, rtk->started); return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); @@ -1423,6 +1506,7 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); + set_bit(APPLE_RTKIT_EP_MGMT, rtk->started); if (mbox_name) rtk->mbox = apple_mbox_get_byname(dev, mbox_name); @@ -1487,6 +1571,18 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); +/* + * A retained wake is not interrupted by a signal to its caller: it has + * already started the co-processor's CPU, and nothing would undo that. + */ +static long apple_rtkit_wait_timeout(struct apple_rtkit *rtk, struct completion *c, + unsigned long timeout) +{ + if (READ_ONCE(rtk->retained_wake)) + return wait_for_completion_timeout(c, timeout); + return wait_for_completion_interruptible_timeout(c, timeout); +} + /* * apple,sync-syslog-ack (AsahiLinux/linux#607): the mailbox can leave * messages queued without another interrupt during bootstrap, so poll it @@ -1507,8 +1603,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) if (ret < 0) return ret; - t = wait_for_completion_interruptible_timeout(c, - msecs_to_jiffies(10)); + t = apple_rtkit_wait_timeout(rtk, c, msecs_to_jiffies(10)); if (t < 0) return t; if (t > 0) @@ -1535,7 +1630,7 @@ static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = wait_for_completion_interruptible_timeout(c, timeout); + t = apple_rtkit_wait_timeout(rtk, c, timeout); if (t < 0) return t; else if (t == 0) @@ -1580,6 +1675,8 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); + bitmap_zero(rtk->started, APPLE_RTKIT_MAX_ENDPOINTS); + set_bit(APPLE_RTKIT_EP_MGMT, rtk->started); reinit_completion(&rtk->epmap_completion); reinit_completion(&rtk->coreanalytics_completion); @@ -1697,6 +1794,9 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) dev_err(rtk->dev, "RTKit: IOP power acknowledgment failed: %d\n", ret); return ret; } + /* A retained wake fails on any IOP state but ON (still 0 otherwise). */ + if (rtk->boot_result) + return rtk->boot_result; return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); @@ -1831,6 +1931,47 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +/* + * Wake a co-processor whose CPU was stopped after apple_rtkit_idle(), with + * its firmware, endpoints and shared buffers kept (and its power domain + * possibly off meanwhile). The caller holds RX since the idle; this restores + * the mailbox, resumes RX and calls @start, which starts the CPU. The + * firmware then says HELLO (possibly after reporting QUIESCED, which gets an + * ON request) and sends its endpoint map, answered with the started + * endpoints; no INIT is sent and nothing is started or allocated again. + * Then IOP ON is awaited and AP ON requested, as for any boot. + */ +int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), void *ctx) +{ + int ret; + + if (apple_rtkit_is_running(rtk) || rtk->crashed || rtk->shutting_down || + rtk->iop_power_state != APPLE_RTKIT_PWR_STATE_IDLE || !start) + return -EINVAL; + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + rtk->boot_result = -EINPROGRESS; + rtk->retained_on_sent = false; + WRITE_ONCE(rtk->retained_wake, true); + + ret = apple_mbox_restore(rtk->mbox); + if (ret) + goto out; + apple_rtkit_resume_rx(rtk); + ret = start(ctx); + if (ret) + goto out; + ret = apple_rtkit_boot_common(rtk, false); +out: + WRITE_ONCE(rtk->retained_wake, false); + if (ret) + dev_err(rtk->dev, "RTKit: retained wake failed: %d\n", ret); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_wake_retained); + void apple_rtkit_free(struct apple_rtkit *rtk) { apple_rtkit_stop_rx(rtk); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 7a07772cc5967..d7fc9dd4134f1 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,6 +100,7 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +int apple_mbox_restore(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); /* * Leave received messages in the hardware FIFO, without an interrupt, until diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 86cdb8426488d..ac646240f8587 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -234,6 +234,14 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); /* Wake helpers which require AP readiness before their IOP power acknowledgment. */ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk); +/* + * Wake a co-processor stopped after apple_rtkit_idle() with its firmware, + * endpoints and buffers kept: restore the mailbox, resume RX, call @start to + * start its CPU, then complete HELLO, the endpoint map (with the started + * endpoints, no INIT and no new endpoint start) and IOP/AP ON. + */ +int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), void *ctx); + /* * Shutdown the co-processor */ diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 160c163758d54..87c3654fb8328 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -49,7 +49,7 @@ def function(source, name): int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; unsigned iop_power_state, ap_power_state; - unsigned long endpoints[4]; + unsigned long endpoints[4], started[4]; char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; diff --git a/tools/asahi/rtkit-retained-tests.py b/tools/asahi/rtkit-retained-tests.py new file mode 100755 index 0000000000000..b79a9cd542bd2 --- /dev/null +++ b/tools/asahi/rtkit-retained-tests.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check the retained-wake endpoint map reply and IOP power acknowledgment.""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = 'drivers/soc/apple/rtkit.c' + + +def function(source, name): + match = re.search(r'^(?:static )?(?:void|int|long) ' + name + r'\(', source, re.M) + assert match, name + opening = source.index('{', match.end()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + '\n' + + +def constants(source): + out = [] + for line in source.splitlines(): + if re.match(r'#define APPLE_RTKIT_MGMT_(TYPE|EPMAP_\w+|PWR_STATE)\b', line): + out.append(line) + enum = re.search(r'enum \{\n\tAPPLE_RTKIT_MGMT_HELLO = 1,.*?\n\};', source, re.S) + states = re.search(r'enum \{\n\tAPPLE_RTKIT_PWR_STATE_OFF.*?\n\};', source, re.S) + assert enum and states + return '\n'.join(out) + '\n' + enum.group(0) + '\n' + states.group(0) + '\n' + + +PREFIX = r''' +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +#define BIT(n) (1UL << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define FIELD_GET(m,v) (((v) & (m)) >> __builtin_ctzll(m)) +#define FIELD_PREP(m,v) (((u64)(v) << __builtin_ctzll(m)) & (m)) +#define APPLE_RTKIT_MAX_ENDPOINTS 256 +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +struct apple_rtkit { + void *dev; + unsigned long started[APPLE_RTKIT_MAX_ENDPOINTS / 64]; + bool retained_wake, retained_on_sent; + int boot_result; + unsigned int iop_power_state; + int epmap_completion, iop_pwr_ack_completion; +}; +static bool test_bit(unsigned n, const unsigned long *b) { return b[n / 64] >> (n % 64) & 1; } +static void set_bit(unsigned n, unsigned long *b) { b[n / 64] |= 1UL << (n % 64); } +static void complete_all(int *c) { (*c)++; } +static unsigned sent; +static u8 sent_type; +static u64 sent_data; +static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 data) +{ + (void)rtk; sent++; sent_type = type; sent_data = data; return 0; +} +/* A signal is pending for every interruptible wait. */ +#define ERESTARTSYS 512 +#define READ_ONCE(x) (x) +struct completion { int unused; }; +static unsigned waited_uninterruptible, waited_interruptible; +static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long t) +{ + (void)c; waited_uninterruptible++; return t; +} +static long wait_for_completion_interruptible_timeout(struct completion *c, unsigned long t) +{ + (void)c; (void)t; waited_interruptible++; return -ERESTARTSYS; +} +''' + +MAIN = r''' +static struct apple_rtkit fresh(void) +{ + struct apple_rtkit r = { .retained_wake = true, .boot_result = -EINPROGRESS }; + + /* Management, syslog, crashlog, debug, ioreport, oslog, tracekit, DCP 0x37. */ + for (unsigned ep = 0; ep < 11; ep++) + if ((0x51dUL >> ep) & 1) + set_bit(ep, r.started); + set_bit(0x37, r.started); + sent = 0; + return r; +} + +int main(void) +{ + struct apple_rtkit r = fresh(); + u64 map0 = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 0) | 0x51d; + u64 map1 = APPLE_RTKIT_MGMT_EPMAP_LAST | FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 1) | + 0x00867f3f; + + /* Each part of the map: exactly the started endpoints, MORE until the last. */ + apple_rtkit_management_rx_epmap_retained(&r, map0, 0, 0x51d); + assert(sent == 1 && sent_type == APPLE_RTKIT_MGMT_EPMAP_REPLY); + assert(sent_data == (0x51d | APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE)); + assert(r.epmap_completion == 0 && r.boot_result == -EINPROGRESS); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(sent == 2 && sent_data == (APPLE_RTKIT_MGMT_EPMAP_LAST | + FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 1) | BIT(0x37 - 32))); + assert(r.epmap_completion == 1 && r.boot_result == 0); + + /* A started endpoint missing from the map fails the wake; nothing is sent. */ + r = fresh(); + apple_rtkit_management_rx_epmap_retained(&r, map1 & ~(u64)BIT(0x37 - 32), 1, + 0x00867f3f & ~BIT(0x37 - 32)); + assert(sent == 0 && r.boot_result == -EPROTO && r.epmap_completion == 1); + + /* QUIESCED gets one ON request, then ON completes. */ + r = fresh(); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 1 && sent_type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE); + assert(sent_data == APPLE_RTKIT_PWR_STATE_ON && r.iop_pwr_ack_completion == 0); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_ON); + assert(sent == 1 && r.iop_pwr_ack_completion == 1); + assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_ON); + + /* A last part cannot echo endpoint 32 * base: bit 0 is also MORE. */ + r = fresh(); + set_bit(32, r.started); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(sent == 0 && r.boot_result == -EPROTO && r.epmap_completion == 1); + + /* + * A second QUIESCED, or any other state, fails the wake, before the map + * or while the wake waits for IOP ON after it. + */ + r = fresh(); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 1 && r.boot_result == -EPROTO); + assert(r.epmap_completion == 1 && r.iop_pwr_ack_completion == 1); + r = fresh(); + apple_rtkit_management_rx_epmap_retained(&r, map0, 0, 0x51d); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(r.epmap_completion == 1 && r.boot_result == 0); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_IDLE); + assert(r.boot_result == -EPROTO && r.iop_pwr_ack_completion == 1); + + /* Outside a retained wake every acknowledgment completes, as before. */ + r = fresh(); + r.retained_wake = false; + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 0 && r.iop_pwr_ack_completion == 1); + assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_QUIESCED); + + /* A retained wake waits through a signal; other waits stay interruptible. */ + struct completion c; + r = fresh(); + assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); + assert(waited_uninterruptible == 1 && waited_interruptible == 0); + r.retained_wake = false; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); + assert(waited_uninterruptible == 1 && waited_interruptible == 1); + + puts("PASS retained endpoint map replies, IOP acknowledgments and waits"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, help='keep build and result files here') + parser.add_argument('--cc', default='cc') + args = parser.parse_args() + source = (ROOT / DRIVER).read_text() + code = PREFIX + constants(source) + code += function(source, 'apple_rtkit_management_rx_epmap_retained') + code += function(source, 'apple_rtkit_management_rx_iop_pwr_ack') + code += function(source, 'apple_rtkit_wait_timeout') + MAIN + + def run(out): + out.mkdir(parents=True, exist_ok=True) + cfile, exe = out / 'retained.c', out / 'retained' + cfile.write_text(code) + subprocess.run([args.cc, '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-fsanitize=address,undefined', + '-fno-sanitize-recover=all', str(cfile), '-o', str(exe)], check=True) + result = subprocess.run([str(exe)], capture_output=True, text=True) + print(result.stdout or result.stderr, end='') + return result.returncode + + if args.out: + return run(args.out) + with tempfile.TemporaryDirectory(prefix='rtkit-retained-') as out: + return run(Path(out)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index b878404b2d9fa..2da6ddc68cd13 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -157,6 +157,7 @@ def mutations(flag, names, base, expect=KILL): 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-oslog-tests.py': [(out(), PASS)] + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), + 'rtkit-retained-tests.py': [(out(), PASS)], 't6030-clock-tests.py': [(out(), PASS)], 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], From 82ed2d32df0911850f6e3cbe61344fd7f301da99 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:48:15 +0100 Subject: [PATCH 253/392] drm/apple: m4: power the internal DCP and its domains down while blanked On DPMS off the J713 internal display gets setPowerState(0) (A473), which stops its pipe, but the DCP keeps running and its CPU, frontend and system power domains stay on. macOS goes further for display sleep: it releases the AP link, idles RTKit, stops the DCP CPU and powers the three domains off, and on wake starts the CPU again on the retained firmware. Do the same once the display has been blanked for appledrm.power_gate_ms (0, the default, never), and before system sleep while blanked. First, changing nothing, ask every provider whether it can take part: both display DARTs, the PMP vote, the PMC interface and the PMGR states. If one cannot, the display just stays logically off. Then: A500, drain the callbacks, apple_rtkit_idle() and hold RX, stop the CPU (after its idle status), take in anything the firmware sent meanwhile (anything there fails the session, before a power-down), power down both display DARTs (streams off, power-down flush, clock protection off), release the PMP DISP vote, clear the PMC interface enables and power the consumer-switched CPU, frontend and system states off. The first frame afterwards reverses it before setPowerState(1), in macOS's order: domains, PMP vote, PMC enables, DARTs (state checked, streams back), then RTKit's retained wake, which restarts the CPU and gets HELLO, the endpoint map and IOP/AP ON without INIT, and the display preparation macOS sends (A414, A412). A frame cancels a power-down not yet started; a power-down under way finishes first. While the domains are down no register of the DCP, its mailbox or its DARTs is used: health checks, status and resume leave them alone, and resume leaves the display down until the next frame. Any failure fails the session, as other DCP failures do, but none refuses system sleep: the power-down is a saving, not a condition of sleep. Setting power_gate_ms to 0 also stops a power-down already scheduled. Only an internal J713 DCP whose node names its PMC interface (apple,pmc-interface) does this. debugfs iomfb_m4 shows the gate state, the number of power-downs and the last power-down and power-up times. On a J713 with power_gate_ms=3000, 204 power-downs and power-ups in one boot completed without an error, one of them started by a suspend 1.5 s after the blank: the display domains read back level 0 while off, stayed off across suspend-to-idle, and the display came back with a completed frame each time. Awake with DPMS off it draws about 15 mW less than setPowerState(0) alone (8 alternating pairs); one pair of suspend-to-idle runs differed by about 10 mW. It is a prerequisite for deeper SoC sleep with the display off. Host test: m4-dpms-retirement covers the power-down scheduling, its absence when not enabled, the power-up before setPowerState(1) and a power-up that fails; m4-power-gate the order of both sequences, a provider that refuses (nothing changes), a late firmware message (the session fails before any power-down) and power_gate_ms turned off after the blank. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- drivers/gpu/drm/apple/dcp.c | 4 +- drivers/gpu/drm/apple/iomfb_m4.c | 308 +++++++++++++++++- .../selftests/drivers/gpu/apple-dcp/m4.py | 194 +++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 1 + 4 files changed, 503 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f233c079997cc..d098c43dbe72b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -6683,7 +6683,9 @@ static int dcp_platform_suspend(struct device *dev) /* * T8132: setPowerState powers the display down; the coprocessor keeps - * running and its power domains stay on, as below. + * running and its power domains stay on, as below, unless the internal + * display is blanked with appledrm.power_gate_ms set: then the + * coprocessor and its domains are powered down too (iomfb_m4_suspend()). */ if (iomfb_is_m4(dcp)) { int ret = iomfb_m4_suspend(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_m4.c b/drivers/gpu/drm/apple/iomfb_m4.c index 48b7179c439e4..b6b301a20ab20 100644 --- a/drivers/gpu/drm/apple/iomfb_m4.c +++ b/drivers/gpu/drm/apple/iomfb_m4.c @@ -24,11 +24,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -39,6 +41,8 @@ #include #include #include +#include +#include #include "afk.h" #include "dcp.h" @@ -82,6 +86,32 @@ module_param_cb(idle_cache_ms, &idle_cache_ms_ops, &idle_cache_ms, 0644); MODULE_PARM_DESC(idle_cache_ms, "J713: cache a static frame after this many ms without a flip (0: never, at most 60000; applies from the next swap)"); +static unsigned int power_gate_ms; + +static int power_gate_ms_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 0, 600000); +} + +static const struct kernel_param_ops power_gate_ms_ops = { + .set = power_gate_ms_set, + .get = param_get_uint, +}; + +module_param_cb(power_gate_ms, &power_gate_ms_ops, &power_gate_ms, 0644); +MODULE_PARM_DESC(power_gate_ms, + "J713: power the display coprocessor and its domains down after this many ms blanked, and before system sleep (0: never, at most 600000)"); + +/* + * DCP CPU control and status, in the coprocessor's ASC block. The idle bit + * is dcp.c's APPLE_DCP_COPROC_CPU_STATUS_IDLE; macOS read status 0x6e at + * its idle check before clearing RUN. + */ +#define DCP_M4_CPU_CONTROL 0x44 +#define DCP_M4_CPU_RUN BIT(4) +#define DCP_M4_CPU_STATUS 0x48 +#define DCP_M4_CPU_STATUS_IDLE BIT(5) + /* One timing of an external sink for its DRM connector. */ struct iomfb_m4_ext_mode { struct drm_display_mode mode; @@ -153,6 +183,17 @@ struct iomfb_m4 { bool display_off; /* setPowerState(0) while blanked, until the next frame */ u64 display_off_cycles; s64 display_off_us, display_off_us_max, display_on_us, display_on_us_max; + /* + * Internal J713 DCP that can power its CPU and display domains down + * while blanked (apple,pmc-interface; power_gate_ms), and whether they + * are down: then no register of the DCP, its mailbox or its display + * DARTs is used until iomfb_m4_power_ungate(). + */ + bool gate_capable; + bool gated; + struct delayed_work gate_work; + u64 gate_cycles; + s64 gate_off_us, gate_on_us; /* Exact completed successor or blank is the only framebuffer retirement. */ struct drm_framebuffer *active_fb; struct drm_framebuffer *uncertain_fb; @@ -216,6 +257,9 @@ static void dcp_m4_runtime_wake(struct iomfb_m4 *dcp); struct dcp_m4_runtime; static void dcp_m4_ext_async_work(struct work_struct *work); static int dcp_m4_set_power_state(struct iomfb_m4 *dcp, bool on); +static int iomfb_m4_power_gate(struct iomfb_m4 *dcp); +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp); +static void iomfb_m4_gate_work(struct work_struct *work); static void dcp_m4_runtime_receive(struct iomfb_m4 *dcp, u64 word); static int dcp_m4_dart_check(struct device *dev) @@ -1025,6 +1069,10 @@ static int dcp_m4_runtime_health(struct dcp_m4_runtime *r) struct iomfb_m4 *dcp = r->dcp; int ret = READ_ONCE(dcp->error); + /* Powered down by its owner: only the failure latch. */ + if (READ_ONCE(dcp->gated)) + return ret; + if (!ret && (!(readl(dcp->coproc + 0x44) & BIT(4)) || dcp_m4_dart_check(dcp->dev) || (!dcp->external && dcp_m4_dart_check(&dcp->piodma->dev)))) ret = -EIO; @@ -1977,6 +2025,14 @@ static void dcp_m4_runtime_idle(struct work_struct *work) int ret; mutex_lock(&dcp->lock); + /* + * Powered down (iomfb_m4_power_gate()): nothing arrives then, and + * nothing may reach the stopped firmware or its powered-down DARTs. + */ + if (READ_ONCE(dcp->gated)) { + mutex_unlock(&dcp->lock); + return; + } ret = dcp_m4_runtime_drain(r, 32); if (ret < 0) { dcp_m4_transaction_fail(&dcp->transaction, ret); @@ -2587,6 +2643,7 @@ int iomfb_m4_init(struct apple_dcp *owner) INIT_WORK(&dcp->ext_put_work, iomfb_m4_ext_put_work); init_waitqueue_head(&dcp->ext_settle_wq); INIT_DELAYED_WORK(&dcp->idle_cache, iomfb_m4_idle_cache_work); + INIT_DELAYED_WORK(&dcp->gate_work, iomfb_m4_gate_work); if (!request_mem_region(res->start, resource_size(res), dev_name(dev))) { ret = -EBUSY; goto release; @@ -2613,6 +2670,9 @@ int iomfb_m4_init(struct apple_dcp *owner) if (ret) return ret; dcp->external = external; + /* The internal display's power-down: its PMC interface names the rest. */ + dcp->gate_capable = !external && apple_pmp_j713_board() && + of_property_present(dev->of_node, "apple,pmc-interface"); owner->v26_6 = dcp; owner->rtk = dcp->rtkit; owner->coproc_reg = dcp->coproc; @@ -3254,12 +3314,16 @@ int iomfb_m4_present(struct apple_dcp *owner, struct drm_framebuffer *fb, fb->pitches[0], fb->format->format == DRM_FORMAT_ARGB8888); if (ret) return ret; + /* A power-down not yet started is not needed; one under way finishes first. */ + cancel_delayed_work(&dcp->gate_work); mutex_lock(&dcp->lock); ret = dcp_m4_runtime_live(dcp->runtime); if (!ret && dcp->powered_off) ret = -EAGAIN; if (!ret) ret = iomfb_m4_check_frame(&frame); + if (!ret && dcp->gated) + ret = iomfb_m4_power_ungate(dcp); /* The first frame after a blank powers the display back up. */ if (!ret && dcp->display_off) { ktime_t start = ktime_get(); @@ -3332,6 +3396,11 @@ int iomfb_m4_blank(struct apple_dcp *owner) dcp->display_off_us = ktime_us_delta(ktime_get(), start); dcp->display_off_us_max = max(dcp->display_off_us, dcp->display_off_us_max); + unsigned int gate_ms = READ_ONCE(power_gate_ms); + + if (dcp->gate_capable && gate_ms) + mod_delayed_work(system_dfl_wq, &dcp->gate_work, + msecs_to_jiffies(gate_ms)); } } } @@ -3381,6 +3450,222 @@ static int dcp_m4_set_power_state(struct iomfb_m4 *dcp, bool on) return ret; } +/* Stop or start the DCP CPU; stopping waits for its idle status first. */ +static int iomfb_m4_cpu_run(struct iomfb_m4 *dcp, bool run) +{ + u32 control = readl(dcp->coproc + DCP_M4_CPU_CONTROL), status; + int ret; + + if (!!(control & DCP_M4_CPU_RUN) == run) + return -EBUSY; + if (!run) { + ret = readl_poll_timeout(dcp->coproc + DCP_M4_CPU_STATUS, status, + status & DCP_M4_CPU_STATUS_IDLE, 100, 100000); + if (ret) + return ret; + } + control = run ? control | DCP_M4_CPU_RUN : control & ~DCP_M4_CPU_RUN; + writel(control, dcp->coproc + DCP_M4_CPU_CONTROL); + return readl(dcp->coproc + DCP_M4_CPU_CONTROL) == control ? 0 : -EIO; +} + +static int iomfb_m4_cpu_start(void *data) +{ + return iomfb_m4_cpu_run(data, true); +} + +/* + * Whether every provider of the power-down can take part now, changing + * nothing: its display DARTs, the PMP vote (held, and releasable without + * losing it), the PMC interface enables and the PMGR states. + */ +static int iomfb_m4_gate_check(struct iomfb_m4 *dcp) +{ + return apple_dart_dcp_power_check(dcp->dev) ?: + apple_dart_dcp_power_check(&dcp->piodma->dev) ?: + apple_pmp_j713_display_check_release(dcp->dev) ?: + apple_pmc_interface_consumer_check(dcp->dev) ?: + apple_pmgr_ps_consumer_check(dcp->dev); +} + +/* + * After the blank's setPowerState(0), power the DCP and the display domains + * down as macOS does for display sleep: release the AP link (A500), let RTKit + * idle, stop the CPU, then power down its display DARTs, release its PMP + * vote, clear its PMC interface enables and power its CPU, frontend and + * system domains off. Firmware, endpoints and shared buffers stay in place + * for iomfb_m4_power_ungate(). Before the first step every provider is + * asked whether it can take part: if one cannot, nothing changes and the + * display just stays logically off. A failure after that fails the session. + */ +static int iomfb_m4_power_gate(struct iomfb_m4 *dcp) +{ + struct dcp_m4_runtime *r = dcp->runtime; + ktime_t start = ktime_get(); + unsigned long flags; + __le32 zero = 0; + const char *step; + int ret; + + lockdep_assert_held(&dcp->lock); + if (!dcp->gate_capable || dcp->gated || !dcp->display_off || dcp->powered_off || + dcp->active_fb || dcp->active_frame.dva || dcp->transaction.calls) + return -EBUSY; + ret = dcp_m4_runtime_live(r); + if (ret) + return ret; + ret = iomfb_m4_gate_check(dcp); + if (ret) { + dev_dbg(dcp->dev, "display power-down not possible now (%d): stays logically off\n", + ret); + return ret; + } + + step = "A500"; + ret = dcp_m4_runtime_call(r, 500, false, &zero, sizeof(zero), NULL, 0); + if (!ret) { + step = "drain"; + ret = dcp_m4_runtime_drain(r, 64); + if (ret > 0 || (!ret && r->count)) + ret = -EBUSY; + } + if (!ret) { + step = "RTKit idle"; + ret = apple_rtkit_idle(dcp->rtkit); + } + if (ret) + goto fail; + apple_rtkit_hold_rx(dcp->rtkit); + step = "CPU stop"; + ret = iomfb_m4_cpu_run(dcp, false); + if (ret) + goto fail; + /* + * The stopped CPU sends nothing more. A message it sent during the idle + * handshake would be handled after the power-down, by firmware that is + * stopped and through DARTs that are off: take everything in now, and + * fail here, with nothing powered down, if anything arrived. + */ + step = "late message"; + ret = apple_rtkit_poll(dcp->rtkit); + if (ret >= 0) { + apple_rtkit_flush_rx(dcp->rtkit); + spin_lock_irqsave(&dcp->state_lock, flags); + ret = r->count ? -EBUSY : 0; + spin_unlock_irqrestore(&dcp->state_lock, flags); + } + if (ret) + goto fail; + + /* From here on nothing may use the DCP's or its DARTs' registers. */ + WRITE_ONCE(dcp->gated, true); + step = "DART"; + ret = apple_dart_dcp_power_off(dcp->dev); + if (!ret) + ret = apple_dart_dcp_power_off(&dcp->piodma->dev); + if (!ret) { + step = "PMP"; + ret = apple_pmp_j713_display_put(dcp->dev); + } + if (!ret) { + step = "PMC"; + ret = apple_pmc_interface_consumer_off(dcp->dev); + } + if (!ret) { + step = "PMGR"; + ret = apple_pmgr_ps_consumer_off(dcp->dev); + } + if (ret) + goto fail; + dcp->gate_cycles++; + dcp->gate_off_us = ktime_us_delta(ktime_get(), start); + return 0; +fail: + dev_err(dcp->dev, "display power-down failed at %s: %d\n", step, ret); + dcp_m4_failed(dcp, ret); + return ret; +} + +/* + * Power the DCP back up for the first frame after a gated blank, in macOS's + * order: the domains, the PMP vote, the PMC interface enables, then the + * display DARTs, then the CPU through RTKit's retained wake, and the display + * preparation macOS sends before setPowerState(1). A failure fails the + * session. + */ +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp) +{ + struct dcp_m4_runtime *r = dcp->runtime; + ktime_t start = ktime_get(); + u8 prepare[8] = {}, prepared[8] = {}; + __le32 main_display = 0; + const char *step; + int ret; + + lockdep_assert_held(&dcp->lock); + if (!dcp->gated) + return 0; + step = "PMGR"; + ret = apple_pmgr_ps_consumer_on(dcp->dev); + if (!ret) { + step = "PMP"; + ret = apple_pmp_j713_display_get(dcp->dev); + } + if (!ret) { + step = "PMC"; + ret = apple_pmc_interface_consumer_on(dcp->dev); + } + if (!ret) { + step = "DART"; + ret = apple_dart_dcp_power_on(dcp->dev); + } + if (!ret) + ret = apple_dart_dcp_power_on(&dcp->piodma->dev); + if (!ret) { + step = "RTKit wake"; + ret = apple_rtkit_wake_retained(dcp->rtkit, iomfb_m4_cpu_start, dcp); + } + if (ret) + goto fail; + WRITE_ONCE(dcp->gated, false); + + step = "A414"; + ret = dcp_m4_runtime_call(r, 414, false, prepare, sizeof(prepare), + prepared, sizeof(prepared)); + if (!ret && get_unaligned_le32(prepared + 4)) + ret = -EREMOTEIO; + if (!ret) { + step = "A412"; + ret = dcp_m4_runtime_call(r, 412, false, NULL, 0, &main_display, + sizeof(main_display)); + } + if (!ret && le32_to_cpu(main_display) != 1) + ret = -EREMOTEIO; + if (ret) + goto fail; + dcp->gate_on_us = ktime_us_delta(ktime_get(), start); + return 0; +fail: + dev_err(dcp->dev, "display power-up failed at %s: %d\n", step, ret); + dcp_m4_failed(dcp, ret); + return ret; +} + +static void iomfb_m4_gate_work(struct work_struct *work) +{ + struct iomfb_m4 *dcp = container_of(to_delayed_work(work), struct iomfb_m4, gate_work); + + mutex_lock(&dcp->lock); + /* + * Still blanked and not yet down: a frame since then cancels it, and so + * does turning the power-down off meanwhile (power_gate_ms = 0). + */ + if (dcp->runtime && dcp->display_off && !dcp->gated && !READ_ONCE(dcp->error) && + !READ_ONCE(dcp->owner->quiescing) && READ_ONCE(power_gate_ms)) + iomfb_m4_power_gate(dcp); + mutex_unlock(&dcp->lock); +} + /* A364: the firmware's record of a timing and colour pair. */ struct dcp_m4_mode_info_req { u8 mode[0x64]; @@ -4976,14 +5261,23 @@ int iomfb_m4_suspend(struct apple_dcp *owner) } if (!dcp || !dcp->runtime) return -ENODEV; + cancel_delayed_work_sync(&dcp->gate_work); mutex_lock(&dcp->lock); /* Also the startup surface, shown before the first frame */ if (dcp->active_fb || dcp->active_frame.dva) { dev_warn(dcp->dev, "panel still scanning out: refusing system sleep\n"); ret = -EBUSY; } else if (dcp->display_off) { - /* Already powered down by the blank; the next frame powers it up. */ + /* + * Already powered down by the blank; the next frame powers it up. + * Power the DCP and its domains down too, if it may. That is an + * extra saving, not a condition of sleep: a power-down that cannot + * start leaves the display logically off, and one that fails has + * failed the session already; neither refuses the sleep. + */ ret = 0; + if (dcp->gate_capable && READ_ONCE(power_gate_ms) && !dcp->gated) + iomfb_m4_power_gate(dcp); } else { ret = dcp_m4_set_power_state(dcp, false); if (!ret) @@ -5011,7 +5305,10 @@ int iomfb_m4_resume(struct apple_dcp *owner) if (!dcp || !dcp->runtime) return -ENODEV; mutex_lock(&dcp->lock); - if (dcp->powered_off) { + if (dcp->gated) { + /* Stays down until the next frame. */ + ret = 0; + } else if (dcp->powered_off) { ret = dcp_m4_set_power_state(dcp, true); if (!ret) dcp->powered_off = false; @@ -5079,6 +5376,7 @@ void iomfb_m4_quiesce(struct apple_dcp *owner) flush_workqueue(dcp->runtime->wq); /* After the flush, as drained callbacks can queue it. */ cancel_delayed_work_sync(&dcp->idle_cache); + cancel_delayed_work_sync(&dcp->gate_work); } static int iomfb_m4_status_show(struct seq_file *s, void *unused) @@ -5097,7 +5395,11 @@ static int iomfb_m4_status_show(struct seq_file *s, void *unused) dcp_m4_failed(dcp, health); seq_printf(s, "firmware=26.6.2 ready=%u qualified=%u error=%d run=%u health=%d\n", dcp->ready, r && r->qualified, dcp->error, - !!(readl(dcp->coproc + 0x44) & BIT(4)), health); + !dcp->gated && !!(readl(dcp->coproc + DCP_M4_CPU_CONTROL) & DCP_M4_CPU_RUN), + health); + seq_printf(s, "gate_capable=%u gated=%u gate_cycles=%llu gate_off_us=%lld gate_on_us=%lld\n", + dcp->gate_capable, dcp->gated, dcp->gate_cycles, dcp->gate_off_us, + dcp->gate_on_us); seq_printf(s, "startup=%u background=%u power=%u scanout=%u callbacks=%u\n", r && r->done, r && r->background_done, r && r->power_done, r && r->scanout_done, r ? r->callbacks : 0); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py index 4d0e9c50a1082..622957c3b6837 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py @@ -493,6 +493,148 @@ def dart(root): ''' +def gate(root): + """The physical power-down and power-up of the internal DCP: their order, + the providers asked before the first step, and a late firmware message.""" + backend = (root / APPLE / 'iomfb_m4.c').read_text() + return PREFIX + r''' +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define container_of(p,t,m) ((t *)((char *)(p) - __builtin_offsetof(t,m))) +#define le32_to_cpu(x) (x) +#define dev_dbg(...) ((void)0) +#define dev_err(d,...) ((void)(d),(void)fprintf(stderr,__VA_ARGS__)) +#define spin_lock_irqsave(p,f) do { (void)(p); (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(p); (void)(f); } while(0) +typedef u32 __le32; +typedef int64_t s64; +typedef s64 ktime_t; +static ktime_t clock_ns; +static ktime_t ktime_get(void) { return clock_ns += 1000; } +static s64 ktime_us_delta(ktime_t later,ktime_t earlier) { return (later-earlier)/1000; } +struct device { int unused; }; +struct platform_device { struct device dev; }; +struct apple_rtkit { int unused; }; +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; }; +#define to_delayed_work(w) container_of(w,struct delayed_work,work) +struct apple_dcp { bool quiescing; }; +struct dcp_m4_frame { u64 dva; }; +struct dcp_m4_transaction { unsigned calls; }; +struct iomfb_m4; +struct dcp_m4_runtime { struct iomfb_m4 *dcp; unsigned count; }; +struct iomfb_m4 { + struct apple_dcp *owner; + struct device *dev; + struct platform_device *piodma; + struct apple_rtkit *rtkit; + struct dcp_m4_runtime *runtime; + int lock,state_lock,error; + struct dcp_m4_transaction transaction; + struct dcp_m4_frame active_frame; + void *active_fb; + bool gate_capable,gated,display_off,powered_off; + struct delayed_work gate_work; + u64 gate_cycles; + s64 gate_off_us,gate_on_us; +}; +static unsigned power_gate_ms; +static struct iomfb_m4 *the_dcp; +/* Every step taken, in order. */ +static char steps[512]; +static void trace(const char *s) { strcat(steps,s); strcat(steps," "); } +static int refuse; /* the provider check that refuses (1..5), or 0 */ +static bool late_word; /* a DCP message arrives during the idle handshake */ +static void mutex_lock(int *l) { assert(!*l); *l=1; } +static void mutex_unlock(int *l) { assert(*l==1); *l=0; } +static void lockdep_assert_held(int *l) { assert(*l==1); } +static void dcp_m4_failed(struct iomfb_m4 *dcp,int error) { if(!dcp->error) dcp->error=error; } +static int dcp_m4_runtime_live(struct dcp_m4_runtime *r) { return r->dcp->error; } +static int dcp_m4_runtime_call(void *cookie,unsigned int method,bool remote, + const void *input,u32 input_size,void *output,u32 output_size) +{ + char name[16]; + (void)cookie; (void)remote; (void)input; (void)input_size; + snprintf(name,sizeof(name),"A%u",method); trace(name); + if(output) memset(output,0,output_size); + if(method==412) { assert(output_size==4); put_unaligned_le32(1,output); } + return 0; +} +static int dcp_m4_runtime_drain(struct dcp_m4_runtime *r,unsigned budget) +{ (void)r; (void)budget; trace("drain"); return 0; } +static int apple_rtkit_idle(struct apple_rtkit *rtk) { (void)rtk; trace("idle"); return 0; } +static void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { (void)rtk; trace("hold"); } +static int apple_rtkit_poll(struct apple_rtkit *rtk) { (void)rtk; trace("poll"); return 0; } +static void apple_rtkit_flush_rx(struct apple_rtkit *rtk) +{ (void)rtk; trace("flush"); if(late_word) the_dcp->runtime->count++; } +static int iomfb_m4_cpu_run(struct iomfb_m4 *dcp,bool run) { (void)dcp; trace(run?"run":"stop"); return 0; } +static int iomfb_m4_cpu_start(void *data) { return iomfb_m4_cpu_run(data,true); } +static int apple_rtkit_wake_retained(struct apple_rtkit *rtk,int (*start)(void *),void *ctx) +{ (void)rtk; trace("wake"); return start(ctx); } +static bool dcp_dev(struct device *dev) { return dev==the_dcp->dev; } +static int apple_dart_dcp_power_check(struct device *dev) { return refuse==(dcp_dev(dev)?1:2)?-EAGAIN:0; } +static int apple_pmp_j713_display_check_release(struct device *dev) { (void)dev; return refuse==3?-ETIMEDOUT:0; } +static int apple_pmc_interface_consumer_check(struct device *dev) { (void)dev; return refuse==4?-ENODEV:0; } +static int apple_pmgr_ps_consumer_check(struct device *dev) { (void)dev; return refuse==5?-EBUSY:0; } +static int apple_dart_dcp_power_off(struct device *dev) { trace(dcp_dev(dev)?"dart-off":"piodma-off"); return 0; } +static int apple_dart_dcp_power_on(struct device *dev) { trace(dcp_dev(dev)?"dart-on":"piodma-on"); return 0; } +static int apple_pmp_j713_display_put(struct device *dev) { (void)dev; trace("pmp-put"); return 0; } +static int apple_pmp_j713_display_get(struct device *dev) { (void)dev; trace("pmp-get"); return 0; } +static int apple_pmc_interface_consumer_off(struct device *dev) { (void)dev; trace("pmc-off"); return 0; } +static int apple_pmc_interface_consumer_on(struct device *dev) { (void)dev; trace("pmc-on"); return 0; } +static int apple_pmgr_ps_consumer_off(struct device *dev) { (void)dev; trace("pmgr-off"); return 0; } +static int apple_pmgr_ps_consumer_on(struct device *dev) { (void)dev; trace("pmgr-on"); return 0; } +''' + '\n'.join(block(backend, name) for name in ( + 'static int iomfb_m4_gate_check(', 'static int iomfb_m4_power_gate(', + 'static int iomfb_m4_power_ungate(', 'static void iomfb_m4_gate_work(')) + r''' +int main(void) +{ + struct apple_dcp owner={0}; + struct device dev={0}; + struct platform_device piodma={{0}}; + struct apple_rtkit rtk={0}; + struct iomfb_m4 dcp={.owner=&owner,.dev=&dev,.piodma=&piodma,.rtkit=&rtk, + .gate_capable=true,.display_off=true}; + struct dcp_m4_runtime r={.dcp=&dcp}; + + dcp.runtime=&r; the_dcp=&dcp; + /* Turned off after the blank: the queued power-down does nothing. */ + iomfb_m4_gate_work(&dcp.gate_work.work); + assert(!steps[0] && !dcp.gated); + /* Wanted: the whole power-down, in its order. */ + power_gate_ms=1000; + iomfb_m4_gate_work(&dcp.gate_work.work); + assert(!strcmp(steps,"A500 drain idle hold stop poll flush dart-off piodma-off " + "pmp-put pmc-off pmgr-off ")); + assert(dcp.gated && !dcp.error && dcp.gate_cycles==1 && !dcp.lock); + + /* The power-up, in macOS's order: domains, PMP vote, PMC enables, DARTs, CPU. */ + steps[0]=0; dcp.lock=1; + assert(!iomfb_m4_power_ungate(&dcp)); + assert(!strcmp(steps,"pmgr-on pmp-get pmc-on dart-on piodma-on wake run A414 A412 ")); + assert(!dcp.gated && !dcp.error); + + /* A provider that cannot take part now: no step at all, and no failure. */ + for(refuse=1;refuse<=5;refuse++) { + steps[0]=0; + assert(iomfb_m4_power_gate(&dcp)<0); + assert(!steps[0] && !dcp.gated && !dcp.error); + } + refuse=0; + + /* A message from the idle handshake: failed with nothing powered down. */ + steps[0]=0; late_word=true; + assert(iomfb_m4_power_gate(&dcp)==-EBUSY); + assert(!strcmp(steps,"A500 drain idle hold stop poll flush ")); + assert(!dcp.gated && dcp.error==-EBUSY); + dcp.lock=0; + puts("PASS the power-down and power-up keep their order, a refusing provider changes nothing, " + "and a late message fails before anything is powered down"); + return 0; +} +''' + + def blank(root): backend = (root / APPLE / 'iomfb_m4.c').read_text() backlight = (root / APPLE / 'dcp_backlight.c').read_text() @@ -551,7 +693,16 @@ def blank(root): bool display_off; u64 display_off_cycles; s64 display_off_us,display_off_us_max,display_on_us,display_on_us_max; + bool gate_capable,gated; + int gate_work; }; +/* The power-down after a blank: its scheduling and the power-up before ON. */ +static unsigned power_gate_ms,scheduled,cancelled,ungates,ungate_error,ons_at_ungate; +#define system_dfl_wq NULL +static unsigned long msecs_to_jiffies(unsigned ms) { return ms; } +static bool mod_delayed_work(void *wq,int *work,unsigned long delay) +{ (void)wq; (void)work; assert(delay==power_gate_ms); scheduled++; return true; } +static bool cancel_delayed_work(int *work) { (void)work; cancelled++; return false; } static void mutex_lock(int *lock) { assert(!*lock); *lock=1; } static void mutex_unlock(int *lock) { assert(*lock==1); *lock=0; } static void lockdep_assert_held(int *lock) { assert(*lock==1); } @@ -568,6 +719,13 @@ def blank(root): static int dcp_m4_set_power_state(struct iomfb_m4 *dcp,bool on) { assert(dcp->lock==1); if(on) power_ons++; else power_offs++; return 0; } static int dcp_m4_runtime_live(struct dcp_m4_runtime *r) { return r->dcp->error; } +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp) +{ + assert(dcp->lock==1 && dcp->gated && dcp->display_off); + ons_at_ungate=power_ons; ungates++; + if(ungate_error) { dcp_m4_failed(dcp,ungate_error); return ungate_error; } + dcp->gated=false; return 0; +} static int iomfb_m4_check_frame(const struct dcp_m4_frame *f) { return dcp_m4_frame_check(f); } static unsigned order,failure,publications; @@ -640,6 +798,42 @@ def blank(root): assert(fb.refs==2 && next.refs==2 && dcp.active_fb==&fb && dcp.uncertain_fb==&next); failure=0; } + /* Not opted in: a blank schedules no power-down; a frame still cancels one. */ + { + struct drm_format_info fmt={.format=DRM_FORMAT_XRGB8888,.num_planes=1}; + struct drm_framebuffer fb={.format=&fmt,.width=2560,.height=1664,.pitches={10240},.refs=1}; + struct apple_dcp owner={0}; + struct iomfb_m4 dcp={.owner=&owner}; + struct dcp_m4_runtime runtime={.dcp=&dcp}; + unsigned c; + owner.v26_6=&dcp; dcp.runtime=&runtime; + assert(!dcp_bl_init(&owner.backlight.state,500,true,120,false,0)); + c=cancelled; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(cancelled==c+1); + assert(!iomfb_m4_blank(&owner) && scheduled==0); + dcp.gate_capable=true; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(!iomfb_m4_blank(&owner) && scheduled==0); /* power_gate_ms is 0 */ + + /* Opted in: one power-down after the blank, after power_gate_ms. */ + power_gate_ms=1000; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(!iomfb_m4_blank(&owner) && scheduled==1); + /* Powered down: the next frame powers it up first, then setPowerState(1). */ + dcp.gated=true; + unsigned ons=power_ons; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(ungates==1 && ons_at_ungate==ons && power_ons==ons+1); + assert(!dcp.gated && !dcp.display_off && dcp.active_fb==&fb); + /* A failed power-up refuses the frame; the session is failed by it. */ + assert(!iomfb_m4_blank(&owner) && scheduled==2); + dcp.gated=true; ungate_error=-EIO; ons=power_ons; + assert(iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)==-EIO); + assert(ungates==2 && power_ons==ons && dcp.error==-EIO && dcp.display_off); + assert(fb.refs==1 && !dcp.active_fb); + ungate_error=0; power_gate_ms=0; + } puts("PASS DPMS-off blank carries zero before delayed work and completes only after both exact receipts"); puts("PASS actual present/blank retain active+uncertain DMA framebuffer references across a missing receipt"); puts("PASS a present while the display is powered off for sleep is refused without a reference"); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index ac69a16514c3e..18374e2337a58 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -83,6 +83,7 @@ def main(): ('m4', 'm4-common-plane', m4.plane, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('m4', 'm4-producer-retirement', m4.prepare, ((),), ('-Wno-unused-function',)), ('m4', 'm4-dpms-retirement', m4.blank, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), + ('m4', 'm4-power-gate', m4.gate, ((),), ('-Wno-unused-function',)), ('m4', 'm4-dart-consumer', m4.dart, ((),), ('-Wno-unused-function',)), ('m4', 'm4-panel-timing', m4.timing, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('boot', 'oslog-events', boot.oslog, ((),), ()), From 539735dc5b714f1d85a251324a361fe46ddaeb83 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:57:33 +0100 Subject: [PATCH 254/392] dt-bindings: display: apple,dcp: name the display pipe's PMC interface A DCP that powers its display domains down while the display is off clears its pipe's PMC interface enables first and restores them after powering the domains up. Add apple,pmc-interface, a phandle to the pipe's apple,consumer-switched PMC interface node. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 3db18e058dce3..b45a737748180 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -274,6 +274,14 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: Height in pixels reserved for the display notch. + apple,pmc-interface: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PMC interface enables of this coprocessor's display pipe, an + apple,consumer-switched interface node. The coprocessor clears them + before it powers its display domains down while the display is off, + and restores them after it powers them up again. + apple,pmp-report: $ref: /schemas/types.yaml#/definitions/phandle description: From 5e31b3588f813b6a8ab5dfbf4db9671464a5b486 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:57:33 +0100 Subject: [PATCH 255/392] arm64: dts: apple: t8132: describe the display power-down resources Describe what a power-down of the internal display domains needs on T8132: - the PMGR register block up to the display CPU's power-gate retention registers at 0x2504c and 0x25070; - the PS_PROT bank of clock protection slices, as a syscon; - the internal display pipe's PMC interface enables (BWR and boost bit 0), as a consumer-switched interface, since its power states are firmware-owned rather than a power domain. No consumer uses them yet. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- arch/arm64/boot/dts/apple/t8132-memory.dtsi | 12 ++++++++++++ arch/arm64/boot/dts/apple/t8132.dtsi | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8132-memory.dtsi b/arch/arm64/boot/dts/apple/t8132-memory.dtsi index c6cafa8219d2a..e0de465b9ffe1 100644 --- a/arch/arm64/boot/dts/apple/t8132-memory.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-memory.dtsi @@ -24,6 +24,18 @@ * boost interface enables. The PMC ignores a pipe's bandwidth * and display clock votes while they are clear. */ + /* + * The internal display pipe's enables. Its power states are + * firmware-owned, so they are not a power domain: its DCP + * clears them before powering the pipe down and restores them. + */ + pmc_disp0: interface-disp0 { + label = "disp0-interface"; + apple,consumer-switched; + apple,bwr-bit = <0>; + apple,boost-bit = <0>; + }; + pmc_dispext0: interface-dispext0 { label = "dispext0-interface"; #power-domain-cells = <0>; diff --git a/arch/arm64/boot/dts/apple/t8132.dtsi b/arch/arm64/boot/dts/apple/t8132.dtsi index 35490237bba94..d6a01e1473200 100644 --- a/arch/arm64/boot/dts/apple/t8132.dtsi +++ b/arch/arm64/boot/dts/apple/t8132.dtsi @@ -474,10 +474,17 @@ compatible = "apple,t8132-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; #size-cells = <1>; - reg = <0x3 0x80700000 0 0x14000>; + /* Up to the display CPU's power-gate retention registers. */ + reg = <0x3 0x80700000 0 0x26000>; /* child nodes are added in t8132-pmgr.dtsi */ }; + /* Clock protection slices guarding power states of DART domains. */ + ps_prot: syscon@38079c000 { + compatible = "apple,t8132-ps-prot", "syscon"; + reg = <0x3 0x8079c000 0 0x4000>; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8132-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; From 23885d96c8087a1c149874ea2e563d8370079ca1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:58:28 -0500 Subject: [PATCH 256/392] arm64: dts: apple: t6050: Put the SEP in die 0's soc node The boot loader finds the SEP by the path a board's sep alias gives. The inventory puts it in the unit-less soc node, and "/soc" names soc@200000000 first, so such a path never reaches it, and the loader could not reserve the SEP firmware or add its manifests. Move the SEP, its mailbox and its DART into die 0's soc node, whose ranges cover them. Signed-off-by: Diljit Singh --- .../dts/apple/adt-coverage/t6050-j716s.txt | 16 ++-- .../dts/apple/adt-coverage/t6050-j873s.txt | 16 ++-- .../boot/dts/apple/t6050-peripherals.dtsi | 88 ++++++++++--------- 3 files changed, 63 insertions(+), 57 deletions(-) diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t6050-j716s.txt b/arch/arm64/boot/dts/apple/adt-coverage/t6050-j716s.txt index 096aa6df9a804..82218ccb772ab 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t6050-j716s.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t6050-j716s.txt @@ -1268,14 +1268,14 @@ /arm-io/dart-pmp1 - -> MISSING /arm-io/dart-pmp1/mapper-pmp1 - -> WAIVED: IOMMU stream, described by iommus or iommu-map; DART hardware is inventoried separately. /arm-io/dart-pmp1/mapper-pmp1-ptd-fwd - -> WAIVED: IOMMU stream, described by iommus or iommu-map; DART hardware is inventoried separately. -/arm-io/sep 0x502600000 -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/Sandcat - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/xART - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/InvalidateHmac - -> /soc/sep@502600000 -/arm-io/dart-sep 0x500b80000 -> /soc/iommu@500b80000 -/arm-io/dart-sep/mapper-sep - -> /soc/iommu@500b80000 -/arm-io/dart-sep/mapper-sep-exclave - -> /soc/iommu@500b80000 +/arm-io/sep 0x502600000 -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/Sandcat - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/xART - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/InvalidateHmac - -> /soc@200000000/sep@502600000 +/arm-io/dart-sep 0x500b80000 -> /soc@200000000/iommu@500b80000 +/arm-io/dart-sep/mapper-sep - -> /soc@200000000/iommu@500b80000 +/arm-io/dart-sep/mapper-sep-exclave - -> /soc@200000000/iommu@500b80000 /arm-io/esm 0x54267c000 -> MISSING /arm-io/ans 0x419600000 -> /soc@200000000/nvme@45dcc0000 /arm-io/ans/iop-ans-nub - -> /soc@200000000/nvme@45dcc0000 diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t6050-j873s.txt b/arch/arm64/boot/dts/apple/adt-coverage/t6050-j873s.txt index 939c40a8c931d..c014ce038d1f0 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t6050-j873s.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t6050-j873s.txt @@ -1192,14 +1192,14 @@ /arm-io/dart-pmp1 - -> MISSING /arm-io/dart-pmp1/mapper-pmp1 - -> WAIVED: IOMMU stream, described by iommus or iommu-map; DART hardware is inventoried separately. /arm-io/dart-pmp1/mapper-pmp1-ptd-fwd - -> WAIVED: IOMMU stream, described by iommus or iommu-map; DART hardware is inventoried separately. -/arm-io/sep 0x502600000 -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/Sandcat - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/xART - -> /soc/sep@502600000 -/arm-io/sep/iop-sep-nub/InvalidateHmac - -> /soc/sep@502600000 -/arm-io/dart-sep 0x500b80000 -> /soc/iommu@500b80000 -/arm-io/dart-sep/mapper-sep - -> /soc/iommu@500b80000 -/arm-io/dart-sep/mapper-sep-exclave - -> /soc/iommu@500b80000 +/arm-io/sep 0x502600000 -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/Sandcat - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/xART - -> /soc@200000000/sep@502600000 +/arm-io/sep/iop-sep-nub/InvalidateHmac - -> /soc@200000000/sep@502600000 +/arm-io/dart-sep 0x500b80000 -> /soc@200000000/iommu@500b80000 +/arm-io/dart-sep/mapper-sep - -> /soc@200000000/iommu@500b80000 +/arm-io/dart-sep/mapper-sep-exclave - -> /soc@200000000/iommu@500b80000 /arm-io/esm 0x54267c000 -> MISSING /arm-io/ans 0x419600000 -> /soc@200000000/nvme@45dcc0000 /arm-io/ans/iop-ans-nub - -> /soc@200000000/nvme@45dcc0000 diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index e82926b257398..7507a40008c23 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2202,47 +2202,6 @@ status = "disabled"; }; - /* iBoot hands over a running SEP; apple_sep enables it and its DART when it loads. */ - sep: sep@502600000 { - compatible = "apple,t6050-sep", "apple,sep"; - reg = <0x5 0x2600000 0x0 0x88000>, - <0x5 0x2050000 0x0 0x4000>; - mboxes = <&sep_mbox>; - mbox-names = "mbox"; - iommus = <&sep_dart 0>; - power-domains = <&ps_sep>; - clocks = <&peripheral_clocks 447>; - status = "disabled"; - }; - - /* The ASC v4 mailbox at sep + 0x8000; the first four ADT interrupts in mailbox order */ - sep_mbox: mailbox@502608000 { - compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x5 0x2608000 0x0 0x4000>; - interrupts = , - , - , - ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; - #mbox-cells = <0>; - status = "disabled"; - }; - - sep_dart: iommu@500b80000 { - compatible = "apple,t6050-dart", "apple,t8110-dart"; - reg = <0x5 0xb80000 0x0 0x20000>; - interrupts = ; - #iommu-cells = <1>; - /* - * The window iBoot programmed (ADT dart-sep vm-base-0, vm-size-0), which - * already carries the BIT(40) alias the DART translates. - */ - apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; - apple,dma-offset = /bits/ 64 <0x0>; - status = "disabled"; - }; - aon_ptd: aon-ptd@28cdc0000 { compatible = "apple,t6050-aon-ptd", "apple,aon-ptd"; reg = <0x2 0x8cdc0000 0x0 0x4000>, @@ -3417,4 +3376,51 @@ +}; + +/* + * The SEP sits in die 0's soc node, not in the unit-less soc above: the boot loader + * resolves the sep alias by path, and "/soc" names soc@200000000 first. + */ +&die0 { + /* iBoot hands over a running SEP; apple_sep enables it and its DART when it loads. */ + sep: sep@502600000 { + compatible = "apple,t6050-sep", "apple,sep"; + reg = <0x5 0x2600000 0x0 0x88000>, + <0x5 0x2050000 0x0 0x4000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + clocks = <&peripheral_clocks 447>; + status = "disabled"; + }; + + /* The ASC v4 mailbox at sep + 0x8000; the first four ADT interrupts in mailbox order */ + sep_mbox: mailbox@502608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x5 0x2608000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + sep_dart: iommu@500b80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x5 0xb80000 0x0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + /* + * The window iBoot programmed (ADT dart-sep vm-base-0, vm-size-0), which + * already carries the BIT(40) alias the DART translates. + */ + apple,dma-range = /bits/ 64 <0x10000004000 0xffff0000>; + apple,dma-offset = /bits/ 64 <0x0>; + status = "disabled"; + }; }; From c7a700e63d119497ccde4aa06c4a1f38a0583d6d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 06:56:20 -0500 Subject: [PATCH 257/392] arm64: dts: apple: t6050-j714s: Point the boot loader at the SEP The boot loader finds the SEP through the sep alias: it reserves the SEP firmware (ADT memory-map SEPFW) and adds the iBoot and local policy manifests the enclave needs to open the machine's secure storage. Without them apple_sep refuses to attach. J616s has the same alias. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 9cd9ee94ab32b..d98f6ac1d63a0 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -20,6 +20,11 @@ model = "Apple MacBook Pro (14-inch, M5 Pro, 2026)"; chassis-type = "laptop"; + aliases { + /* the boot loader reserves the SEP firmware and adds its manifests through this alias */ + sep = &sep; + }; + chosen { #address-cells = <2>; #size-cells = <2>; From b0517f95c68b364a6513d78451d5b487f740cfdc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 07:32:34 -0500 Subject: [PATCH 258/392] soc: apple: pmp: Add T6050 The T6050 PMP DART's window starts at 1 TiB and spans 64 GiB (ADT dart-pmp0 vm-base and vm-size), as T8142's does, so the firmware gets a 42-bit DMA mask. iBoot loads the firmware without starting it; the driver starts it as on the other SoCs. Unlike T8142's, this firmware never readies endpoint 7 (CoreAnalytics): waiting for it fails the probe with -ETIME, so it isn't started. Tested on J714s (M5 Pro) with the PMP described as in the following commit and enabled, board-id, DRAM vendor, PMC boot arguments and the ADT's iop-pmp0-nub properties as tunables: "PMP firmware and endpoint ready" 1.6 ms after RTKit init, and a PMC gated on it binds. With endpoint 7 started the probe failed with -62. Signed-off-by: Diljit Singh --- drivers/soc/apple/pmp.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 2212509efd82c..58aa64cee28c5 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -701,6 +701,14 @@ const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { coreanalytics: Ep7::Always, }; +/// T6050: the DART window starts at 1 TiB (ADT dart-pmp0 vm-base) and spans +/// 64 GiB, as on T8142. Its firmware never readies endpoint 7 (-ETIME). +const HW_CFG_T6050: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), + check_bootargs_uuid: false, + coreanalytics: Ep7::No, +}; + const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, @@ -722,6 +730,7 @@ kernel::of_device_table!( (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), (of::DeviceId::new(c"apple,t8142-pmp-v2"), &HW_CFG_T8142), (of::DeviceId::new(c"apple,t6041-pmp-v2"), &HW_CFG_T6041), + (of::DeviceId::new(c"apple,t6050-pmp-v2"), &HW_CFG_T6050), (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), ] ); From c72b3e4309e98856dbedc29c5f9c1e0ba219b13d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 07:32:34 -0500 Subject: [PATCH 259/392] arm64: dts: apple: t6050: Describe the PMP for its driver The PMP driver maps the firmware's SRAM as "pmp" (ADT pmp0 reg[2]) and the ASC as "asc" (reg[0]), talks through the ASC mailbox at +0x8000 and allocates through its DART, whose window starts at 1 TiB and spans 64 GiB (ADT dart-pmp0 vm-base, vm-size). Describe them that way; the mailbox interrupts are the ADT's four in ascending order, as for the other T6050 ASC mailboxes. Everything stays disabled: a board enables the PMP once it supplies the boot arguments and firmware properties. Signed-off-by: Diljit Singh --- .../boot/dts/apple/t6050-peripherals.dtsi | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index eef667d647d3d..c4be3fcf47ca7 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2176,27 +2176,42 @@ status = "disabled"; }; + /* ADT pmp0 reg[2] (firmware SRAM) and reg[0] (ASC); iBoot loads the firmware, the OS starts it */ pmp0: pmp@284e00000 { compatible = "apple,t6050-pmp-v2", "apple,t6000-pmp-v2"; - reg = <0x2 0x84e00000 0x0 0x88000>, - <0x2 0x84850000 0x0 0x4000>, - <0x2 0x84500000 0x0 0x100000>, - <0x2 0x84250000 0x0 0x4000>; - interrupts = , - , - , - ; + reg = <0x2 0x84500000 0x0 0x100000>, + <0x2 0x84e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp0_mbox>; + mbox-names = "mbox"; iommus = <&pmp0_dart 0>; power-domains = <&ps_pmp>, <&ps_pms_sram>; status = "disabled"; }; + pmp0_mbox: mailbox@284e08000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x84e08000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + pmp0_dart: iommu@2841a0000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x2 0x841a0000 0x0 0xc000>, <0x2 0x841b0000 0x0 0x4000>; interrupts = ; #iommu-cells = <1>; + /* ADT dart-pmp0 vm-base, vm-size */ + apple,dma-range = /bits/ 64 <0x10000000000 0x1000000000>; + power-domains = <&ps_pmp>; status = "disabled"; }; From 90eb7840c1517e2e1cf0bfb0cd222531a76e9e8c Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Fri, 6 Nov 2026 18:57:33 +0100 Subject: [PATCH 260/392] arm64: dts: apple: t8132-j713: let the DCP power its display domains down Let the internal DCP power its display down while blanked (appledrm.power_gate_ms): - the display CPU, frontend and system power states are consumer-switched, the CPU with the retention workaround macOS sets while it is off (0x3 in 0x2504c, 0x0 in 0x25070); - the DCP firmware DART and the display frontend DART name their clock protection slices, 42 and 8, as the ADT does; - the DCP names the internal pipe's PMC interface. The power states are in the policy the J715 shares, where nothing uses them yet: only the J713 names the PMC interface, and the J715 is untested. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bartosz Nitka --- arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi | 6 ++++++ arch/arm64/boot/dts/apple/t8132-j713.dts | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi index 7f0e91c4002eb..e6a1ab6ebeb34 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi @@ -105,13 +105,16 @@ status = "disabled"; }; +/* The DCP powers these down while the display is off. */ &ps_disp_sys { apple,firmware-owned; + apple,consumer-switched; status = "okay"; }; &ps_disp_fe { apple,firmware-owned; + apple,consumer-switched; status = "okay"; }; @@ -531,6 +534,9 @@ &ps_disp_cpu { apple,firmware-owned; + apple,consumer-switched; + /* Power-gate retention workaround, as macOS sets it while off. */ + apple,retention-workaround = <0x2504c 0x3 0x3>, <0x25070 0x1 0x0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index b4a0813268ebf..94bfc49c5e279 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -150,3 +150,19 @@ firmware-name = "apple/mesacal-j713.bin", "apple/msrk-j713.bin"; }; +/* + * The DCP powers its display domains down while blanked + * (appledrm.power_gate_ms): the clock protection slices of its DARTs, as the + * ADT has them, and the internal pipe's PMC interface. + */ +&dcp_dart { + apple,clock-protection = <&ps_prot 42>; +}; + +&disp0_dart { + apple,clock-protection = <&ps_prot 8>; +}; + +&dcp { + apple,pmc-interface = <&pmc_disp0>; +}; From 7f520fa2006068602587206edc2f86c846319a75 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:54:09 +1100 Subject: [PATCH 261/392] dt-bindings: pci: apple,pcie: allow a second power domain The internal PCIe controller on T6040 and T8112 needs two power domain chains to stay on while its links are up. Allow up to two entries. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- Documentation/devicetree/bindings/pci/apple,pcie.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 245231ecfc3a9..bbf26a8960b43 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -135,7 +135,13 @@ properties: iommu-map-mask: true power-domains: - maxItems: 1 + description: + The power domains the controller needs. Internal controllers on some + SoCs sit behind two separate domain chains (for example the PCIe + system and PHY domains), all of which must stay on while the links + are up. + minItems: 1 + maxItems: 2 resets: description: From 389034cd53054245df387431df885c1245cbf684 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:14:26 +1100 Subject: [PATCH 262/392] mmc: sdhci-pci-gli: model no-media board power sequencing Add a mutex-serialized sequencing core with ordered rail, port, config and endpoint operations, bounded retries, conservative failure cleanup and deferred helper events. Keep the live reader always powered until an independent insertion wake source and config-access exclusion can be supplied by a complete platform adapter. Build the core only with its KUnit suite. Cover transitions, ordering, partial failures, media races, concurrent events and suspend/resume with mocks, and document adapter requirements and hardware qualification. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/mmc/host/Kconfig | 10 + drivers/mmc/host/Makefile | 3 + drivers/mmc/host/sdhci-pci-gli-pwrseq-test.c | 500 ++++++++++++++++++ drivers/mmc/host/sdhci-pci-gli-pwrseq.c | 165 ++++++ drivers/mmc/host/sdhci-pci-gli-pwrseq.h | 73 +++ .../mmc/host/sdhci-pci-gli-pwrseq.kunitconfig | 3 + drivers/mmc/host/sdhci-pci-gli-pwrseq.rst | 107 ++++ 7 files changed, 861 insertions(+) create mode 100644 drivers/mmc/host/sdhci-pci-gli-pwrseq-test.c create mode 100644 drivers/mmc/host/sdhci-pci-gli-pwrseq.c create mode 100644 drivers/mmc/host/sdhci-pci-gli-pwrseq.h create mode 100644 drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig create mode 100644 drivers/mmc/host/sdhci-pci-gli-pwrseq.rst diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 4f060d3e56365..6b525bcdb3d7f 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -107,6 +107,16 @@ config MMC_SDHCI_UHS2 If you have a controller with this feature, say Y or M here. +config MMC_SDHCI_PCI_GLI_PWRSEQ_KUNIT_TEST + tristate "KUnit tests for GL9755 board power sequencing" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test the GL9755 no-media board power state machine with mock rail, + port and PCI operations. No hardware power gating is enabled: a + platform adapter with an always-on insertion wake source is needed. + These tests run without a PCI controller or an attached SD card. + config MMC_SDHCI_PCI tristate "SDHCI support on PCI bus" depends on MMC_SDHCI && PCI diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index ee412e6b84d6c..a065b54b4bbe5 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -115,3 +115,6 @@ endif obj-$(CONFIG_MMC_SDHCI_XENON) += sdhci-xenon-driver.o sdhci-xenon-driver-y += sdhci-xenon.o sdhci-xenon-phy.o + +obj-$(CONFIG_MMC_SDHCI_PCI_GLI_PWRSEQ_KUNIT_TEST) += gli-pwrseq-test.o +gli-pwrseq-test-y := sdhci-pci-gli-pwrseq.o sdhci-pci-gli-pwrseq-test.o diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq-test.c b/drivers/mmc/host/sdhci-pci-gli-pwrseq-test.c new file mode 100644 index 0000000000000..d13066ba623b7 --- /dev/null +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq-test.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +#include "sdhci-pci-gli-pwrseq.h" + +struct gli_mock { + struct kunit *test; + struct gli_pwrseq seq; + /* Models the outer PCI rescan/remove lock for concurrent events. */ + struct mutex bus_lock; + bool bus_locked; + bool rail; + bool port; + bool config; + bool endpoint; + int present; + char fail; + int failures; + bool insert_on_remove; + char log[256]; + unsigned int count; +}; + +static void gli_record(struct gli_mock *m, char op) +{ + KUNIT_EXPECT_TRUE(m->test, m->bus_locked); + KUNIT_EXPECT_TRUE(m->test, mutex_is_locked(&m->seq.lock)); + KUNIT_EXPECT_TRUE(m->test, !m->port || m->rail); + KUNIT_EXPECT_TRUE(m->test, !m->config || (m->port && m->rail)); + if (m->count < sizeof(m->log) - 1) + m->log[m->count++] = op; +} + +static int gli_fail(struct gli_mock *m, char op) +{ + if (m->fail != op || !m->failures) + return 0; + m->failures--; + return -EIO; +} + +static void gli_mock_lock(void *ctx) +{ + struct gli_mock *m = ctx; + + mutex_lock(&m->bus_lock); + KUNIT_EXPECT_FALSE(m->test, m->bus_locked); + m->bus_locked = true; +} + +static void gli_mock_unlock(void *ctx) +{ + struct gli_mock *m = ctx; + + KUNIT_EXPECT_FALSE(m->test, mutex_is_locked(&m->seq.lock)); + KUNIT_EXPECT_TRUE(m->test, m->bus_locked); + m->bus_locked = false; + mutex_unlock(&m->bus_lock); +} + +static int gli_mock_rail(void *ctx, bool on) +{ + struct gli_mock *m = ctx; + char op = on ? 'R' : 'r'; + int ret; + + gli_record(m, op); + KUNIT_EXPECT_EQ(m->test, m->seq.state, + on ? GLI_POWERING : GLI_QUIESCING); + if (!on) { + KUNIT_EXPECT_FALSE(m->test, m->port); + KUNIT_EXPECT_FALSE(m->test, m->config); + KUNIT_EXPECT_FALSE(m->test, m->endpoint); + } + ret = gli_fail(m, op); + /* Failed enables may have changed hardware, failed disables may not. */ + if (on || !ret) + m->rail = on; + return ret; +} + +static int gli_mock_port(void *ctx, bool on) +{ + struct gli_mock *m = ctx; + char op = on ? 'P' : 'p'; + int ret; + + gli_record(m, op); + KUNIT_EXPECT_TRUE(m->test, m->rail); + KUNIT_EXPECT_EQ(m->test, m->seq.state, + on ? GLI_POWERING : GLI_QUIESCING); + if (!on) { + KUNIT_EXPECT_FALSE(m->test, m->config); + KUNIT_EXPECT_FALSE(m->test, m->endpoint); + } + ret = gli_fail(m, op); + if (on || !ret) + m->port = on; + return ret; +} + +static void gli_mock_config(void *ctx, bool allow) +{ + struct gli_mock *m = ctx; + + gli_record(m, allow ? 'C' : 'c'); + if (allow) { + KUNIT_EXPECT_TRUE(m->test, m->rail && m->port); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_ENUMERATING); + } + m->config = allow; +} + +static int gli_mock_rescan(void *ctx) +{ + struct gli_mock *m = ctx; + + gli_record(m, 'E'); + KUNIT_EXPECT_TRUE(m->test, m->config); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_ENUMERATING); + m->endpoint = true; + return gli_fail(m, 'E'); +} + +static int gli_mock_media(void *ctx) +{ + struct gli_mock *m = ctx; + + gli_record(m, 'M'); + KUNIT_EXPECT_TRUE(m->test, m->config && m->endpoint); + return m->present; +} + +static int gli_mock_remove(void *ctx) +{ + struct gli_mock *m = ctx; + int ret; + + gli_record(m, 'Q'); + KUNIT_EXPECT_TRUE(m->test, m->config); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_QUIESCING); + if (m->insert_on_remove) + m->present = 1; + if (m->present > 0) + return -EBUSY; + ret = gli_fail(m, 'Q'); + if (!ret) + m->endpoint = false; + return ret; +} + +static int gli_mock_gen2(void *ctx) +{ + struct gli_mock *m = ctx; + + gli_record(m, 'G'); + KUNIT_EXPECT_TRUE(m->test, m->config && m->endpoint); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_ON_MEDIA); + return gli_fail(m, 'G'); +} + +static void gli_mock_delay(void *ctx, unsigned int ms) +{ + struct gli_mock *m = ctx; + + gli_record(m, ms == GLI_PWRSEQ_ON_MS ? 's' : 'd'); + if (ms == GLI_PWRSEQ_ON_MS) { + KUNIT_EXPECT_TRUE(m->test, m->rail); + KUNIT_EXPECT_FALSE(m->test, m->port); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_POWERING); + } else { + KUNIT_EXPECT_EQ(m->test, ms, GLI_PWRSEQ_OFF_MS); + KUNIT_EXPECT_FALSE(m->test, m->rail || m->port || m->config); + KUNIT_EXPECT_EQ(m->test, m->seq.state, GLI_QUIESCING); + } +} + +static const struct gli_pwrseq_ops gli_mock_ops = { + .bus_lock = gli_mock_lock, + .bus_unlock = gli_mock_unlock, + .rail = gli_mock_rail, + .port = gli_mock_port, + .config = gli_mock_config, + .rescan = gli_mock_rescan, + .media = gli_mock_media, + .remove = gli_mock_remove, + .gen2 = gli_mock_gen2, + .delay = gli_mock_delay, +}; + +static int gli_test_init(struct kunit *test) +{ + struct gli_mock *m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + + if (!m) + return -ENOMEM; + m->test = test; + mutex_init(&m->bus_lock); + gli_pwrseq_init(&m->seq, &gli_mock_ops, m); + test->priv = m; + return 0; +} + +static void gli_expect_off(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_OFF_NO_MEDIA); + KUNIT_EXPECT_FALSE(test, m->rail || m->port || m->config || m->endpoint); + KUNIT_EXPECT_FALSE(test, m->seq.rail_on || m->seq.port_on || m->seq.endpoint); +} + +static void gli_insert_remove_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->present = 1; + KUNIT_ASSERT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMG"); + /* Duplicate wake and stale removal indications cannot drop media. */ + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMGM"); + m->present = 0; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMGMMQcprd"); + gli_expect_off(test); + /* A fresh helper event can enumerate again after removal. */ + m->present = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); +} + +static void gli_empty_wake_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + KUNIT_EXPECT_STREQ(test, m->log, ""); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMQcprd"); + gli_expect_off(test); +} + +static void gli_retry_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + const char failures[] = { 'R', 'P', 'E' }; + const char * const logs[] = { + "RcrdRcrdRcrd", + "RsPcprdRsPcprdRsPcprd", + "RsPCEQcprdRsPCEQcprdRsPCEQcprd", + }; + int i; + + for (i = 0; i < ARRAY_SIZE(failures); i++) { + m->fail = failures[i]; + m->failures = GLI_PWRSEQ_ATTEMPTS; + m->count = 0; + memset(m->log, 0, sizeof(m->log)); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EIO); + KUNIT_EXPECT_EQ(test, m->failures, 0); + KUNIT_EXPECT_STREQ(test, m->log, logs[i]); + gli_expect_off(test); + } +} + +static void gli_retry_success_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->fail = 'P'; + m->failures = 1; + m->present = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_STREQ(test, m->log, "RsPcprdRsPCEMG"); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); +} + +static void gli_gen2_failure_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->present = 1; + m->fail = 'G'; + m->failures = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_TRUE(test, m->rail && m->port && m->endpoint); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMG"); +} + +static void gli_unknown_media_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->present = -EIO; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EIO); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), -EIO); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMM"); +} + +static void gli_reinsert_race_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->insert_on_remove = true; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EBUSY); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMQ"); + KUNIT_EXPECT_TRUE(test, m->rail && m->port && m->endpoint); + /* Repeat the reinsertion race after a confirmed ON_MEDIA removal. */ + m->present = 0; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), -EBUSY); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMQMQ"); +} + +static void gli_remove_failure_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->fail = 'Q'; + m->failures = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EIO); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMQ"); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + gli_expect_off(test); +} + +static void gli_partial_rescan_media_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + /* A failed rescan may already have exposed media to the MMC layer. */ + m->present = 1; + m->fail = 'E'; + m->failures = GLI_PWRSEQ_ATTEMPTS; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EBUSY); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEQ"); + KUNIT_EXPECT_EQ(test, m->failures, GLI_PWRSEQ_ATTEMPTS - 1); +} + +static void gli_shutdown_failure_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + const char failures[] = { 'p', 'r' }; + int i; + + for (i = 0; i < ARRAY_SIZE(failures); i++) { + m->fail = failures[i]; + m->failures = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EIO); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_QUIESCING); + KUNIT_EXPECT_TRUE(test, m->rail); + KUNIT_EXPECT_FALSE(test, m->config); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_SUSPEND), -EBUSY); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + gli_expect_off(test); + } +} + +static void gli_shutdown_pending_wake_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + m->fail = 'r'; + m->failures = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), -EIO); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_TRUE(test, m->seq.pending_insert); + m->present = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), 0); + KUNIT_EXPECT_FALSE(test, m->seq.pending_insert); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMQcprcrdRsPCEMG"); +} + +struct gli_worker { + struct gli_pwrseq *seq; + enum gli_pwrseq_event event; + struct completion done; + int ret; +}; + +static int gli_event_thread(void *data) +{ + struct gli_worker *worker = data; + + worker->ret = gli_pwrseq_event(worker->seq, worker->event); + complete(&worker->done); + return 0; +} + +static void gli_concurrent_events_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + struct gli_worker workers[3]; + struct task_struct *threads[3]; + int i, started = 0; + + /* Stale removal and two helper events race; media must stay powered. */ + m->present = 1; + for (i = 0; i < ARRAY_SIZE(workers); i++) { + workers[i].seq = &m->seq; + workers[i].event = i ? GLI_HELPER_INSERT : GLI_MEDIA_CHANGE; + init_completion(&workers[i].done); + threads[i] = kthread_create(gli_event_thread, &workers[i], "gli-event"); + if (IS_ERR(threads[i])) { + KUNIT_FAIL(test, "could not start event thread"); + break; + } + get_task_struct(threads[i]); + wake_up_process(threads[i]); + started++; + } + for (i = 0; i < started; i++) { + wait_for_completion(&workers[i].done); + kthread_stop(threads[i]); + put_task_struct(threads[i]); + KUNIT_EXPECT_EQ(test, workers[i].ret, 0); + } + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + /* Only one bring-up, regardless of which event won the locks. */ + KUNIT_EXPECT_EQ(test, m->count, m->log[7] ? 8 : 7); + KUNIT_EXPECT_EQ(test, m->log[0], 'R'); + KUNIT_EXPECT_EQ(test, m->log[6], 'G'); + KUNIT_EXPECT_TRUE(test, m->rail && m->port && m->endpoint); +} + +static void gli_suspend_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_SUSPEND), 0); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_RESUME), 0); + gli_expect_off(test); + KUNIT_EXPECT_STREQ(test, m->log, ""); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_SUSPEND), 0); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_HELPER_INSERT), 0); + KUNIT_EXPECT_TRUE(test, m->seq.pending_insert); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), -EBUSY); + KUNIT_EXPECT_STREQ(test, m->log, ""); + m->present = 1; + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_RESUME), 0); + KUNIT_EXPECT_FALSE(test, m->seq.pending_insert); + KUNIT_EXPECT_EQ(test, m->seq.state, GLI_ON_MEDIA); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMG"); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_SUSPEND), 0); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_MEDIA_CHANGE), -EBUSY); + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, GLI_RESUME), 0); + KUNIT_EXPECT_STREQ(test, m->log, "RsPCEMG"); +} + +static void gli_invalid_event_test(struct kunit *test) +{ + struct gli_mock *m = test->priv; + + KUNIT_EXPECT_EQ(test, gli_pwrseq_event(&m->seq, -1), -EINVAL); + gli_expect_off(test); + KUNIT_EXPECT_FALSE(test, m->bus_locked); +} + +static struct kunit_case gli_pwrseq_cases[] = { + KUNIT_CASE(gli_insert_remove_test), + KUNIT_CASE(gli_empty_wake_test), + KUNIT_CASE(gli_retry_test), + KUNIT_CASE(gli_retry_success_test), + KUNIT_CASE(gli_gen2_failure_test), + KUNIT_CASE(gli_unknown_media_test), + KUNIT_CASE(gli_reinsert_race_test), + KUNIT_CASE(gli_remove_failure_test), + KUNIT_CASE(gli_partial_rescan_media_test), + KUNIT_CASE(gli_shutdown_failure_test), + KUNIT_CASE(gli_shutdown_pending_wake_test), + KUNIT_CASE(gli_concurrent_events_test), + KUNIT_CASE(gli_suspend_test), + KUNIT_CASE(gli_invalid_event_test), + {} +}; + +static struct kunit_suite gli_pwrseq_suite = { + .name = "sdhci-pci-gli-pwrseq", + .init = gli_test_init, + .test_cases = gli_pwrseq_cases, +}; + +kunit_test_suite(gli_pwrseq_suite); + +MODULE_DESCRIPTION("GL9755 board power sequencing tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.c b/drivers/mmc/host/sdhci-pci-gli-pwrseq.c new file mode 100644 index 0000000000000..e446b8399ff1a --- /dev/null +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * GL9755 board power sequencing core. No hardware adapter is enabled until an + * always-on insertion wake source and config-access exclusion are available. + */ +#include +#include + +#include "sdhci-pci-gli-pwrseq.h" + +void gli_pwrseq_init(struct gli_pwrseq *seq, + const struct gli_pwrseq_ops *ops, void *ctx) +{ + *seq = (struct gli_pwrseq) { + .ops = ops, + .ctx = ctx, + .state = GLI_OFF_NO_MEDIA, + }; + mutex_init(&seq->lock); +} + +static int gli_pwrseq_off(struct gli_pwrseq *seq) +{ + const struct gli_pwrseq_ops *ops = seq->ops; + int ret; + + lockdep_assert_held(&seq->lock); + seq->state = GLI_QUIESCING; + if (seq->endpoint) { + ret = ops->remove(seq->ctx); + if (ret) { + /* A new card or outstanding I/O vetoes power gating. */ + seq->state = GLI_ON_MEDIA; + return ret; + } + seq->endpoint = false; + } + + ops->config(seq->ctx, false); + if (seq->port_on) { + ret = ops->port(seq->ctx, false); + if (ret) + return ret; + seq->port_on = false; + } + if (seq->rail_on) { + ret = ops->rail(seq->ctx, false); + if (ret) + return ret; + seq->rail_on = false; + } + ops->delay(seq->ctx, GLI_PWRSEQ_OFF_MS); + seq->state = GLI_OFF_NO_MEDIA; + return 0; +} + +static int gli_pwrseq_insert(struct gli_pwrseq *seq) +{ + const struct gli_pwrseq_ops *ops = seq->ops; + int attempt, ret, cleanup; + + lockdep_assert_held(&seq->lock); + for (attempt = 0; attempt < GLI_PWRSEQ_ATTEMPTS; attempt++) { + seq->state = GLI_POWERING; + /* Track possibly partial writes, including failed enables. */ + seq->rail_on = true; + ret = ops->rail(seq->ctx, true); + if (ret) + goto failed; + ops->delay(seq->ctx, GLI_PWRSEQ_ON_MS); + seq->port_on = true; + ret = ops->port(seq->ctx, true); + if (ret) + goto failed; + seq->state = GLI_ENUMERATING; + ops->config(seq->ctx, true); + seq->endpoint = true; + ret = ops->rescan(seq->ctx); + if (ret) + goto failed; + ret = ops->media(seq->ctx); + if (ret < 0) { + /* Unknown presence cannot authorize dropping a card's power. */ + seq->state = GLI_ON_MEDIA; + return ret; + } + if (!ret) + return gli_pwrseq_off(seq); + seq->state = GLI_ON_MEDIA; + /* Speed is optional; never power-cycle media to obtain Gen2. */ + ops->gen2(seq->ctx); + return 0; +failed: + cleanup = gli_pwrseq_off(seq); + if (cleanup) + return cleanup; + } + return ret; +} + +int gli_pwrseq_event(struct gli_pwrseq *seq, enum gli_pwrseq_event event) +{ + int ret = 0; + + /* This entry point is for threaded IRQ/workqueue and PM callbacks. */ + seq->ops->bus_lock(seq->ctx); + mutex_lock(&seq->lock); + + switch (event) { + case GLI_SUSPEND: + if (seq->state == GLI_QUIESCING) { + ret = -EBUSY; + break; + } + seq->suspended = true; + break; + case GLI_RESUME: + seq->suspended = false; + if (!seq->pending_insert) + break; + /* Service only a helper event deferred across suspend. */ + seq->pending_insert = false; + fallthrough; + case GLI_HELPER_INSERT: + if (seq->suspended) { + seq->pending_insert = true; + break; + } + if (seq->state == GLI_QUIESCING) { + seq->pending_insert = true; + break; + } + if (seq->state == GLI_OFF_NO_MEDIA) + ret = gli_pwrseq_insert(seq); + break; + case GLI_MEDIA_CHANGE: + if (seq->suspended) { + ret = -EBUSY; + break; + } + if (seq->state == GLI_QUIESCING) { + /* Retry failed shutdown without touching endpoint config. */ + ret = gli_pwrseq_off(seq); + if (!ret && seq->pending_insert) { + seq->pending_insert = false; + ret = gli_pwrseq_insert(seq); + } + break; + } + if (seq->state != GLI_ON_MEDIA) + break; + ret = seq->ops->media(seq->ctx); + if (!ret) + ret = gli_pwrseq_off(seq); + else if (ret > 0) + ret = 0; + break; + default: + ret = -EINVAL; + } + + mutex_unlock(&seq->lock); + seq->ops->bus_unlock(seq->ctx); + return ret; +} diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.h b/drivers/mmc/host/sdhci-pci-gli-pwrseq.h new file mode 100644 index 0000000000000..c12620249069d --- /dev/null +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SDHCI_PCI_GLI_PWRSEQ_H +#define SDHCI_PCI_GLI_PWRSEQ_H + +#include +#include + +#define GLI_PWRSEQ_ATTEMPTS 3 +#define GLI_PWRSEQ_ON_MS 100 +#define GLI_PWRSEQ_OFF_MS 250 + +enum gli_pwrseq_state { + GLI_OFF_NO_MEDIA, + GLI_POWERING, + GLI_ENUMERATING, + GLI_ON_MEDIA, + GLI_QUIESCING, +}; + +enum gli_pwrseq_event { + GLI_HELPER_INSERT, + GLI_MEDIA_CHANGE, + GLI_SUSPEND, + GLI_RESUME, +}; + +/* + * Sleepable operations, never called from an IRQ or re-entered by callbacks. + * bus_lock/unlock must implement pci_lock_rescan_remove()/unlock in an adapter. + * Lock order: PCI rescan/remove lock, then the single power-state mutex. + * + * rail/port may partially succeed on error. port(false) must synchronously + * stop link/config traffic before rail(false). config(false) must exclude + * and drain all config access, including users outside this state machine. + * port(true) enables and trains at Gen1; gen2 must retain Gen1 on failure. + * rescan may partially enumerate on error. remove must drain I/O, DMA and + * endpoint interrupts, publish removal and remove the endpoint. It must + * recheck debounced card presence and refuse with -EBUSY if media or I/O owns + * the slot, leaving it usable. media returns debounced, polarity-corrected + * SDHCI card presence (0/1), or a negative error. The helper is only a hint. + */ +struct gli_pwrseq_ops { + void (*bus_lock)(void *ctx); + void (*bus_unlock)(void *ctx); + int (*rail)(void *ctx, bool on); + int (*port)(void *ctx, bool on); + void (*config)(void *ctx, bool allow); + int (*rescan)(void *ctx); + int (*media)(void *ctx); + int (*remove)(void *ctx); + int (*gen2)(void *ctx); + void (*delay)(void *ctx, unsigned int ms); +}; + +struct gli_pwrseq { + /* Serializes all state and callbacks, inside the PCI bus lock. */ + struct mutex lock; + const struct gli_pwrseq_ops *ops; + void *ctx; + enum gli_pwrseq_state state; + bool rail_on; + bool port_on; + bool endpoint; + bool suspended; + bool pending_insert; +}; + +/* Initialize only an already-off, empty slot with config access blocked. */ +void gli_pwrseq_init(struct gli_pwrseq *seq, + const struct gli_pwrseq_ops *ops, void *ctx); +int gli_pwrseq_event(struct gli_pwrseq *seq, enum gli_pwrseq_event event); + +#endif diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig b/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig new file mode 100644 index 0000000000000..642f88b5255ad --- /dev/null +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_MMC=y +CONFIG_MMC_SDHCI_PCI_GLI_PWRSEQ_KUNIT_TEST=y diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.rst b/drivers/mmc/host/sdhci-pci-gli-pwrseq.rst new file mode 100644 index 0000000000000..deabf7a9ef5f5 --- /dev/null +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.rst @@ -0,0 +1,107 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +GL9755 board power sequencing +============================ + +Status +------ + +The sequencing core is built only by the KUnit option. It has no live PCI, +GPIO, IRQ or MMC adapter and does not alter the always-powered SD reader. +In particular, do not initialize it against an enumerated, powered endpoint: +its initial condition is an empty, fully powered-off slot with config access +already excluded. A future adapter must safely establish that condition. + +The SMC GPIO driver currently has no IRQ implementation. The Apple SD root +port has no independent insertion interrupt binding or callback that can +restore the rail, retrain and rescan. Endpoint MSI/card-detect interrupts +cannot wake an endpoint whose rail is off. Therefore no new DT property, +module parameter or runtime power switch is provided here. + +Adapter requirements +-------------------- + +* Prove an always-on insertion event, including its wake behavior in s2idle, + before enabling no-media gating. The helper event is a wake hint; steady + card presence comes from debounced SDHCI state with the board polarity. +* Deliver helper events through threaded IRQ/work, never hard IRQ context. + All events take the PCI rescan/remove lock before the power-state mutex. + Callback implementations must not acquire these locks recursively. MMC + teardown must not wait for work that is itself blocked on either lock. +* Exclude and drain external config accesses before disabling the port. A + rescan lock alone does not exclude sysfs config access, PCI error recovery, + or PM callbacks. The adapter must supply that exclusion as well as order + recovery, suspend and rescan against this state machine. +* Power the rail, wait 100 ms, enable/train the port at Gen1, allow config, + then rescan. Request Gen2 only with a usable endpoint; a Gen2 failure must + retain Gen1 and must not cycle media power. +* Quiesce MMC and remove the PCI endpoint before blocking config, disabling + the port and dropping the rail. Recheck card presence during quiesce and + veto removal if media or outstanding I/O still owns it. Keep the helper + armed throughout the off state. Never use the unidentified I2C sideband. +* Failed enable operations may have partially changed hardware. Cleanup + disables the port before the rail, and waits 250 ms before a retry. There + are at most three bring-up attempts. A cleanup error stops retries. A + failed port disable retains the rail; a failed rail disable retains its + conservative on state. Neither is reported as OFF_NO_MEDIA. A subsequent + media-change event retries shutdown without accessing endpoint config. + A helper received in this error state is retained until cleanup succeeds. +* Unknown card presence or a quiesce veto retains power. Suspend retains a + live endpoint, leaves an off slot off, and defers helper work to resume. + Resume without a pending helper event never accesses an off endpoint. + +Tests +----- + +Run on a Linux build host:: + + python3 tools/testing/kunit/kunit.py run --jobs=8 \ + --build_dir=out/kunit-gli \ + --kunitconfig=drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig \ + 'sdhci-pci-gli-pwrseq*' + +Mocks check the bus lock and state mutex at every operation, intermediate +states, rail/port/config ordering, removal before power-off, bounded retry +and both delays. Faults cover partially successful enables, enumeration, +card-state reads, quiesce, port/rail shutdown and Gen2. Empty/spurious wakes, +stale removals, reinsertion at quiesce, repeated wakes, concurrent event +threads and deferred suspend work exercise the media decisions. No tests access hardware. + +Hardware qualification plan +--------------------------- + +First verify this preparatory change on a J316s with the normal driver and +DT: no-card boot must retain the existing powered/enumerated reader, and +insert, mount/read, unmount, remove, reinsert and s2idle must behave as before. +This change alone cannot demonstrate a power saving. + +After a complete adapter is separately reviewed, run the following with +controller-authorized access to the intended machine and a disposable test +card. Capture kernel logs, state/callback traces and PCI topology. Observe +rail state only through a qualified GPIO readback or power measurement. + +1. No-card boot: establish the empty-slot condition, quiesce/remove the + GL9755, block config, disable the port, then drop the rail. Verify no + config request reaches the off endpoint and the helper stays armed. +2. Insert: verify helper wake, rail on, at least 100 ms settling, Gen1 port + training and rescan before Gen2. Confirm exactly one host is registered. +3. Mount/read: read a known file and verify its digest. Keep it mounted across + idle and s2idle; verify no rail or port disable, no DMA/IOMMU errors and + preserved data. Never physically remove a mounted test filesystem. +4. Unmount and remove: verify removal/debounce and MMC/block teardown before + PCI removal, then port off before rail off. Verify at least 250 ms off + time. Reinsert rapidly during debounce and quiesce to exercise veto and + queued insertion handling; verify eventual enumeration without duplicate + hosts or stale block devices. +5. Suspend/resume: repeat with no card, with a mounted card, with insertion + during suspend and removal before suspend. Verify insertion wakes s2idle + when off, and resume never blindly accesses config with the rail off. +6. Fault injection: fail rail enable, Gen1 training and rescan separately. + Verify three attempts maximum and both off after successful cleanup. + Fail Gen2 and verify usable Gen1 without cycling media. Fail shutdown and + verify a truthful error state, config exclusion and no unsafe rail drop. +7. Confirm no new I2C client, scan or transaction to the unidentified sideband + address occurs throughout the suite. + +Rollback is the prior kernel/DT with the normal always-powered reader. Keep +failure logs and traces; unmount the test card before switching kernels. From a9a6b2fbe9ae64946710b20e76bb0d48c5d4a0ed Mon Sep 17 00:00:00 2001 From: "Michal S." Date: Mon, 9 Nov 2026 13:15:40 +0000 Subject: [PATCH 263/392] wifi: brcmfmac: preserve the requested RSNE for WCC SAE offload Clearing wpaie in the common SAE path discards userspace cipher constraints. On BCM4388, WCC firmware then constructs a GCMP-256 association RSNE for a CCMP-only request, which supplicant rejects with reason 19. Retaining the requested RSNE produces a CCMP SAE connection, and firmware still appends its PMKID. Keep CYW station/client RSNE clearing in its firmware vendor callback. Leave AP password setup unchanged and propagate clearing failures. Cover the production CONNECT offload stage and WCC/CYW callbacks with 13 host controls and an original-code negative control. Hardware validation is limited to J614s. Signed-off-by: Michal S. Assisted-By: LLM --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 7 - .../broadcom/brcm80211/brcmfmac/cyw/core.c | 10 + tools/asahi/brcmf-sae-vendor-tests.py | 325 ++++++++++++++++++ tools/asahi/run-host-tests.py | 2 + 4 files changed, 337 insertions(+), 7 deletions(-) create mode 100755 tools/asahi/brcmf-sae-vendor-tests.py diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 65d38c4b0952b..398d252c3928a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2595,13 +2595,6 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, err = brcmf_set_pmk(ifp, params->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - /* clean up user-space RSNE */ - if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { - bphy_err( - drvr, - "failed to clean up user-space RSNE\n"); - goto done; - } err = brcmf_fwvid_set_sae_password(ifp, ¶ms->crypto); if (!err && params->crypto.psk) err = brcmf_set_pmk(ifp, params->crypto.psk, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c index 6d5098f6f00f9..ababf34df8b0f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c @@ -31,6 +31,16 @@ static int brcmf_cyw_set_sae_pwd(struct brcmf_if *ifp, u16 pwd_len = crypto->sae_pwd_len; int err; + /* CYW firmware generates the station RSNE during SAE offload. */ + if (ifp->vif->wdev.iftype == NL80211_IFTYPE_STATION || + ifp->vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT) { + err = brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0); + if (err) { + bphy_err(drvr, "failed to clean up user-space RSNE\n"); + return err; + } + } + if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { bphy_err(drvr, "sae_password must be less than %d\n", BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); diff --git a/tools/asahi/brcmf-sae-vendor-tests.py b/tools/asahi/brcmf-sae-vendor-tests.py new file mode 100755 index 0000000000000..1d8052c5ecdac --- /dev/null +++ b/tools/asahi/brcmf-sae-vendor-tests.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise SAE offload RSNE handling through the firmware vendor callbacks.""" + +import argparse +import importlib.util +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = Path("drivers/net/wireless/broadcom/brcm80211/brcmfmac") +spec = importlib.util.spec_from_file_location( + "association", ROOT / "tools/asahi/brcmf-assoc-tests.py" +) +association = importlib.util.module_from_spec(spec) +spec.loader.exec_module(association) + +PREFIX = r""" + +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint16_t __le16; +#define BIT(n) (1U << (n)) +#define WARN_ON(condition) (condition) +#define brcmf_dbg(...) ((void)0) +#define bphy_err(context, ...) ((void)(context)) +#define BRCMF_FEAT_FWSUP 1 +enum { + NL80211_IFTYPE_STATION, + NL80211_IFTYPE_P2P_CLIENT, + NL80211_IFTYPE_AP, + NL80211_IFTYPE_P2P_GO +}; +struct cfg80211_crypto_settings { + const u8 *sae_pwd, *psk; + u16 sae_pwd_len; +}; +struct cfg80211_connect_params { + struct cfg80211_crypto_settings crypto; +}; +struct brcmf_if; +struct brcmf_fwvid_ops { + int (*set_sae_password)(struct brcmf_if *, + struct cfg80211_crypto_settings *); +}; +struct brcmf_pub { + const struct brcmf_fwvid_ops *vops; +}; +struct brcmf_profile { + int use_fwsup; +}; +struct brcmf_vif { + struct { + int iftype; + } wdev; + struct brcmf_profile profile; +}; +struct brcmf_if { + struct brcmf_pub *drvr; + struct brcmf_vif *vif; +}; +enum { CLEAR_RSNE, CYW_PASSWORD, WCC_PASSWORD, SET_PMK }; +static int commands[8], count; +static int clear_error, password_error, pmk_error, sup_error; +static bool fwsup; +static u16 cpu_to_le16(u16 value) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return value; +#else + return __builtin_bswap16(value); +#endif +} +static bool brcmf_feat_is_enabled(struct brcmf_if *ifp, int feature) +{ + (void)ifp; + assert(feature == BRCMF_FEAT_FWSUP); + return fwsup; +} +static int brcmf_fil_iovar_int_set(struct brcmf_if *ifp, const char *name, + int value) +{ + (void)ifp; + assert(!strcmp(name, "sup_wpa") && (value == 0 || value == 1)); + return sup_error; +} +static int brcmf_fil_iovar_data_set(struct brcmf_if *ifp, const char *name, + const void *data, unsigned int length) +{ + (void)ifp; + assert(count < 8); + if (!strcmp(name, "wpaie")) { + assert(!data && !length); + commands[count++] = CLEAR_RSNE; + return clear_error; + } + assert(!strcmp(name, "sae_password") && data && length); + commands[count++] = CYW_PASSWORD; + return password_error; +} +static int brcmf_set_wsec(struct brcmf_if *ifp, const u8 *password, u16 length, + unsigned int flags) +{ + (void)ifp; + assert(password && length == 8 && flags == BRCMF_WSEC_PASSPHRASE); + commands[count++] = WCC_PASSWORD; + return password_error; +} +static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk, u16 length) +{ + (void)ifp; + assert(pmk && length == BRCMF_WSEC_MAX_PSK_LEN); + commands[count++] = SET_PMK; + return pmk_error; +} +""" + +MAIN = r""" + +static const struct brcmf_fwvid_ops wcc = { .set_sae_password = + brcmf_wcc_set_sae_pwd }; +static const struct brcmf_fwvid_ops cyw = { .set_sae_password = + brcmf_cyw_set_sae_pwd }; +static struct brcmf_pub pub; +static struct brcmf_vif vif; +static struct brcmf_if ifp = { &pub, &vif }; +static struct cfg80211_connect_params params; +static const u8 password[] = "test-sae"; +static const u8 pmk[BRCMF_WSEC_MAX_PSK_LEN]; +static void reset(const struct brcmf_fwvid_ops *vendor, int type) +{ + pub.vops = vendor; + vif.wdev.iftype = type; + vif.profile.use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + params.crypto = (struct cfg80211_crypto_settings){ + .sae_pwd = password, + .sae_pwd_len = sizeof(password) - 1, + }; + count = clear_error = password_error = pmk_error = sup_error = 0; + fwsup = true; +} +int main(void) +{ + unsigned int controls = 0; + reset(&wcc, NL80211_IFTYPE_STATION); + assert(!offload_setup(&ifp, ¶ms)); + assert(count == 1 && commands[0] == WCC_PASSWORD); + controls++; + + const int types[] = { NL80211_IFTYPE_STATION, NL80211_IFTYPE_P2P_CLIENT, + NL80211_IFTYPE_AP, NL80211_IFTYPE_P2P_GO }; + for (unsigned int i = 0; i < 4; i++) { + reset(&cyw, types[i]); + if (i < 2) { + assert(!offload_setup(&ifp, ¶ms)); + assert(count == 2 && commands[0] == CLEAR_RSNE && + commands[1] == CYW_PASSWORD); + } else { + assert(!brcmf_fwvid_set_sae_password(&ifp, + ¶ms.crypto)); + assert(count == 1 && commands[0] == CYW_PASSWORD); + } + controls++; + } + reset(&cyw, NL80211_IFTYPE_STATION); + clear_error = -EIO; + assert(offload_setup(&ifp, ¶ms) == -EIO); + assert(count == 1 && commands[0] == CLEAR_RSNE); + controls++; + + reset(&cyw, NL80211_IFTYPE_STATION); + password_error = -EIO; + params.crypto.psk = pmk; + assert(offload_setup(&ifp, ¶ms) == -EIO); + assert(count == 2 && commands[1] == CYW_PASSWORD); + controls++; + + reset(&wcc, NL80211_IFTYPE_STATION); + password_error = -EIO; + assert(offload_setup(&ifp, ¶ms) == -EIO); + assert(count == 1 && commands[0] == WCC_PASSWORD); + controls++; + + reset(&wcc, NL80211_IFTYPE_STATION); + params.crypto.psk = pmk; + assert(!offload_setup(&ifp, ¶ms)); + assert(count == 2 && commands[0] == WCC_PASSWORD && + commands[1] == SET_PMK); + controls++; + + reset(&wcc, NL80211_IFTYPE_STATION); + params.crypto.psk = pmk; + pmk_error = -EIO; + assert(offload_setup(&ifp, ¶ms) == -EIO); + assert(count == 2 && commands[1] == SET_PMK); + controls++; + + reset(&wcc, NL80211_IFTYPE_STATION); + vif.profile.use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + params.crypto.psk = pmk; + assert(!offload_setup(&ifp, ¶ms)); + assert(count == 1 && commands[0] == SET_PMK); + controls++; + + reset(NULL, NL80211_IFTYPE_STATION); + assert(offload_setup(&ifp, ¶ms) == -EOPNOTSUPP); + assert(!count); + controls++; + + reset(&wcc, NL80211_IFTYPE_STATION); + sup_error = -EIO; + assert(offload_setup(&ifp, ¶ms) == -EIO && !count); + controls++; + + printf("%u SAE vendor setup controls passed\n", controls); + return 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref") + parser.add_argument("--out", type=Path) + args = parser.parse_args() + + def read(name): + path = DRIVER / name + if args.ref: + return subprocess.check_output( + ["git", "show", f"{args.ref}:{path}"], cwd=ROOT, text=True + ) + return (ROOT / path).read_text() + + cfg = read("cfg80211.c") + header = read("cfg80211.h") + types = read("fwil_types.h") + start = header.index("enum brcmf_profile_fwsup {") + constants = re.findall( + r"^#define\s+(?:BRCMF_WSEC_MAX_(?:PSK_LEN|SAE_PASSWORD_LEN)|BRCMF_WSEC_PASSPHRASE)\s+.*$", + types, + re.M, + ) + # Execute the production CONNECT offload stage, including its error exits. + connect = association.function(cfg, "brcmf_cfg80211_connect") + start_block = connect.index("\tif (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) {") + opening = connect.index("{", start_block) + depth, end = 1, opening + 1 + while depth: + depth += (connect[end] == "{") - (connect[end] == "}") + end += 1 + setup = """ +static int offload_setup(struct brcmf_if *ifp, struct cfg80211_connect_params *params) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_profile *profile = &ifp->vif->profile; + int err = 0; +""" + connect[start_block:end] + "\ndone:\n return err;\n}\n" + parts = [ + "\n".join(constants), + PREFIX, + header[start : header.index("};", start) + 2], + association.structure(types, "brcmf_wsec_sae_pwd_le"), + association.function(read("cyw/core.c"), "brcmf_cyw_set_sae_pwd"), + association.function(read("wcc/core.c"), "brcmf_wcc_set_sae_pwd"), + association.function(read("fwvid.h"), "brcmf_fwvid_set_sae_password"), + setup, + MAIN, + ] + + def run(out): + out.mkdir(parents=True, exist_ok=True) + source, binary = out / "controls.c", out / "controls" + source.write_text("\n".join(parts)) + subprocess.run( + [ + os.environ.get("CC", "clang"), + "-std=gnu11", + "-O1", + "-g", + "-Wall", + "-Wextra", + "-Werror", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=all", + str(source), + "-o", + str(binary), + ], + check=True, + ) + result = subprocess.run([str(binary)], capture_output=True, text=True) + (out / "results.json").write_text( + json.dumps( + { + "exit": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "limits": "CONNECT offload stage and vendor callbacks with host firmware stubs; no radio.", + }, + indent=2, + ) + + "\n" + ) + print(result.stdout or result.stderr, end="") + return result.returncode != 0 + + if args.out: + return run(args.out.resolve()) + with tempfile.TemporaryDirectory(prefix="brcmf-sae-vendor-") as out: + return run(Path(out)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 9c7816cdba386..e999457ca8d04 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -93,6 +93,8 @@ def mutations(flag, names, base, expect=KILL): 'brcmf-pcie-dma-tests.py': [(out('--output'), PASS)] + mutations('--mutation', ('narrow-stream', 'ignore-aperture', 'revision-wide'), out('--output')), 'brcmf-roam-tests.py': [(out(), PASS)], + 'brcmf-sae-vendor-tests.py': [(out(), PASS)] + + [(out() + ['--ref', '9260e1f021e075713931f3c23c0a5fa1da936b0b'], KILL)], 'brcmf-suspend-tests.py': [(out(), PASS)] + mutations('--mutation', ('country-gate', 'queue-reasons', 'worker-gate', 'resume-order', 'tx-drain'), out()), From fdc0471b71abb18c1affb79bec67dac2ae0c75c2 Mon Sep 17 00:00:00 2001 From: "Michal S." Date: Mon, 9 Nov 2026 13:15:40 +0000 Subject: [PATCH 264/392] wifi: brcmfmac: use firmware join scan budgets with AWDL On BCM4388, fixed channel-directed join dwell and probe budgets can stall association with AWDL enabled. Use documented firmware defaults when an AWDL interface shares the radio, preserving channel and BSSID hints and the existing budget without AWDL. Keep AWDL configuration and lifecycle intact. Test both join formats and register an original-code negative control. J614s boot, reconnect and subsequent AirDrop transfer passed; quick firmware join failures requiring automatic retries remain unresolved. Signed-off-by: Michal S. Assisted-By: LLM --- .../broadcom/brcm80211/brcmfmac/core.c | 21 ++ .../broadcom/brcm80211/brcmfmac/core.h | 1 + .../broadcom/brcm80211/brcmfmac/join_param.c | 13 +- tools/asahi/brcmf-join-scan-tests.py | 302 ++++++++++++++++++ tools/asahi/run-host-tests.py | 2 + 5 files changed, 332 insertions(+), 7 deletions(-) create mode 100755 tools/asahi/brcmf-join-scan-tests.py diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 2b6ec64fe0bbc..329c40dac2db7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -767,6 +767,27 @@ static const struct net_device_ops brcmf_netdev_ops_pri = { .ndo_set_rx_mode = brcmf_netdev_set_multicast_list }; +bool brcmf_has_awdl_interface(struct brcmf_pub *drvr) +{ + struct net_device *ndev; + struct brcmf_if *ifp; + bool present = false; + + rcu_read_lock(); + for_each_netdev_rcu(wiphy_net(drvr->wiphy), ndev) { + if (ndev->netdev_ops != &brcmf_netdev_ops_awdl) + continue; + ifp = netdev_priv(ndev); + if (ifp->drvr == drvr) { + present = true; + break; + } + } + rcu_read_unlock(); + + return present; +} + int brcmf_net_attach(struct brcmf_if *ifp, bool locked) { struct brcmf_pub *drvr = ifp->drvr; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 229c24abad328..f7b31b17becfb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -283,6 +283,7 @@ char *brcmf_ifname(struct brcmf_if *ifp); struct brcmf_if *brcmf_get_ifp(struct brcmf_pub *drvr, int ifidx); void brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable); int brcmf_net_attach(struct brcmf_if *ifp, bool locked); +bool brcmf_has_awdl_interface(struct brcmf_pub *drvr); struct brcmf_if *brcmf_add_if(struct brcmf_pub *drvr, s32 bsscfgidx, s32 ifidx, bool is_p2pdev, const char *name, u8 *mac_addr); void brcmf_remove_interface(struct brcmf_if *ifp, bool locked); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c index 4f026571c7e7e..5dba0fc7968ff 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c @@ -118,14 +118,15 @@ brcmf_get_prepped_struct_for_ibss_v1(struct brcmf_cfg80211_info *cfg, } static void -brcmf_joinscan_set_common_v0v1_params(struct brcmf_join_scan_params_le *scan_le, - bool have_channel) +brcmf_joinscan_set_common_v0v1_params(struct brcmf_cfg80211_info *cfg, + struct brcmf_join_scan_params_le *scan_le) { /* Set up join scan parameters */ scan_le->scan_type = 0; scan_le->home_time = cpu_to_le32(-1); - if (have_channel) { + /* Let firmware budget the scan around AWDL availability windows. */ + if (cfg->channel && !brcmf_has_awdl_interface(cfg->pub)) { /* Increase dwell time to receive probe response or detect * beacon from target AP at a noisy air only during connect * command. @@ -166,8 +167,7 @@ brcmf_get_struct_for_connect_v0(struct brcmf_cfg80211_info *cfg, } brcmf_joinscan_set_ssid(&ext_v0->ssid_le, params->ssid, params->ssid_len); - brcmf_joinscan_set_common_v0v1_params(&ext_v0->scan_le, - cfg->channel != 0); + brcmf_joinscan_set_common_v0v1_params(cfg, &ext_v0->scan_le); brcmf_joinscan_set_bssid(ext_v0->assoc_le.bssid, params->bssid); if (cfg->channel) { struct ieee80211_channel *chan = params->channel_hint ? @@ -201,8 +201,7 @@ brcmf_get_struct_for_connect_v1(struct brcmf_cfg80211_info *cfg, ext_v1->assoc_le.version = cpu_to_le16(1); brcmf_joinscan_set_ssid(&ext_v1->ssid_le, params->ssid, params->ssid_len); - brcmf_joinscan_set_common_v0v1_params(&ext_v1->scan_le, - cfg->channel != 0); + brcmf_joinscan_set_common_v0v1_params(cfg, &ext_v1->scan_le); brcmf_joinscan_set_bssid(ext_v1->assoc_le.bssid, params->bssid); if (cfg->channel) { struct ieee80211_channel *chan = params->channel_hint ? diff --git a/tools/asahi/brcmf-join-scan-tests.py b/tools/asahi/brcmf-join-scan-tests.py new file mode 100755 index 0000000000000..2ce46e52e55e0 --- /dev/null +++ b/tools/asahi/brcmf-join-scan-tests.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the v0/v1 join constructors with and without AWDL.""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = Path("drivers/net/wireless/broadcom/brcm80211/brcmfmac") + +PREFIX = r""" + +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint16_t __le16; +typedef uint32_t __le32; +#define ETH_ALEN 6 +#define IEEE80211_MAX_SSID_LEN 32 +#define GFP_KERNEL 0 +#define bphy_err(...) ((void)0) +#define brcmf_dbg(...) ((void)0) +#define min_t(type, a, b) ((type)(a) < (type)(b) ? (type)(a) : (type)(b)) +#define struct_size(p, member, count) \ + (sizeof(*(p)) + sizeof((p)->member[0]) * (count)) +static uint16_t cpu_to_le16(uint16_t value) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return value; +#else + return __builtin_bswap16(value); +#endif +} +static uint32_t cpu_to_le32(uint32_t value) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return value; +#else + return __builtin_bswap32(value); +#endif +} +#define le32_to_cpu(value) cpu_to_le32(value) +struct brcmf_pub { + bool awdl; +}; +struct brcmf_cfg80211_info { + u32 channel; + unsigned int d11inf; + struct brcmf_pub *pub; +}; +struct ieee80211_channel { + u16 chanspec; +}; +struct cfg80211_connect_params { + const u8 *ssid; + size_t ssid_len; + const u8 *bssid; + struct ieee80211_channel *channel, *channel_hint; +}; +bool brcmf_has_awdl_interface(struct brcmf_pub *pub) +{ + return pub->awdl; +} +static void *kzalloc(size_t size, int flags) +{ + (void)flags; + return calloc(1, size); +} +static void eth_broadcast_addr(u8 *address) +{ + memset(address, 0xff, ETH_ALEN); +} +static u16 channel_to_chanspec(unsigned int *d11inf, + struct ieee80211_channel *channel) +{ + (void)d11inf; + return channel->chanspec; +} +""" + +MAIN = r""" + +int main(void) +{ + const u8 ssid[] = "test"; + const u8 bssid[ETH_ALEN] = { 0x02, 0, 0, 0, 0, 1 }; + struct ieee80211_channel channel = { .chanspec = 0xd034 }; + unsigned int controls = 0; + + for (int version = 0; version <= 1; version++) { + for (int selection = 0; selection < 3; selection++) { + for (int awdl = 0; awdl <= 1; awdl++) { + struct brcmf_pub pub = { .awdl = awdl }; + struct brcmf_cfg80211_info cfg = { + .channel = selection ? 52 : 0, + .pub = &pub, + }; + struct cfg80211_connect_params params = { + .ssid = ssid, + .ssid_len = sizeof(ssid) - 1, + .bssid = bssid, + .channel_hint = selection == 1 ? + &channel : + NULL, + .channel = selection == 2 ? &channel : + NULL, + }; + u32 size; + struct brcmf_ext_join_params_le *v0 = NULL; + struct brcmf_ext_join_params_v1_le *v1 = NULL; + struct brcmf_join_scan_params_le *scan; + struct brcmf_ssid_le *sent_ssid; + const u8 *sent_bssid; + u32 channels; + u16 chanspec = 0; + bool extended = selection && !awdl; + + if (version) { + v1 = brcmf_get_struct_for_connect_v1( + &cfg, &size, ¶ms); + assert(v1); + assert(v1->version == cpu_to_le16(1)); + assert(v1->assoc_le.version == + cpu_to_le16(1)); + assert(v1->assoc_le.flags == 0); + scan = &v1->scan_le; + sent_ssid = &v1->ssid_le; + sent_bssid = v1->assoc_le.bssid; + channels = le32_to_cpu( + v1->assoc_le.chanspec_num); + if (selection) + chanspec = + v1->assoc_le + .chanspec_list[0]; + assert(size == + sizeof(*v1) + + (selection ? + sizeof(u16) : + 0)); + } else { + v0 = brcmf_get_struct_for_connect_v0( + &cfg, &size, ¶ms); + assert(v0); + scan = &v0->scan_le; + sent_ssid = &v0->ssid_le; + sent_bssid = v0->assoc_le.bssid; + channels = le32_to_cpu( + v0->assoc_le.chanspec_num); + if (selection) + chanspec = + v0->assoc_le + .chanspec_list[0]; + assert(size == + sizeof(*v0) + + (selection ? + sizeof(u16) : + 0)); + } + + assert(scan->scan_type == 0); + assert(scan->home_time == cpu_to_le32(-1)); + assert(scan->active_time == + cpu_to_le32(extended ? 320 : -1)); + assert(scan->passive_time == + cpu_to_le32(extended ? 400 : -1)); + assert(scan->nprobes == + cpu_to_le32(extended ? 16 : -1)); + assert(channels == (selection ? 1U : 0U)); + if (selection) + assert(chanspec == + cpu_to_le16(channel.chanspec)); + assert(le32_to_cpu(sent_ssid->SSID_len) == + sizeof(ssid) - 1); + assert(!memcmp(sent_ssid->SSID, ssid, + sizeof(ssid) - 1)); + assert(!memcmp(sent_bssid, bssid, ETH_ALEN)); + free(version ? (void *)v1 : (void *)v0); + controls++; + } + } + } + printf("%u join scan controls passed\n", controls); + return 0; +} +""" + + +def function(source, name): + match = re.search(r"^static\s+void\s*\*?\s*" + name + r"\(", source, re.M) + if not match: + raise ValueError(name) + opening = source.index("{", match.end()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start() : end] + "\n" + + +def structure(source, name): + match = re.search(r"struct " + name + r"\s*\{", source) + if not match: + raise ValueError(name) + end = source.index("};", match.end()) + 2 + return source[match.start() : end] + "\n" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--ref", help="Git source revision to exercise") + parser.add_argument("--out", type=Path, help="retain executable and results") + parser.add_argument("--cc", default=os.environ.get("CC", "clang")) + args = parser.parse_args() + + def read(name): + path = DRIVER / name + if args.ref: + return subprocess.check_output( + ["git", "show", f"{args.ref}:{path}"], cwd=args.root, text=True + ) + return (args.root / path).read_text() + + source = read("join_param.c") + types = read("fwil_types.h") + constants = "\n".join(re.findall(r"^#define BRCMF_SCAN_JOIN_.*$", source, re.M)) + structures = "".join( + structure(types, name) + for name in [ + "brcmf_ssid_le", + "brcmf_join_scan_params_le", + "brcmf_assoc_params_le", + "brcmf_assoc_params_v1_le", + "brcmf_ext_join_params_le", + "brcmf_ext_join_params_v1_le", + ] + ) + bodies = "".join( + function(source, name) + for name in [ + "brcmf_joinscan_set_ssid", + "brcmf_joinscan_set_bssid", + "brcmf_joinscan_set_single_chanspec_from_channel", + "brcmf_joinscan_set_common_v0v1_params", + "brcmf_get_struct_for_connect_v0", + "brcmf_get_struct_for_connect_v1", + ] + ) + code = PREFIX + constants + "\n" + structures + bodies + MAIN + + def run(out): + out.mkdir(parents=True, exist_ok=True) + src, exe = out / "controls.c", out / "controls" + src.write_text(code) + subprocess.run( + [ + args.cc, + "-std=gnu11", + "-O1", + "-g", + "-Wall", + "-Wextra", + "-Werror", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=all", + str(src), + "-o", + str(exe), + ], + check=True, + ) + result = subprocess.run([str(exe)], capture_output=True, text=True, cwd=out) + receipt = { + "exit": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "limits": "Join constructors run with host stubs; no firmware or radio.", + } + (out / "results.json").write_text(json.dumps(receipt, indent=2) + "\n") + sys.stdout.write(result.stdout or result.stderr) + return result.returncode != 0 + + if args.out: + return run(args.out.resolve()) + with tempfile.TemporaryDirectory(prefix="brcmf-join-scan-") as out: + return run(Path(out)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index b878404b2d9fa..49e9b5540107c 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -88,6 +88,8 @@ def mutations(flag, names, base, expect=KILL): # The old revision reports a failed join against the last associated BSS. 'brcmf-join-bssid-tests.py': [(out(), PASS)] + mutations('--mutation', ('stale-target', 'zero-named', 'no-call', 'lost-bss-success'), out()), + 'brcmf-join-scan-tests.py': [(out(), PASS)] + + [(out() + ['--ref', '33bec4016c466985bfbb5ff4c2089fcab8242dcf'], KILL)], 'brcmf-pcie-dma-tests.py': [(out('--output'), PASS)] + mutations('--mutation', ('narrow-stream', 'ignore-aperture', 'revision-wide'), out('--output')), 'brcmf-roam-tests.py': [(out(), PASS)], From 6997a2c29d171c352412b3fa67eac0bcbbc45f2b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:16:15 +1100 Subject: [PATCH 265/392] mmc: enable UML I/O emulation for GL9755 sequencing tests MMC requires HAS_IOMEM. Enable UML I/O emulation in the test configuration so that the mock suite can be selected by the UML KUnit runner. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig b/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig index 642f88b5255ad..cd5367e2297e8 100644 --- a/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.kunitconfig @@ -1,3 +1,4 @@ CONFIG_KUNIT=y +CONFIG_UML_IOMEM_EMULATION=y CONFIG_MMC=y CONFIG_MMC_SDHCI_PCI_GLI_PWRSEQ_KUNIT_TEST=y From 5ad2c72abf9864c3b5b1c87266a16b7e4abd3517 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 08:46:47 -0500 Subject: [PATCH 266/392] cpufreq: apple: Add the T6050 thermal-owned profile The T6050 clusters' DVFS command register reads back the last request it accepted, not a measured state, so the profile verifies each transition against that readback, with a 2 ms timeout. On this chip the firmware owns the clusters' thermal limits; the profile refuses a cluster whose node doesn't say so (apple,firmware-managed-thermal), and the binding requires that property for apple,t6050-cluster-cpufreq. Tested on J714s (M5 Pro, bare metal under aura Stage 2): all three clusters bind (M 300-4380 MHz, P 300-4608 MHz, schedutil) with no DVFS error; idle clusters sit at 300 MHz and loaded ones at their top state. Clocks measured with a dependent add chain go from iBoot's 2.65/1.34/ 1.31 GHz to 4.29/4.38/4.60 GHz (with WFI idle); one-thread work is 1.6x faster on M and 3.4x on P, and idle system power drops from 20.1 to 7.2-7.9 W while idle still spins under idle=nop. Carried for the bare-metal kit by Erik Bourget (#944). Signed-off-by: Diljit Singh --- .../cpufreq/apple,cluster-cpufreq.yaml | 10 ++++++++ drivers/cpufreq/apple-soc-cpufreq.c | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 807eb00b967eb..b7ed80708377e 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -84,6 +84,16 @@ required: - reg - '#performance-domain-cells' +allOf: + - if: + properties: + compatible: + contains: + const: apple,t6050-cluster-cpufreq + then: + required: + - apple,firmware-managed-thermal + additionalProperties: false examples: diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 85e592faa0538..fb87ae6ae137a 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -206,6 +206,7 @@ struct apple_soc_cpufreq_info { /* Fail closed when the DT describes no fast die-temperature limit. */ bool needs_thermal_policy; bool needs_thermal_owner; + bool needs_firmware_thermal_control; u32 transition_timeout_us; /* Highest P-state allowed without a thermal policy; 0 if not capped. */ u32 max_unmanaged_pstate; @@ -340,6 +341,16 @@ static const struct apple_soc_cpufreq_info soc_t8132_info = { .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, }; +/* Command-state readback, not a measurement of the clock frequency. */ +static const struct apple_soc_cpufreq_info soc_t6050_info = { + .verify_transition = true, + .needs_firmware_thermal_control = true, + .transition_timeout_us = 2000, + .max_pstate = 31, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + static const struct apple_soc_cpufreq_info soc_default_info = { .has_ps2 = false, .max_pstate = 15, @@ -373,6 +384,10 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8140-cluster-cpufreq", .data = &soc_t8140_info, }, + { + .compatible = "apple,t6050-cluster-cpufreq", + .data = &soc_t6050_info, + }, { .compatible = "apple,t8142-cluster-cpufreq", .data = &soc_t8142_info, @@ -1355,6 +1370,14 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) goto out_free_priv; } + /* The boot producer must retain and qualify firmware thermal control. */ + if (info->needs_firmware_thermal_control && + !of_property_read_bool(cluster, "apple,firmware-managed-thermal")) { + dev_err(cpu_dev, "firmware thermal ownership is not declared\n"); + ret = -ENODEV; + goto out_iounmap; + } + ret = dev_pm_opp_of_cpumask_add_table(policy->cpus); if (ret < 0) { dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); From 227ed461886c243a633535468d3549e267440094 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:48:16 +1100 Subject: [PATCH 267/392] soc: apple: sep: Report endpoints, timing, identity and errors in diag macOS lists every SEP endpoint as its own service (sep-endpoint,cntl, sep-endpoint,sks, ...), but diag only had a count, and the names were only in two info lines per endpoint in the kernel log: 24 lines on every boot, gone again once the ring wrapped. Several states had no report at all: where the OS identity for the xART exchange came from, whether the hwrng survey registered, how long the bring-up took, and the xART requests the driver failed, which did not log. Add, keeping every existing attribute and value as it was: - endpoint_names: the advertised endpoints by their four-character names, in endpoint order; - attach_ms, ready_ms: milliseconds from probe to the attach decision and to the end of the boot-time bring-up; - os_identity: explicit, chosen or unavailable; - hwrng: pending, registered or failed; - xarm_failures: every xART request answered with the failure status, the guard's refusals included; the first is now logged; - unhandled_messages: messages on endpoints without a handler, logged only the first few times; - sleep, sleeps: held between the PM prepare and post events, and the number of completed sleep cycles. The per-message discovery trace moves to debug level, and one line replaces it at the end of the bring-up: ready in 4106 ms: 12 endpoints (cntl sbio scrd sse hdcp xars sks xarm hibe pnon skdl stac), key store open The attributes are only added, so the diag ABI stays at 2. Each read still only loads cached words and sends nothing to the SEP. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/sysfs-driver-apple-sep | 96 +++++++ drivers/soc/apple/diag.rs | 58 ++++ drivers/soc/apple/diag_shim.c | 248 ++++++++++++++++++ drivers/soc/apple/pm_shim.c | 2 + drivers/soc/apple/sep.rs | 62 ++++- drivers/soc/apple/shim.h | 17 ++ 6 files changed, 469 insertions(+), 14 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-apple-sep b/Documentation/ABI/testing/sysfs-driver-apple-sep index b48d26187f67d..2bddd85d523d4 100644 --- a/Documentation/ABI/testing/sysfs-driver-apple-sep +++ b/Documentation/ABI/testing/sysfs-driver-apple-sep @@ -359,3 +359,99 @@ Description: tools/aurora-sep/XART-SAFETY.md. During a boot the value can only move from "unverified" to "blocked", once the guard refuses a write. + +What: /sys/bus/platform/devices/.sep/diag/attach_ms +What: /sys/bus/platform/devices/.sep/diag/ready_ms +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) How long the bring-up took, as unsigned decimal + milliseconds counted from probe on the boot-time clock. + attach_ms is the time to the attach decision (see attach); + ready_ms is the time to the end of the boot-time bring-up, + when keystore is decided. Each reads "0" until its decision + is made. A large ready_ms with keystore "open" is an enclave + that was slow to advertise the key store; the driver waits + up to 90 s for it. + +What: /sys/bus/platform/devices/.sep/diag/endpoint_names +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) The endpoints the SEP has advertised, as the + four-character names their descriptors carry with trailing + spaces dropped, separated by single spaces, in endpoint + number order, for example "cntl sbio scrd sse hdcp xars sks + xarm hibe pnon skdl stac". A name that is not lower-case + letters and digits is shown as the endpoint number in two + hex digits. Empty until the first endpoint is advertised; + discovery goes on after the attach, so the list can still + grow, like endpoints. + +What: /sys/bus/platform/devices/.sep/diag/os_identity +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) Where the OS identity that the driver gives the + enclave's xART exchange came from. Decided at the attach. + + =========== ========================================= + unknown Not decided yet. + explicit The os_uuid_hi and os_uuid_lo parameters. + chosen /chosen/apfs-preboot-uuid, as the boot + loader passed it. + unavailable Neither; the enclave is told the identity + is unavailable and its xART records for + this OS cannot be found. + =========== ========================================= + +What: /sys/bus/platform/devices/.sep/diag/hwrng +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) The enclave's random number source. "pending" until + the survey after the bring-up decides, then "registered" + (it is an hwrng, named apple-sep) or "failed" (the survey + gave up; the kernel log says why). + +What: /sys/bus/platform/devices/.sep/diag/xarm_failures +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) How many of the enclave's xART requests the driver has + answered with the failure status since probe, as an unsigned + decimal: a store read or write that failed, a bad checksum, + an unknown request, a buffer that was not ready, and the + writes the xART guard refused (also counted in + xart_guard_refusals). The first is logged; this counts them + all. Anything above xart_guard_refusals is a failure that + was not the guard's choice. + +What: /sys/bus/platform/devices/.sep/diag/unhandled_messages +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) How many messages arrived on an endpoint the driver has + no handler for, as an unsigned decimal. The first few are + logged. Normally 0. + +What: /sys/bus/platform/devices/.sep/diag/sleep +What: /sys/bus/platform/devices/.sep/diag/sleeps +Date: November 2026 +KernelVersion: 7.3 +Contact: Ryan Murray +Description: + (RO) The driver's view of system sleep. sleep reads "held" + from the PM notifier's prepare event to its post event, + while Touch ID captures and entropy requests are held, and + "awake" otherwise. sleeps counts completed prepare/post + pairs since probe, aborted suspends included. The enclave's + power domains stay on across sleep, so every other attribute + keeps its value over a suspend; a reader checking that the + enclave still answers after a resume reads the hwrng. diff --git a/drivers/soc/apple/diag.rs b/drivers/soc/apple/diag.rs index 1b6fdd2288c52..c1eae7458e64d 100644 --- a/drivers/soc/apple/diag.rs +++ b/drivers/soc/apple/diag.rs @@ -43,6 +43,12 @@ extern "C" { fn sep_diag_set_xart_guard(level: c_uint, list_loaded: bool, matching: c_int); fn sep_diag_xart_guard_refused(matching: c_int); fn sep_diag_bringup_ended(); + fn sep_diag_set_endpoint(id: u8, name: u32); + fn sep_diag_set_os_identity(source: c_int); + fn sep_diag_set_hwrng(registered: bool); + fn sep_diag_xarm_failed() -> c_uint; + fn sep_diag_unhandled() -> c_uint; + fn sep_diag_log_summary(dev: *mut c_void); } /// The identity keybag, as `SEP_DIAG_KEYBAG_*` in `shim.h` spells it. @@ -53,6 +59,18 @@ pub(crate) enum Keybag { Failed = 3, } +/// Where the OS identity the driver answers xART's GET_OS_UUID with came from, +/// as `SEP_DIAG_OS_IDENTITY_*` in `shim.h` spells it. +#[derive(Clone, Copy)] +pub(crate) enum OsIdentity { + /// The `os_uuid_hi`/`os_uuid_lo` parameters. + Explicit = 1, + /// `/chosen/apfs-preboot-uuid`. + Chosen = 2, + /// Neither: GET_OS_UUID is answered "unavailable". + Unavailable = 3, +} + /// The enrolled fingerprints at attach, as `SEP_DIAG_RESTORE_*` in `shim.h` /// spells it. #[derive(Clone, Copy)] @@ -209,3 +227,43 @@ pub(crate) fn bringup_ended() { // SAFETY: no preconditions; updates two words. unsafe { sep_diag_bringup_ended() }; } + +/// An endpoint the SEP advertised, with the four-character name its +/// descriptor carries in the upper half of `msg0`. +pub(crate) fn endpoint(id: u8, name: u32) { + // SAFETY: no preconditions; stores one word and sets one bit. + unsafe { sep_diag_set_endpoint(id, name) }; +} + +pub(crate) fn os_identity(source: OsIdentity) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_os_identity(source as c_int) }; +} + +/// The hwrng survey registered the enclave's entropy source, or gave up. +pub(crate) fn hwrng(registered: bool) { + // SAFETY: no preconditions; stores one word. + unsafe { sep_diag_set_hwrng(registered) }; +} + +/// The driver answered an xART request with the failure status. Returns how +/// many it has answered so since probe, this one included. +pub(crate) fn xarm_failed() -> u32 { + // SAFETY: no preconditions; updates one word. + unsafe { sep_diag_xarm_failed() } +} + +/// A message arrived on an endpoint the driver has no handler for. Returns +/// how many have, this one included. +pub(crate) fn unhandled() -> u32 { + // SAFETY: no preconditions; updates one word. + unsafe { sep_diag_unhandled() } +} + +/// Logs the one-line summary of the boot-time bring-up. Call it after +/// [`bringup_ended`]. +pub(crate) fn log_summary(dev: &device::Device) { + // SAFETY: `dev` is the bound SEP device for the duration of the call; the + // shim only passes it to `dev_info()` and keeps no reference. + unsafe { sep_diag_log_summary(dev.as_raw().cast()) }; +} diff --git a/drivers/soc/apple/diag_shim.c b/drivers/soc/apple/diag_shim.c index 5c54d5d8410b3..fe405306d7969 100644 --- a/drivers/soc/apple/diag_shim.c +++ b/drivers/soc/apple/diag_shim.c @@ -15,7 +15,11 @@ * the state before the driver has decided anything. */ #include +#include +#include +#include #include +#include #include #include #include @@ -58,6 +62,9 @@ #define SEP_DIAG_TOUCHID_STARTED 3 #define SEP_DIAG_TOUCHID_FAILED 4 +/* The SEP's endpoint numbers are one byte. */ +#define SEP_DIAG_MAX_EP 256 + static struct { /* Set at probe, before the group is added. */ char profile[32]; @@ -78,6 +85,26 @@ static struct { atomic_t restore; atomic_t xart_refusals; atomic_t matching; + + /* Boot-time clock at probe, and at the attach and bring-up decisions. */ + u64 probe_ns; + atomic64_t attach_ns; + atomic64_t ready_ns; + + /* + * The four-character name each endpoint advertised, by endpoint number. + * A name is stored before its bit is set, so a reader that sees the bit + * sees the name. + */ + u32 ep_name[SEP_DIAG_MAX_EP]; + DECLARE_BITMAP(ep_seen, SEP_DIAG_MAX_EP); + + atomic_t os_identity; + atomic_t hwrng; + atomic_t xarm_failures; + atomic_t unhandled; + atomic_t sleeps; + atomic_t sleeping; } sep_diag; static unsigned int sep_diag_sensor(void) @@ -340,6 +367,149 @@ static ssize_t matching_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(matching); +/* Milliseconds from probe to a decision, or 0 while it is still pending. */ +static unsigned int sep_diag_ms_since_probe(atomic64_t *at) +{ + u64 t = atomic64_read(at); + + if (!t || t < sep_diag.probe_ns) + return 0; + return (unsigned int)min_t(u64, div_u64(t - sep_diag.probe_ns, + NSEC_PER_MSEC), UINT_MAX); +} + +static ssize_t attach_ms_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%u\n", sep_diag_ms_since_probe(&sep_diag.attach_ns)); +} +static DEVICE_ATTR_RO(attach_ms); + +static ssize_t ready_ms_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%u\n", sep_diag_ms_since_probe(&sep_diag.ready_ns)); +} +static DEVICE_ATTR_RO(ready_ms); + +/* + * An endpoint's name as the SEP spells it, trailing spaces dropped ("sks "). + * A name with a byte that is not a lower-case letter or digit is shown as its + * number, so the list stays one line of space-separated tokens. + */ +static int sep_diag_ep_name(unsigned int id, char out[5]) +{ + u32 name = READ_ONCE(sep_diag.ep_name[id]); + int len = 0, i; + + for (i = 0; i < 4; i++) { + char c = (char)(name >> (24 - 8 * i)); + + if (c == ' ' && len == i) + break; + if (!(islower(c) || isdigit(c)) || len != i) + return scnprintf(out, 5, "%02x", id); + out[len++] = c; + } + if (!len) + return scnprintf(out, 5, "%02x", id); + out[len] = '\0'; + return len; +} + +static ssize_t endpoint_names_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int id; + ssize_t n = 0; + + for_each_set_bit(id, sep_diag.ep_seen, SEP_DIAG_MAX_EP) { + char name[5]; + + /* Pairs with the barrier in sep_diag_set_endpoint(). */ + smp_rmb(); + sep_diag_ep_name(id, name); + n += sysfs_emit_at(buf, n, "%s%s", n ? " " : "", name); + } + n += sysfs_emit_at(buf, n, "\n"); + return n; +} +static DEVICE_ATTR_RO(endpoint_names); + +static ssize_t os_identity_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.os_identity)) { + case SEP_DIAG_OS_IDENTITY_EXPLICIT: + state = "explicit"; + break; + case SEP_DIAG_OS_IDENTITY_CHOSEN: + state = "chosen"; + break; + case SEP_DIAG_OS_IDENTITY_UNAVAILABLE: + state = "unavailable"; + break; + default: + state = "unknown"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(os_identity); + +static ssize_t hwrng_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *state; + + switch (atomic_read(&sep_diag.hwrng)) { + case SEP_DIAG_HWRNG_REGISTERED: + state = "registered"; + break; + case SEP_DIAG_HWRNG_FAILED: + state = "failed"; + break; + default: + state = "pending"; + break; + } + return sysfs_emit(buf, "%s\n", state); +} +static DEVICE_ATTR_RO(hwrng); + +static ssize_t xarm_failures_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", + (unsigned int)atomic_read(&sep_diag.xarm_failures)); +} +static DEVICE_ATTR_RO(xarm_failures); + +static ssize_t unhandled_messages_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", + (unsigned int)atomic_read(&sep_diag.unhandled)); +} +static DEVICE_ATTR_RO(unhandled_messages); + +static ssize_t sleeps_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%u\n", (unsigned int)atomic_read(&sep_diag.sleeps)); +} +static DEVICE_ATTR_RO(sleeps); + +static ssize_t sleep_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", + atomic_read(&sep_diag.sleeping) ? "held" : "awake"); +} +static DEVICE_ATTR_RO(sleep); + static struct attribute *sep_diag_attrs[] = { &dev_attr_abi.attr, &dev_attr_profile.attr, @@ -358,6 +528,15 @@ static struct attribute *sep_diag_attrs[] = { &dev_attr_xart_guard_list.attr, &dev_attr_xart_guard_refusals.attr, &dev_attr_matching.attr, + &dev_attr_attach_ms.attr, + &dev_attr_ready_ms.attr, + &dev_attr_endpoint_names.attr, + &dev_attr_os_identity.attr, + &dev_attr_hwrng.attr, + &dev_attr_xarm_failures.attr, + &dev_attr_unhandled_messages.attr, + &dev_attr_sleeps.attr, + &dev_attr_sleep.attr, NULL }; @@ -379,12 +558,14 @@ int sep_diag_register(struct device *dev, const char *profile, sep_diag.cold = cold; sep_diag.sepos13 = sepos13; sep_diag.xart = xart; + sep_diag.probe_ns = ktime_get_boottime_ns(); return devm_device_add_group(dev, &sep_diag_group); } void sep_diag_set_attach(bool attached) { + atomic64_cmpxchg(&sep_diag.attach_ns, 0, ktime_get_boottime_ns()); atomic_set(&sep_diag.attach, attached ? SEP_DIAG_ATTACH_ATTACHED : SEP_DIAG_ATTACH_FAILED); } @@ -394,6 +575,72 @@ void sep_diag_set_endpoints(unsigned int count) atomic_set(&sep_diag.endpoints, count); } +void sep_diag_set_endpoint(u8 id, u32 name) +{ + WRITE_ONCE(sep_diag.ep_name[id], name); + /* The name is visible before the bit; see endpoint_names_show(). */ + smp_wmb(); + set_bit(id, sep_diag.ep_seen); +} + +void sep_diag_set_os_identity(int source) +{ + atomic_set(&sep_diag.os_identity, source); +} + +void sep_diag_set_hwrng(bool registered) +{ + atomic_set(&sep_diag.hwrng, registered ? SEP_DIAG_HWRNG_REGISTERED : + SEP_DIAG_HWRNG_FAILED); +} + +unsigned int sep_diag_xarm_failed(void) +{ + return (unsigned int)atomic_inc_return(&sep_diag.xarm_failures); +} + +unsigned int sep_diag_unhandled(void) +{ + return (unsigned int)atomic_inc_return(&sep_diag.unhandled); +} + +/* From the PM notifier (pm_shim.c), which runs its calls one at a time. */ +void sep_diag_sleep(bool entering) +{ + if (entering) { + atomic_set(&sep_diag.sleeping, 1); + } else if (atomic_xchg(&sep_diag.sleeping, 0)) { + atomic_inc(&sep_diag.sleeps); + } +} + +/* + * One line for the kernel log when the boot-time bring-up has ended: what the + * per-endpoint discovery lines (now debug level) used to show, and the states + * a reader would otherwise collect from several attributes. + */ +void sep_diag_log_summary(struct device *dev) +{ + char names[192]; + unsigned int id; + size_t n = 0; + + names[0] = '\0'; + for_each_set_bit(id, sep_diag.ep_seen, SEP_DIAG_MAX_EP) { + char name[5]; + + smp_rmb(); + sep_diag_ep_name(id, name); + n += scnprintf(names + n, sizeof(names) - n, "%s%s", + n ? " " : "", name); + } + dev_info(dev, "ready in %u ms: %u endpoints (%s), key store %s\n", + sep_diag_ms_since_probe(&sep_diag.ready_ns), + (unsigned int)atomic_read(&sep_diag.endpoints), names, + atomic_read(&sep_diag.keystore) == SEP_DIAG_KEYSTORE_OPEN ? + "open" : "closed"); +} + void sep_diag_set_keystore_open(void) { atomic_set(&sep_diag.keystore, SEP_DIAG_KEYSTORE_OPEN); @@ -501,6 +748,7 @@ void sep_diag_xart_guard_refused(int matching) */ void sep_diag_bringup_ended(void) { + atomic64_cmpxchg(&sep_diag.ready_ns, 0, ktime_get_boottime_ns()); atomic_cmpxchg(&sep_diag.keystore, SEP_DIAG_KEYSTORE_UNKNOWN, SEP_DIAG_KEYSTORE_CLOSED); atomic_cmpxchg(&sep_diag.touchid, SEP_DIAG_TOUCHID_PENDING, diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 884ecbb354a06..76846ad5aaac3 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -28,12 +28,14 @@ static int sep_pm_notify(struct notifier_block *nb, unsigned long action, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: + sep_diag_sleep(true); event(true); return NOTIFY_OK; case PM_POST_SUSPEND: case PM_POST_HIBERNATION: case PM_POST_RESTORE: event(false); + sep_diag_sleep(false); return NOTIFY_OK; default: return NOTIFY_DONE; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 1f436a983271f..ff21f59e65b01 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -91,6 +91,9 @@ use kernel::{ }; const SETTLE_MS: time::Msecs = 200; +/// Messages on an endpoint without a handler are logged this many times per +/// boot; diag/unhandled_messages counts them all. +const UNHANDLED_LOG_LIMIT: u32 = 4; const FIRST_RESPONSE_MS: time::Msecs = 3000; const RNG_SURVEY_ATTEMPTS: u32 = 5; @@ -1546,11 +1549,14 @@ impl SepData { }; this.rng_survey_attempts.store(attempt, Relaxed); match next { - control::Survey::Registered => dev_info!( - this.dev, - "hwrng: registered after ready-phase survey (attempt {})\n", - attempt - ), + control::Survey::Registered => { + diag::hwrng(true); + dev_info!( + this.dev, + "hwrng: registered after ready-phase survey (attempt {})\n", + attempt + ) + } // Asleep, or holding back after a request that went unanswered: // this attempt did not ask the enclave anything. control::Survey::Held => Self::arm_settle(this), @@ -1558,12 +1564,15 @@ impl SepData { dev_warn!(this.dev, "hwrng: survey attempt {} failed ({:?}); retrying\n", attempt, e); Self::arm_settle(this); } - control::Survey::GiveUp(e) => dev_err!( - this.dev, - "hwrng: survey failed after {} attempts ({:?})\n", - attempt, - e - ), + control::Survey::GiveUp(e) => { + diag::hwrng(false); + dev_err!( + this.dev, + "hwrng: survey failed after {} attempts ({:?})\n", + attempt, + e + ) + } } } @@ -1677,14 +1686,17 @@ impl SepData { match os_identity::select(hi, lo, dt::preboot_uuid) { os_identity::Selection::Explicit(uuid) => { self.xarm.lock().os_uuid = Some(uuid); + diag::os_identity(diag::OsIdentity::Explicit); dev_info!(self.dev, "xART: using explicit OS identity\n"); } os_identity::Selection::Chosen(uuid) => { self.xarm.lock().os_uuid = Some(uuid); + diag::os_identity(diag::OsIdentity::Chosen); dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); } os_identity::Selection::Unavailable => { self.xarm.lock().os_uuid = None; + diag::os_identity(diag::OsIdentity::Unavailable); dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); } } @@ -1885,6 +1897,15 @@ impl SepData { } fn send_xarm_reply(&self, reply: &crate::xarm::XarmReply) { + // Each of these used to fail the enclave's request without a trace. + // Count them all and log the first, so a refusal is never silent. + if reply.status == xarm::STATUS_FAILED && diag::xarm_failed() == 1 { + dev_warn!( + self.dev, + "xART: answered a request (tag {:#04x}) with the failure status; diag/xarm_failures counts them\n", + reply.tag + ); + } let msg = crate::xarm::encode_xarm_reply(reply); let _ = self.send(msg); } @@ -2513,6 +2534,7 @@ impl SepData { this.run_bringup(); diag::bringup_ended(); + diag::log_summary(&this.dev); this.finish_bringup(); Self::arm_settle(this); } @@ -2585,7 +2607,13 @@ impl SepData { proto::EP_BOOT => self.on_boot(msg), - ep => dev_info!(self.dev, "rx: unhandled endpoint {:#04x} (msg0 {:#018x})\n", ep, msg.msg0), + ep => { + // Counted in diag/unhandled_messages; logged the first few + // times only, so a chatty endpoint cannot flood the log. + if diag::unhandled() <= UNHANDLED_LOG_LIMIT { + dev_info!(self.dev, "rx: unhandled endpoint {:#04x} (msg0 {:#018x})\n", ep, msg.msg0); + } + } } } @@ -2621,7 +2649,13 @@ impl SepData { proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { let _ = table.slot(f.param); diag::endpoints(table.eps.len()); - dev_info!( + if f.ty == proto::DISCOVER_TYPE_DESCRIPTOR { + diag::endpoint(f.param, (msg.msg0 >> 32) as u32); + } + // One summary line follows at the end of the bring-up, and + // diag/endpoint_names lists them; keep the per-message trace + // at debug level. + dev_dbg!( self.dev, "discover: endpoint {:#04x} (type {}, msg0 {:#018x}); {} known\n", f.param, @@ -2630,7 +2664,7 @@ impl SepData { table.eps.len() ); } - ty => dev_info!(self.dev, "discover: other type {} (msg0 {:#018x})\n", ty, msg.msg0), + ty => dev_dbg!(self.dev, "discover: other type {} (msg0 {:#018x})\n", ty, msg.msg0), } } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index ca0d3d8a797ee..d58ae573fbbad 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -175,6 +175,16 @@ int sep_sensor_irq_wait(unsigned int timeout_ms); #define SEP_DIAG_MATCHING_UNVERIFIED 2 #define SEP_DIAG_MATCHING_BLOCKED 3 +/* Mirrored by diag.rs (OsIdentity). */ +#define SEP_DIAG_OS_IDENTITY_UNKNOWN 0 +#define SEP_DIAG_OS_IDENTITY_EXPLICIT 1 +#define SEP_DIAG_OS_IDENTITY_CHOSEN 2 +#define SEP_DIAG_OS_IDENTITY_UNAVAILABLE 3 + +#define SEP_DIAG_HWRNG_PENDING 0 +#define SEP_DIAG_HWRNG_REGISTERED 1 +#define SEP_DIAG_HWRNG_FAILED 2 + struct device; int sep_diag_register(struct device *dev, const char *profile, @@ -194,6 +204,13 @@ void sep_diag_set_xart_guard(unsigned int level, bool list_loaded, int matching); void sep_diag_xart_guard_refused(int matching); void sep_diag_bringup_ended(void); +void sep_diag_set_endpoint(u8 id, u32 name); +void sep_diag_set_os_identity(int source); +void sep_diag_set_hwrng(bool registered); +unsigned int sep_diag_xarm_failed(void); +unsigned int sep_diag_unhandled(void); +void sep_diag_sleep(bool entering); +void sep_diag_log_summary(struct device *dev); /* -- bio_shim.c --------------------------------------------------------- */ From 227953e770b7ed807f7c82de46a95d2c1975b407 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:48:43 +1100 Subject: [PATCH 268/392] arm64: dts: apple: t604x-j614-j616: Describe the boot NOR flash supply The J614/J616 boot NOR has no vcc-supply, so spi-nor falls back to a dummy regulator and warns on every boot: spi-nor spi0.0: supply vcc not found, using dummy regulator The flash runs from a PMU rail that is on whenever the SoC is and that no OS switches; the ADT gives the flash no supply. Describe it as an always-on fixed regulator, as the T8140 J700 and the M1/M2 boards do. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi index 864e87400bb87..e3d0da5945528 100644 --- a/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi +++ b/arch/arm64/boot/dts/apple/t604x-j614-j616.dtsi @@ -175,6 +175,16 @@ #include "t6040-gpu-thermal.dtsi" / { + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; + /* * The speaker amps' shared shutdown line (SDZ, AP pin 54, low = * shutdown). One fixed regulator, the SDZ supply of all six, not @@ -306,6 +316,7 @@ spi-max-frequency = <10000000>; spi-rx-bus-width = <1>; spi-tx-bus-width = <1>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; From 21df09678d31909698ee69dc297156bbd1d56dca Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:57:03 +1100 Subject: [PATCH 269/392] pinctrl: apple: let GPIO interrupts be wakeup sources enable_irq_wake() on an Apple GPIO interrupt fails with -ENXIO: the irqchip has neither an irq_set_wake callback nor IRQCHIP_SKIP_SET_WAKE. A driver therefore cannot arm a GPIO-routed device, such as the SPI keyboard and trackpad on t8103/t600x laptops, as a wakeup source. The per-group interrupts are chained handlers on the AIC. suspend_device_irqs() leaves chained interrupts enabled, and the AIC already skips set_wake, so nothing needs programming. The pin interrupt only has to be marked as wakeup-armed so the PM core keeps it live in suspend-to-idle and treats it as a wakeup event. Set IRQCHIP_SKIP_SET_WAKE. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/pinctrl/pinctrl-apple-gpio.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index d3c482fa4234c..7390d28dfafab 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -376,7 +376,12 @@ static const struct irq_chip apple_gpio_irqchip = { .irq_mask = apple_gpio_irq_mask, .irq_unmask = apple_gpio_irq_unmask, .irq_set_type = apple_gpio_irq_set_type, - .flags = IRQCHIP_IMMUTABLE, + /* + * The group interrupts are chained handlers on the AIC, which + * the PM core never disables across system sleep, so a pin + * interrupt only has to be marked as a wakeup source here. + */ + .flags = IRQCHIP_IMMUTABLE | IRQCHIP_SKIP_SET_WAKE, GPIOCHIP_IRQ_RESOURCE_HELPERS, }; From e6cbd748e2d4021d8048f021136073fc98f4364e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:57:03 +1100 Subject: [PATCH 270/392] HID: spi-hid-apple: wake the system from the keyboard and trackpad The wakeup path was compiled out behind an IRQ_WAKE_SUPPORT define that nothing sets, so the controller was always powered off in suspend and the built-in keyboard and trackpad of the SPI HID laptops could not wake the system. Only the lid and the power button could. Make the device wakeup-capable, enabled by default like a USB keyboard. In suspend, a wakeup device keeps the controller powered and arms its interrupt, and resume only re-enables the interrupt because the controller kept its state. When wakeup is disabled through power/wakeup, or arming fails, the controller is powered off as before and rebooted on resume. A failed arm no longer leaves the controller powered with no wake armed. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/hid/spi-hid/spi-hid-apple-core.c | 59 ++++++++++++------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index f2a12d1fe56af..851d1c06e1206 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -130,9 +130,7 @@ struct spihid_apple { /* devm_kzalloc leaves this SPIHID_PWR_OFF. */ atomic_t power; -#ifdef IRQ_WAKE_SUPPORT bool irq_wake_enabled; -#endif }; /** @@ -1105,6 +1103,16 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops spihid_set_power(spihid, SPIHID_PWR_ON); + /* + * The built-in keyboard and trackpad wake the system, as they do + * under macOS. User space can turn this off through power/wakeup. + */ + if (ops->enable_irq_wake && ops->disable_irq_wake) { + err = devm_device_init_wakeup(dev); + if (err) + dev_warn(dev, "cannot enable wakeup: %d\n", err); + } + /* request device information */ dev_dbg(dev, "request device info"); spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); @@ -1191,9 +1199,6 @@ EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); static int spihid_apple_core_suspend(struct device *dev) { int ret; -#ifdef IRQ_WAKE_SUPPORT - int wake_status; -#endif struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); if (spihid->tp.hid) { @@ -1222,22 +1227,23 @@ static int spihid_apple_core_suspend(struct device *dev) /* Save some power */ spihid->ops->disable_irq(spihid->ops); -#ifdef IRQ_WAKE_SUPPORT + /* + * A wakeup device stays powered: the controller asserts its + * interrupt on a key press or a trackpad click, which wakes the + * system. Its state is kept, so resume does not reboot it. + */ + spihid->irq_wake_enabled = false; if (device_may_wakeup(dev)) { - wake_status = spihid->ops->enable_irq_wake(spihid->ops); - if (!wake_status) + ret = spihid->ops->enable_irq_wake(spihid->ops); + if (!ret) { spihid->irq_wake_enabled = true; - else - dev_warn(dev, "Failed to enable irq wake: %d\n", - wake_status); - } else { - spihid_set_power(spihid, SPIHID_PWR_OFF); - spihid->ops->power_off(spihid->ops); + return 0; + } + dev_warn(dev, "Failed to enable irq wake: %d\n", ret); } -#else + spihid_set_power(spihid, SPIHID_PWR_OFF); spihid->ops->power_off(spihid->ops); -#endif return 0; } @@ -1276,23 +1282,18 @@ static int spihid_apple_core_resume(struct device *dev) int ret_tp = 0, ret_kbd = 0; int err = 0; struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); -#ifdef IRQ_WAKE_SUPPORT - int wake_status; - - if (device_may_wakeup(dev) && spihid->irq_wake_enabled) { - wake_status = spihid->ops->disable_irq_wake(spihid->ops); - if (!wake_status) - spihid->irq_wake_enabled = false; - else - dev_warn(dev, "Failed to disable irq wake: %d\n", - wake_status); + + if (spihid->irq_wake_enabled) { + err = spihid->ops->disable_irq_wake(spihid->ops); + if (err) + dev_warn(dev, "Failed to disable irq wake: %d\n", err); + spihid->irq_wake_enabled = false; + /* Still powered and booted: just take interrupts again. */ spihid->ops->enable_irq(spihid->ops); + err = 0; } else { err = spihid_wake_controller(spihid); } -#else - err = spihid_wake_controller(spihid); -#endif if (err) dev_warn(dev, "SPI HID controller resume incomplete: %d\n", err); From 9bba309e315895577fcd837b8d25901168c40cec Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:57:57 +1100 Subject: [PATCH 271/392] soc: apple: sep: Log a lock-state change once The enclave sends a lock-state notification after each query, so the same state for the same user reached the log at info level ten to forty times a boot (38 lines on a J316s boot with two users). Keep the line for a change of state or user; a repeat of the last notification is logged at debug level. The state itself is recorded as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 43 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8d977de6a1cda..b2847c7f96d80 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -4359,25 +4359,36 @@ impl SepData { fn on_lock_state_notification(&self, word: u64) { let state = (word >> proto::MSG_TYPE_SHIFT) as u16; let user = (word >> proto::MSG_DATA_SHIFT) as u32; - self.bio_lock_state - .store(u64::from(state) | (u64::from(user) << 16), Relaxed); + let word = u64::from(state) | (u64::from(user) << 16); + let previous = self.bio_lock_state.xchg(word, Relaxed); self.bio_lock_state_seq.fetch_add(1, Relaxed); let lockout = if state & LOCK_STATE_NEEDS_PASSWORD != 0 { "set" } else { "clear" }; let n = lock_state_names(state); - dev_info!( - self.dev, - "sbio: lock state of user {} is {:#06x} (lockout {}:{}{}{}{}{}{}{})\n", - user, - state, - lockout, - n[0], - n[1], - n[2], - n[3], - n[4], - n[5], - n[6] - ); + // The enclave repeats the same state after each query; log a change + // once, and a repeat of the last notification only at debug level. + if previous != word { + dev_info!( + self.dev, + "sbio: lock state of user {} is {:#06x} (lockout {}:{}{}{}{}{}{}{})\n", + user, + state, + lockout, + n[0], + n[1], + n[2], + n[3], + n[4], + n[5], + n[6] + ); + } else { + dev_dbg!( + self.dev, + "sbio: lock state of user {} is still {:#06x}\n", + user, + state + ); + } self.sbio_wq.notify_all(); } From 2828b2f987f260b9067fca8988d5eaec7d273925 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:58:32 +1100 Subject: [PATCH 272/392] tools/aurora-sep: Add aurora-sep status, a self-check of the SEP state Users and the test suites had no single place to see whether the SEP came up as it should. The answer was spread over seventeen diag attributes, the device nodes, the hwrng and the kernel log, and only aurora-sep-ready looked at part of it, for Touch ID. aurora-sep status checks each state the driver reports and prints PASS, WARN, FAIL or INFO with the value and what it means: attach, the endpoints by name against the twelve every supported SEP advertises (the set macOS lists), bring-up time, key store, keybag, xART, OS identity and the MacBook Neo's os_uuid pin, hwrng, Touch ID, sensor, restore, matching, xART guard refusals, failed xART requests, unhandled messages, sleep, aurora-sep-ready.service, and the apple_sep error and warning lines of the boot. It exits 1 if a check failed and 2 without a SEP. --json and --summary are for test suites. It only reads, and never opens /dev/sep-bio, which would activate Touch ID. As root it reads 32 bytes from /dev/hwrng when the enclave is the current hwrng, which shows the enclave still answers, after a resume too. On kernels without the newer attributes it checks what they have. Installed as /usr/bin/aurora-sep by the Fedora and Arch packages. The tests run it against a synthetic sysfs, one broken state at a time. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 20 + tools/aurora-sep/aurora-sep | 363 ++++++++++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 5 +- .../packaging/fedora/aurora-sep.spec | 3 + .../tests/test_aurora_sep_status.py | 182 +++++++++ 5 files changed, 572 insertions(+), 1 deletion(-) create mode 100755 tools/aurora-sep/aurora-sep create mode 100644 tools/aurora-sep/tests/test_aurora_sep_status.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 15772498d4225..715a91171503d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -198,6 +198,26 @@ lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. +## Checking the SEP state + +`aurora-sep status` prints every state the driver reports, one line each +with PASS, WARN, FAIL or INFO: the attach, the twelve endpoints by name +(macOS lists the same twelve as `sep-endpoint,` services), the +bring-up time, the key store and identity keybag, xART, the OS identity +(and the os_uuid pin on the MacBook Neo), the hwrng, Touch ID, the sensor, +the fingerprint restore and matching, the xART guard, failed xART +requests, unhandled messages, sleep, `aurora-sep-ready.service` and the +apple_sep error and warning lines of this boot. It exits 1 if anything +FAILed. `--summary` prints only the last line, `--quiet` hides PASS and +INFO lines and `--json` prints the report as JSON. + +It only reads: it never opens `/dev/sep-bio`, since that activates Touch +ID. Run as root and with the enclave as the current hwrng, it reads 32 +bytes from `/dev/hwrng`, which shows that the enclave still answers, also +after a resume; `--no-live` skips that. The kernel log check needs root. +Older kernels without some of the diag attributes are checked on what +they have (the endpoint count instead of the names, for example). + ## Packages `packaging/` builds `aurora-sep`, with the loader, its service and the fprintd diff --git a/tools/aurora-sep/aurora-sep b/tools/aurora-sep/aurora-sep new file mode 100755 index 0000000000000..00e3adc4979de --- /dev/null +++ b/tools/aurora-sep/aurora-sep @@ -0,0 +1,363 @@ +#!/usr/bin/python3 -I +# SPDX-License-Identifier: GPL-2.0-only +"""aurora-sep: report the Secure Enclave (SEP) state on an Aurora kernel. + + aurora-sep status [--json] [--summary] [--no-live] [--quiet] + +`status` checks every state the apple_sep driver reports and prints one +line per check: PASS, WARN, FAIL or INFO, the value, and what it means. +It reads the driver's diag attributes, the device nodes, the hwrng and +the kernel log. It never opens /dev/sep-bio (that activates Touch ID) +and never writes anything. With root and the enclave as the current +hwrng it reads 32 random bytes from /dev/hwrng, which proves the enclave +still answers (after a resume, too); --no-live skips that. + +Exit status: 0 when nothing FAILed, 1 when something did, 2 when this is +not a Mac with a SEP driver. + +Environment, for tests: AURORA_SEP_SYSFS (default /sys/bus/platform/devices), +AURORA_SEP_ROOT (prefix for /proc, /dev and /sys/class paths, default ""), +AURORA_SEP_NO_SYSTEM=1 (skip systemctl and the kernel log). +""" + +import json +import os +import subprocess +import sys + +SYSFS = os.environ.get("AURORA_SEP_SYSFS", "/sys/bus/platform/devices") +ROOT = os.environ.get("AURORA_SEP_ROOT", "") +# Tests set this to keep systemctl and the journal of the machine out of it. +NO_SYSTEM = os.environ.get("AURORA_SEP_NO_SYSTEM") == "1" + +# The endpoints every supported SEP advertises. macOS lists the same twelve +# (ioreg: sep-endpoint, under AppleSEPManager) on M1 Pro, M2 and later. +EXPECTED_ENDPOINTS = ["cntl", "hdcp", "hibe", "pnon", "sbio", "scrd", + "skdl", "sks", "sse", "stac", "xarm", "xars"] +# The ones the driver serves itself; the rest have no Linux client yet. +SERVED = {"cntl": "control, hwrng", "xarm": "xART store", "sks": "key store", + "sbio": "Touch ID", "scrd": "credentials"} + +# A bring-up slower than this is reported (the driver waits up to 90 s). +READY_WARN_MS = 15000 + +PASS, WARN, FAIL, INFO = "PASS", "WARN", "FAIL", "INFO" + + +def path(p): + return ROOT + p + + +def read(p): + try: + with open(p, encoding="ascii", errors="replace") as f: + return f.read().strip() + except OSError: + return None + + +class Report: + def __init__(self): + self.rows = [] + + def add(self, verdict, name, value, why=""): + self.rows.append({"check": name, "verdict": verdict, + "value": "" if value is None else str(value), + "detail": why}) + + def count(self, verdict): + return sum(1 for r in self.rows if r["verdict"] == verdict) + + +def find_sep(): + try: + names = sorted(os.listdir(SYSFS)) + except OSError: + return None + for n in names: + if n.endswith(".sep"): + return os.path.join(SYSFS, n) + return None + + +def compatible(): + raw = None + try: + with open(path("/proc/device-tree/compatible"), "rb") as f: + raw = f.read() + except OSError: + return [] + return [c.decode("ascii", "replace") for c in raw.split(b"\0") if c] + + +def kernel_log_levels(): + """(err, warning) apple_sep line counts, or None.""" + res = [] + for prio in ("err", "warning"): + try: + out = subprocess.run(["journalctl", "-k", "-b", "-p", prio, "-o", + "cat", "--no-pager", "-q"], + capture_output=True, text=True, timeout=20, + check=False) + except (OSError, subprocess.TimeoutExpired): + return None + if out.returncode != 0: + return None + res.append([l for l in out.stdout.splitlines() + if "apple_sep" in l or "sep sensor" in l]) + return res + + +def status(live=True): + r = Report() + dev = find_sep() + if dev is None: + r.add(FAIL, "sep-device", "absent", + "no .sep platform device: not an Apple silicon Mac, or the DT has no SEP node") + return r, 2 + drv = os.path.basename(os.path.realpath(os.path.join(dev, "driver"))) \ + if os.path.exists(os.path.join(dev, "driver")) else None + if drv != "apple_sep": + r.add(FAIL, "driver", drv or "unbound", + "apple_sep is not bound to the SEP (module not loaded, or probe failed)") + return r, 1 + r.add(PASS, "driver", "apple_sep bound to " + os.path.basename(dev)) + + diag = os.path.join(dev, "diag") + if not os.path.isdir(diag): + r.add(FAIL, "diag", "absent", "this kernel has no diag attributes; too old to check") + return r, 1 + + def d(name): + return read(os.path.join(diag, name)) + + r.add(INFO, "profile", d("profile"), "platform profile the driver chose") + r.add(INFO, "bootstrap", d("boot"), "cold: the driver boots the SEP; warm: iBoot left it running") + r.add(INFO, "protocol", d("protocol"), "key-store protocol") + + attach = d("attach") + if attach == "attached": + r.add(PASS, "attach", attach, "the SEP advertised its endpoints") + elif attach == "pending": + r.add(FAIL, "attach", attach, "the SEP has not answered the driver") + else: + r.add(FAIL, "attach", attach, "the SEP advertised no endpoint") + + names = d("endpoint_names") + count = d("endpoints") + if names is not None: + have = names.split() + missing = [e for e in EXPECTED_ENDPOINTS if e not in have] + extra = [e for e in have if e not in EXPECTED_ENDPOINTS] + if missing: + r.add(FAIL, "endpoints", "%d: %s" % (len(have), names), + "missing " + " ".join(missing)) + else: + r.add(PASS, "endpoints", "%d: %s" % (len(have), names), + "all %d expected endpoints, as macOS lists them" % len(EXPECTED_ENDPOINTS) + + ("; also " + " ".join(extra) if extra else "")) + unserved = [e for e in have if e not in SERVED] + r.add(INFO, "endpoint-clients", + ", ".join("%s=%s" % (e, SERVED[e]) for e in have if e in SERVED), + "no Linux client yet: " + " ".join(unserved) if unserved else "") + elif count is not None: + n = int(count) if count.isdigit() else 0 + r.add(PASS if n >= len(EXPECTED_ENDPOINTS) else FAIL, "endpoints", n, + "expected %d (this kernel does not list names)" % len(EXPECTED_ENDPOINTS)) + + for name, why in (("attach_ms", "probe to attach"), + ("ready_ms", "probe to the end of the bring-up")): + v = d(name) + if v is None: + continue + ms = int(v) if v.isdigit() else 0 + if ms == 0: + r.add(WARN, name, v, why + ": not decided yet") + elif name == "ready_ms" and ms > READY_WARN_MS: + r.add(WARN, name, v, why + ": slow (over %d ms)" % READY_WARN_MS) + else: + r.add(INFO, name, v, why) + + ks = d("keystore") + r.add(PASS if ks == "open" else FAIL, "keystore", ks, + {"open": "key store open", "closed": "the bring-up ended without the key store", + "unknown": "bring-up still running"}.get(ks, "")) + + kb = d("keybag") + xart = d("xart") + if kb == "present": + r.add(PASS, "keybag", kb, "identity keybag present") + elif kb == "missing": + r.add(WARN, "keybag", kb, "no identity keybag: Touch ID is not provisioned on this install") + else: + r.add(FAIL, "keybag", kb, "identity keybag failed or undecided") + + if xart == "enabled": + r.add(PASS, "xart", xart, "the enclave's state writes reach the disk") + else: + r.add(WARN, "xart", xart, "xART writes are off (xart_writes=0): enrol and verify are refused") + + osid = d("os_identity") + if osid is not None: + if osid in ("explicit", "chosen"): + r.add(PASS, "os-identity", osid, "the xART exchange has an OS identity") + else: + r.add(FAIL, "os-identity", osid, "no OS identity: the enclave cannot find this OS's xART records") + comp = compatible() + cmdline = read(path("/proc/cmdline")) or "" + if "apple,j700" in comp: + pinned = "apple_sep.os_uuid_hi=" in cmdline and "apple_sep.os_uuid_lo=" in cmdline + r.add(PASS if pinned else FAIL, "os-uuid-pin", "present" if pinned else "missing", + "the MacBook Neo needs the os_uuid pin on every boot entry") + for p in ("xarm_trace", "sensor_trace"): + if "apple_sep.%s=1" % p in cmdline: + r.add(WARN, "cmdline-" + p, "on", "debug tracing is on in the boot entry (log noise)") + + hw = d("hwrng") + cur = read(path("/sys/class/misc/hw_random/rng_current")) + avail = read(path("/sys/class/misc/hw_random/rng_available")) or "" + if hw is not None: + r.add(PASS if hw == "registered" else (WARN if hw == "pending" else FAIL), + "hwrng", hw, "the enclave's random source") + elif "apple-sep" in avail.split(): + r.add(PASS, "hwrng", "registered", "apple-sep is an available hwrng") + else: + r.add(FAIL, "hwrng", "absent", "apple-sep is not an hwrng") + if live: + if cur == "apple-sep" and os.access(path("/dev/hwrng"), os.R_OK): + try: + with open(path("/dev/hwrng"), "rb", buffering=0) as f: + got = f.read(32) + ok = len(got) == 32 and any(got) + r.add(PASS if ok else FAIL, "live", "%d bytes" % len(got), + "the enclave answered an entropy request now" if ok + else "the enclave did not answer an entropy request") + except OSError as e: + r.add(FAIL, "live", "error", "reading /dev/hwrng: %s" % e.strerror) + elif cur == "apple-sep": + r.add(INFO, "live", "skipped", "needs root to read /dev/hwrng") + else: + r.add(INFO, "live", "skipped", "the current hwrng is %s, not apple-sep" % (cur or "none")) + + for node, why in (("/dev/sep-bio", "Touch ID"), ("/dev/sep-key", "enclave keys")): + if os.path.exists(path(node)): + r.add(PASS, "node" + node[4:], "present", why) + else: + r.add(WARN if ks != "open" else FAIL, "node" + node[4:], "absent", why) + + tid = d("touchid") + sensor = d("sensor") + sattach = d("sensor_attach") + restore = d("restore") + match = d("matching") + if tid == "ready": + r.add(PASS, "touchid", tid, "Touch ID activated and the sensor online") + elif tid == "unknown": + r.add(WARN, "touchid", tid, "not activated yet (aurora-sep-ready.service opens /dev/sep-bio at boot)") + elif tid == "failed" and kb == "missing": + r.add(WARN, "touchid", tid, "not provisioned (no identity keybag)") + else: + r.add(FAIL, "touchid", tid, "Touch ID cannot work this boot") + r.add(PASS if sensor == "online" else (WARN if sensor in ("bound", "unbound") and tid != "ready" else FAIL), + "sensor", sensor, "fingerprint sensor on SPI") + if sattach is not None: + r.add(PASS if sattach == "attached" else (WARN if sattach in ("unknown", "retrying") else FAIL), + "sensor-attach", sattach, "") + if restore is not None: + r.add(PASS if restore in ("restored", "empty") else (WARN if restore == "unknown" else FAIL), + "restore", restore, {"restored": "enrolled fingerprints restored", + "empty": "no fingerprint enrolled", + "unproven": "fingerprints listed but not proven: matching refused"}.get(restore, "")) + if match is not None: + r.add({"usable": PASS, "unverified": PASS, "blocked": FAIL}.get(match, WARN), + "matching", match, + {"usable": "a match can save its state", + "unverified": "the xART guard runs and has refused nothing", + "blocked": "a match cannot save its state; see XART-SAFETY.md"}.get(match, "")) + + refusals = d("xart_guard_refusals") + failures = d("xarm_failures") + ref = int(refusals) if refusals and refusals.isdigit() else 0 + if refusals is not None: + r.add(PASS if ref == 0 else WARN, "xart-guard-refusals", refusals, + "xART guard level %s" % d("xart_guard")) + if failures is not None: + fail = int(failures) if failures.isdigit() else 0 + r.add(PASS if fail <= ref else FAIL, "xarm-failures", failures, + "xART requests the driver failed (guard refusals included)") + unh = d("unhandled_messages") + if unh is not None: + r.add(PASS if unh == "0" else WARN, "unhandled-messages", unh, + "messages on endpoints without a handler") + + sl = d("sleep") + if sl is not None: + r.add(PASS if sl == "awake" else WARN, "sleep", sl, "sleep cycles since probe: %s" % d("sleeps")) + + if NO_SYSTEM: + return r, (1 if r.count(FAIL) else 0) + + unit = None + try: + out = subprocess.run(["systemctl", "show", "-p", "Result", "-p", "ActiveState", + "-p", "LoadState", "aurora-sep-ready.service"], + capture_output=True, text=True, timeout=10, check=False) + props = dict(l.split("=", 1) for l in out.stdout.splitlines() if "=" in l) + if props.get("LoadState") == "loaded": + unit = props.get("Result"), props.get("ActiveState") + except (OSError, subprocess.TimeoutExpired): + pass + if unit and unit[1]: + r.add(PASS if unit[0] == "success" else FAIL, "aurora-sep-ready", + "%s/%s" % (unit[1], unit[0]), "boot-time Touch ID activation") + + levels = kernel_log_levels() + if levels is None: + r.add(INFO, "kernel-log", "unreadable", "run as root to check the kernel log") + else: + errs, warns = levels + r.add(PASS if not errs else FAIL, "log-errors", len(errs), + errs[0][:160] if errs else "no apple_sep error lines this boot") + r.add(PASS if not warns else WARN, "log-warnings", len(warns), + warns[0][:160] if warns else "no apple_sep warning lines this boot") + + return r, (1 if r.count(FAIL) else 0) + + +def main(argv): + args = argv[1:] + if not args or args[0] in ("-h", "--help", "help"): + print(__doc__.strip()) + return 0 if args else 2 + if args[0] != "status": + print("aurora-sep: unknown command %r (try: aurora-sep status)" % args[0], file=sys.stderr) + return 2 + opts = set(args[1:]) + unknown = opts - {"--json", "--summary", "--no-live", "--quiet"} + if unknown: + print("aurora-sep: unknown option %s" % " ".join(sorted(unknown)), file=sys.stderr) + return 2 + rep, rc = status(live="--no-live" not in opts) + totals = {v: rep.count(v) for v in (PASS, WARN, FAIL, INFO)} + if "--json" in opts: + json.dump({"result": "FAIL" if rep.count(FAIL) else "PASS", + "totals": totals, "checks": rep.rows}, sys.stdout, indent=1) + print() + return rc + if "--summary" not in opts: + for row in rep.rows: + if "--quiet" in opts and row["verdict"] in (PASS, INFO): + continue + line = "%-4s %-20s %s" % (row["verdict"], row["check"], row["value"]) + if row["detail"]: + line += " (" + row["detail"] + ")" + print(line) + bad = [row["check"] for row in rep.rows if row["verdict"] == FAIL] + print("aurora-sep: %s pass %d warn %d fail %d%s" % ( + "FAIL" if bad else "PASS", totals[PASS], totals[WARN], totals[FAIL], + (" [" + " ".join(bad) + "]") if bad else "")) + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 013c7fe5df861..379484f162cbb 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -34,7 +34,8 @@ source=(load-driver README.md XART-SAFETY.md OS-IDENTITY.md - sep-bio-identities) + sep-bio-identities + aurora-sep) sha256sums=('SKIP' 'SKIP' 'SKIP' @@ -49,6 +50,7 @@ sha256sums=('SKIP' 'SKIP' 'SKIP' 'SKIP' + 'SKIP' 'SKIP') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -83,5 +85,6 @@ package() { install -Dm644 logind-aurora-sep.conf \ "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" install -Dm755 sep-bio-identities "$pkgdir/usr/bin/sep-bio-identities" + install -Dm755 aurora-sep "$pkgdir/usr/bin/aurora-sep" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md OS-IDENTITY.md } diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 359ec582e1877..15a5b2a1bf2e0 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -30,6 +30,7 @@ Source11: aurora_sep.te Source12: aurora_sep.fc Source13: OS-IDENTITY.md Source16: sep-bio-identities +Source17: aurora-sep BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -99,6 +100,7 @@ sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ install -Dpm 0644 %{SOURCE8} \ %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf install -Dpm 0755 %{SOURCE16} %{buildroot}%{_bindir}/sep-bio-identities +install -Dpm 0755 %{SOURCE17} %{buildroot}%{_bindir}/aurora-sep # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -140,6 +142,7 @@ fi %{_libexecdir}/aurora-sep-ready %{_libexecdir}/aurora-sep-greeter-fingerprint %{_bindir}/sep-bio-identities +%{_bindir}/aurora-sep %{_sbindir}/aurora-sep-policy-change %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service diff --git a/tools/aurora-sep/tests/test_aurora_sep_status.py b/tools/aurora-sep/tests/test_aurora_sep_status.py new file mode 100644 index 0000000000000..d99ffdd7d783e --- /dev/null +++ b/tools/aurora-sep/tests/test_aurora_sep_status.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""aurora-sep status: one verdict per SEP state, from a synthetic sysfs. + +The healthy values are the ones a fleet44 boot reports on the M1 Pro, M2, +J616s and MacBook Neo; each test then breaks one state and checks that the +tool names it and exits 1. +""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "aurora-sep" + +HEALTHY = { + "abi": "2", "profile": "T8112/J415", "boot": "warm", "protocol": "variant5", + "xart": "enabled", "attach": "attached", "endpoints": "12", + "endpoint_names": "cntl sbio scrd sse hdcp xars sks xarm hibe pnon skdl stac", + "attach_ms": "845", "ready_ms": "4444", "keystore": "open", + "keybag": "present", "sensor": "online", "touchid": "ready", + "sensor_attach": "attached", "restore": "restored", "xart_guard": "1", + "xart_guard_list": "loaded", "xart_guard_refusals": "0", + "matching": "unverified", "os_identity": "explicit", "hwrng": "registered", + "xarm_failures": "0", "unhandled_messages": "0", "sleep": "awake", + "sleeps": "3", +} + + +class StatusTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.devices = self.root / "devices" + self.sep = self.devices / "25e400000.sep" + self.diag = self.sep / "diag" + self.diag.mkdir(parents=True) + drv = self.root / "drivers" / "apple_sep" + drv.mkdir(parents=True) + (self.sep / "driver").symlink_to(drv) + for name, value in HEALTHY.items(): + self.set(name, value) + (self.root / "proc" / "device-tree").mkdir(parents=True) + self.compatible(b"apple,j413\0apple,t8112\0apple,arm-platform\0") + (self.root / "proc" / "cmdline").write_text("root=UUID=x ro quiet\n") + rng = self.root / "sys" / "class" / "misc" / "hw_random" + rng.mkdir(parents=True) + (rng / "rng_available").write_text("apple-sep\n") + (rng / "rng_current").write_text("apple-sep\n") + (self.root / "dev").mkdir() + for node in ("sep-bio", "sep-key"): + (self.root / "dev" / node).write_text("") + (self.root / "dev" / "hwrng").write_bytes(bytes(range(1, 33))) + + def tearDown(self): + self.tmp.cleanup() + + def set(self, name, value): + path = self.diag / name + if value is None: + path.unlink(missing_ok=True) + else: + path.write_text(value + "\n") + + def compatible(self, raw): + (self.root / "proc" / "device-tree" / "compatible").write_bytes(raw) + + def run_status(self, *args): + env = dict(os.environ, AURORA_SEP_SYSFS=str(self.devices), + AURORA_SEP_ROOT=str(self.root), AURORA_SEP_NO_SYSTEM="1") + out = subprocess.run([sys.executable, "-I", str(SCRIPT), "status", "--json", *args], + capture_output=True, text=True, env=env, timeout=30) + report = json.loads(out.stdout) + verdicts = {row["check"]: row["verdict"] for row in report["checks"]} + return out.returncode, report, verdicts + + def test_healthy_boot_passes(self): + rc, report, v = self.run_status() + self.assertEqual(rc, 0, report) + self.assertEqual(report["result"], "PASS") + self.assertEqual(report["totals"]["FAIL"], 0) + self.assertEqual(v["endpoints"], "PASS") + self.assertEqual(v["live"], "PASS") + + def test_a_missing_endpoint_fails(self): + self.set("endpoint_names", "cntl sbio scrd sse hdcp xars xarm hibe pnon skdl stac") + rc, report, v = self.run_status() + self.assertEqual(rc, 1) + self.assertEqual(v["endpoints"], "FAIL") + row = [r for r in report["checks"] if r["check"] == "endpoints"][0] + self.assertIn("missing sks", row["detail"]) + + def test_an_old_kernel_falls_back_to_the_count(self): + for name in ("endpoint_names", "attach_ms", "ready_ms", "os_identity", + "hwrng", "xarm_failures", "unhandled_messages", "sleep", "sleeps"): + self.set(name, None) + rc, _, v = self.run_status() + self.assertEqual(rc, 0) + self.assertEqual(v["endpoints"], "PASS") + self.assertEqual(v["hwrng"], "PASS") + self.set("endpoints", "7") + rc, _, v = self.run_status() + self.assertEqual((rc, v["endpoints"]), (1, "FAIL")) + + def test_a_closed_key_store_fails(self): + self.set("keystore", "closed") + rc, _, v = self.run_status() + self.assertEqual((rc, v["keystore"]), (1, "FAIL")) + + def test_a_driver_failure_beyond_the_guard_fails(self): + self.set("xart_guard_refusals", "2") + self.set("xarm_failures", "2") + rc, _, v = self.run_status() + self.assertEqual(v["xarm-failures"], "PASS") + self.assertEqual(v["xart-guard-refusals"], "WARN") + self.set("xarm_failures", "3") + rc, _, v = self.run_status() + self.assertEqual((rc, v["xarm-failures"]), (1, "FAIL")) + + def test_no_os_identity_fails(self): + self.set("os_identity", "unavailable") + rc, _, v = self.run_status() + self.assertEqual((rc, v["os-identity"]), (1, "FAIL")) + + def test_the_neo_needs_its_pin(self): + self.compatible(b"apple,j700\0apple,t8140\0apple,arm-platform\0") + rc, _, v = self.run_status() + self.assertEqual((rc, v["os-uuid-pin"]), (1, "FAIL")) + (self.root / "proc" / "cmdline").write_text( + "ro apple_sep.os_uuid_hi=1 apple_sep.os_uuid_lo=2\n") + rc, _, v = self.run_status() + self.assertEqual((rc, v["os-uuid-pin"]), (0, "PASS")) + + def test_a_silent_enclave_fails_the_live_check(self): + (self.root / "dev" / "hwrng").write_bytes(b"") + rc, _, v = self.run_status() + self.assertEqual((rc, v["live"]), (1, "FAIL")) + rc, _, v = self.run_status("--no-live") + self.assertEqual(rc, 0) + self.assertNotIn("live", v) + + def test_unprovisioned_touch_id_warns(self): + self.set("keybag", "missing") + self.set("touchid", "failed") + self.set("sensor", "unbound") + self.set("sensor_attach", "unknown") + self.set("restore", "unknown") + rc, report, v = self.run_status() + self.assertEqual(rc, 0, report) + self.assertEqual((v["keybag"], v["touchid"]), ("WARN", "WARN")) + + def test_blocked_matching_fails(self): + self.set("matching", "blocked") + rc, _, v = self.run_status() + self.assertEqual((rc, v["matching"]), (1, "FAIL")) + + def test_no_sep_device_exits_2(self): + os.unlink(self.sep / "driver") + for f in self.diag.iterdir(): + f.unlink() + self.diag.rmdir() + self.sep.rmdir() + rc, report, _ = self.run_status() + self.assertEqual(rc, 2) + self.assertEqual(report["result"], "FAIL") + + def test_summary_is_one_line(self): + env = dict(os.environ, AURORA_SEP_SYSFS=str(self.devices), + AURORA_SEP_ROOT=str(self.root), AURORA_SEP_NO_SYSTEM="1") + out = subprocess.run([sys.executable, "-I", str(SCRIPT), "status", "--summary"], + capture_output=True, text=True, env=env, timeout=30) + self.assertEqual(out.returncode, 0) + self.assertEqual(len(out.stdout.splitlines()), 1) + self.assertTrue(out.stdout.startswith("aurora-sep: PASS")) + + +if __name__ == "__main__": + unittest.main() From 728bb25e573c82cfd0f9edd123b9150f5dec4b82 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:59:14 +1100 Subject: [PATCH 273/392] soc: apple: sep: log a capture without a finger at info level A Touch ID capture that ends without a frame means nobody touched the sensor before the capture window closed. Verify answers with a retry and PLACE guidance and enrol counts it as no finger; neither is a driver or hardware problem. Every PAM prompt that times out or is answered with the password logged a kernel warning, so the warning count of a normal session grew with each unused prompt. Log both at info level. Sensor faults during a capture stay errors. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/soc/apple/sbio.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8d977de6a1cda..2da07846c2cd3 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2913,7 +2913,7 @@ impl SepData { let advertised = match self.await_capture() { CaptureWait::Ready(n) => n, CaptureWait::Timeout => { - dev_warn!(self.dev, "verify: the capture ended without a frame\n"); + dev_info!(self.dev, "verify: the capture ended without a frame\n"); // No finger came: nothing was compared, so ask again rather // than report an error. PLACE names the cause. self.retry_cause(bio::Guidance::Place); @@ -4024,7 +4024,7 @@ impl SepData { let available = match self.await_capture() { CaptureWait::Ready(n) => n, CaptureWait::Timeout => { - dev_warn!(self.dev, "enrol: capture {} ended without a frame\n", counter); + dev_info!(self.dev, "enrol: capture {} ended without a frame\n", counter); return ImageOutcome::NoFinger; } CaptureWait::Fault(state) => { From 7be62fbfde6a8a280b52b975126cb08f33498e59 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:05:29 +1100 Subject: [PATCH 274/392] dt-bindings: hwmon: apple,smc-hwmon: no-data-below takes the standard unit type A -millicelsius property already has its type from the dtschema unit suffixes (an int32 array). Describe it as a single value instead of redefining the type, which made every user fail validation. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index e98bdcc7b7897..a4e4dfd3c0886 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -34,7 +34,7 @@ $defs: description: Human-readable name for the sensor apple,no-data-below-millicelsius: - $ref: /schemas/types.yaml#/definitions/int32 + maxItems: 1 description: The SMC publishes a placeholder below this temperature, in millidegrees Celsius, while the sensor is powered down (the GPU From 33a82d7d9c6d8168b8c215000bdb5173f0c2ffd8 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:53:09 +1100 Subject: [PATCH 275/392] arm64: dts: apple: t600x-j314-j316: add the GPU temperature sensors The M1 Pro and M1 Max MacBook Pros exposed no GPU temperature. Their SMC has four GPU keys: Tg04 and Tg0C, and Tg05 and Tg0D, which read Tg04 and Tg0C plus a fixed 9.2 C. Measured on a J316s (M1 Pro): 33.9-44.3 C during a glmark2 run. Like the M4 Pro, the SMC publishes them only while the GPU is powered. With the GPU off at idle, Tg04/Tg0C read 0 C and Tg05/Tg0D 9.2 C. Mark those placeholders as no data: below 10 C on Tg04/Tg0C, and below 15 C on the offset keys, whose real readings sit 9.2 C above the base keys. They are read-only hwmon channels with no thermal zone or trip. Keys a machine lacks are skipped. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 9dbbcd52d4ec4..9a5f76f4fb069 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -997,6 +997,37 @@ #include "spi1-nvram.dtsi" #include "hwmon-laptop.dtsi" #include "hwmon-fan-dual.dtsi" + +/* + * GPU temperatures. The SMC publishes them only while the GPU is powered: + * with the GPU off, Tg04/Tg0C read 0 C and Tg05/Tg0D read 9.2 C (they + * track Tg04/Tg0C plus a fixed 9.2 C). Those placeholders are no data. + */ +&smc_hwmon { + temperature-Tg04 { + apple,key-id = "Tg04"; + label = "GPU Tg04"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg05 { + apple,key-id = "Tg05"; + label = "GPU Tg05"; + apple,no-data-below-millicelsius = <15000>; + }; + + temperature-Tg0C { + apple,key-id = "Tg0C"; + label = "GPU Tg0C"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0D { + apple,key-id = "Tg0D"; + label = "GPU Tg0D"; + apple,no-data-below-millicelsius = <15000>; + }; +}; #include "isp-imx558.dtsi" &isp { From 3e7a79c567fc388652c7dc8b9cc786f2652cf997 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:53:10 +1100 Subject: [PATCH 276/392] arm64: dts: apple: t8140-j700: add the GPU temperature sensors The J700 exposed no GPU temperature. Its SMC has eight GPU keys (Tg04, Tg05, Tg0C, Tg0D, Tg0K, Tg0L, Tg0d, Tg0e): 57-67 C on a J700 during a glmark2 run. As on the M4 Pro, the SMC publishes them only while the GPU is powered; with the GPU off at idle they read placeholders between -4.0 and 0.2 C. Mark readings below 10 C as no data. They are read-only hwmon channels with no thermal zone or trip, added after the existing temperature nodes so the channel numbers the thermal zones use do not move. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8140-j700.dts | 53 ++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index e4634a965e642..a9acaf70b90d5 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1117,6 +1117,59 @@ label = "SMC Tsx1"; }; + /* + * GPU temperatures, last so the indexed channels above keep their + * numbers. The SMC publishes them only while the GPU is powered; with + * the GPU off they read -4.0 to 0.2 C placeholders, which are no data. + */ + temperature-Tg04 { + apple,key-id = "Tg04"; + label = "GPU Tg04"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg05 { + apple,key-id = "Tg05"; + label = "GPU Tg05"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0C { + apple,key-id = "Tg0C"; + label = "GPU Tg0C"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0D { + apple,key-id = "Tg0D"; + label = "GPU Tg0D"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0K { + apple,key-id = "Tg0K"; + label = "GPU Tg0K"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0L { + apple,key-id = "Tg0L"; + label = "GPU Tg0L"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0d { + apple,key-id = "Tg0d"; + label = "GPU Tg0d"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0e { + apple,key-id = "Tg0e"; + label = "GPU Tg0e"; + apple,no-data-below-millicelsius = <10000>; + }; + /* * Current, power and voltage rails: every other read-only float * I*, P* and V* key in the J700 SMC's key table. The units follow From 94c25f2c5b861ef6416d8d0c8a30153592687bfc Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:06:03 +1100 Subject: [PATCH 277/392] net: thunderbolt: Do not block the control channel on a login request tbnet_handle_packet() is the ThunderboltIP protocol handler. It is called from tb_xdomain_handle_request(), i.e. from the control channel Rx ring callback. For a login request it sends the response and then takes connection_lock to update the login state. tbnet_connected_work() holds connection_lock for the whole connection setup, including tb_xdomain_enable_paths(). Programming the DMA paths reads and writes the host router config space through the same control channel. When the remote host sends a second login request while we are connecting, which happens whenever both ends start logging in at the same time (for example after "ip link set thunderbolt0 down/up" on the remote end), the handler blocks on connection_lock, the control channel Rx ring stops making progress and the HOPS read in tb_path_activate() times out: thunderbolt-net 0-1.0 thunderbolt0: login successful, enabling paths ... activating DMA RX path from 0:1 to 0:7 thunderbolt-net 0-1.0 thunderbolt0: remote login request received thunderbolt-net 0-1.0 thunderbolt0: remote login response sent ... 0: timeout reading config space 0 from 0x10 ... 0: timeout reading config space 0 from 0x10 ... DMA RX path activation failed: -110 thunderbolt-net 0-1.0 thunderbolt0: failed to enable DMA paths On an Apple M4 Pro (T6040) host connected to an M2 this happened in 7 of 17 remote down/up cycles; the host router never answered a config read again until the link was reset, so the network did not come back. Every failing cycle had a second login request arriving within microseconds of the RX path activation, none of the passing ones did. Record the request under a spinlock in the handler and answer it from a worker instead. The worker takes connection_lock, so the response is also no longer transmitted on the control channel in the middle of the path setup. A newer request replaces one that has not been answered yet. The Rx HopID is only updated while the rings are stopped, since the one in use is released at teardown. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/thunderbolt/main.c | 122 ++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 23 deletions(-) diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 8f3a05e07d717..dcac37524056b 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -150,6 +150,22 @@ struct tbnet_ring { struct page_pool *pool; }; +/** + * struct tbnet_login_rx - Remote login request waiting for an answer + * @pending: A request has been received and not answered yet + * @route: Route the request came from + * @sequence: Sequence number of the request + * @command_id: Command ID of the request + * @transmit_path: HopID the remote host transmits on + */ +struct tbnet_login_rx { + bool pending; + u64 route; + u8 sequence; + u32 command_id; + int transmit_path; +}; + /** * struct tbnet - ThunderboltIP network driver private data * @svc: XDomain service the driver is bound to @@ -172,6 +188,9 @@ struct tbnet_ring { * teardown and NAPI enable state. * @login_retries: Number of login retries currently done * @login_work: Worker to send ThunderboltIP login packets + * @login_rx_lock: Protects @login_rx + * @login_rx: Latest remote login request that has not been answered yet + * @login_rx_work: Worker that answers @login_rx * @rx_refill_work: Worker to reschedule Rx polling after a refill failure * @connected_work: Worker that finalizes the ThunderboltIP connection * setup and enables DMA paths for high speed data @@ -205,6 +224,9 @@ struct tbnet { struct mutex connection_lock; int login_retries; struct delayed_work login_work; + spinlock_t login_rx_lock; + struct tbnet_login_rx login_rx; + struct work_struct login_rx_work; struct delayed_work rx_refill_work; struct work_struct connected_work; struct work_struct disconnect_work; @@ -333,6 +355,10 @@ static void start_login(struct tbnet *net) static void stop_login(struct tbnet *net) { + cancel_work_sync(&net->login_rx_work); + spin_lock(&net->login_rx_lock); + net->login_rx.pending = false; + spin_unlock(&net->login_rx_lock); cancel_delayed_work_sync(&net->login_work); cancel_work_sync(&net->connected_work); @@ -484,29 +510,27 @@ static int tbnet_handle_packet(const void *buf, size_t size, void *data) if (!netif_running(net->dev)) break; - ret = tbnet_login_response(net, route, sequence, - pkg->hdr.command_id); - if (!ret) { - netdev_dbg(net->dev, "remote login response sent\n"); - - mutex_lock(&net->connection_lock); - net->login_received = true; - net->remote_transmit_path = pkg->transmit_path; - - /* If we reached the number of max retries or - * previous logout, schedule another round of - * login retries - */ - if (net->login_retries >= TBNET_LOGIN_RETRIES || - !net->login_sent) { - net->login_retries = 0; - queue_delayed_work(system_long_wq, - &net->login_work, 0); - } - mutex_unlock(&net->connection_lock); - - queue_work(system_long_wq, &net->connected_work); - } + /* + * This runs in the control channel Rx path. Answering the + * request and updating the login state needs + * @connection_lock, which tbnet_connected_work() holds + * while it programs the DMA paths through this same + * control channel. Waiting for the lock here stalls the + * channel, so those config space accesses time out. The + * remote host can send a second login request while we + * are connecting (both ends log in at once), so hand the + * request to a worker. A newer request replaces one that + * has not been answered yet. + */ + spin_lock(&net->login_rx_lock); + net->login_rx.route = route; + net->login_rx.sequence = sequence; + net->login_rx.command_id = command_id; + net->login_rx.transmit_path = pkg->transmit_path; + net->login_rx.pending = true; + spin_unlock(&net->login_rx_lock); + + queue_work(system_long_wq, &net->login_rx_work); break; case TBIP_LOGOUT: @@ -733,6 +757,53 @@ static void tbnet_connected_work(struct work_struct *work) mutex_unlock(&net->connection_lock); } +static void tbnet_login_rx_work(struct work_struct *work) +{ + struct tbnet *net = container_of(work, typeof(*net), login_rx_work); + struct tbnet_login_rx req; + int ret; + + spin_lock(&net->login_rx_lock); + req = net->login_rx; + net->login_rx.pending = false; + spin_unlock(&net->login_rx_lock); + + if (!req.pending || !netif_running(net->dev)) + return; + + /* + * Answer under @connection_lock so that the response does not go + * out on the control channel while tbnet_connected_work() is + * programming the DMA paths. + */ + mutex_lock(&net->connection_lock); + ret = tbnet_login_response(net, req.route, req.sequence, + req.command_id); + if (ret) { + mutex_unlock(&net->connection_lock); + netdev_warn(net->dev, "failed to send ThunderboltIP response\n"); + return; + } + + netdev_dbg(net->dev, "remote login response sent\n"); + + net->login_received = true; + /* The Rx HopID in use is released at teardown, keep it until then */ + if (!net->rings_started) + net->remote_transmit_path = req.transmit_path; + + /* If we reached the number of max retries or previous logout, + * schedule another round of login retries + */ + if (net->login_retries >= TBNET_LOGIN_RETRIES || !net->login_sent) { + net->login_retries = 0; + queue_delayed_work(system_long_wq, &net->login_work, 0); + } + mutex_unlock(&net->connection_lock); + + queue_work(system_long_wq, &net->connected_work); +} + static void tbnet_login_work(struct work_struct *work) { struct tbnet *net = container_of(work, typeof(*net), login_work.work); @@ -1461,6 +1532,8 @@ static int tbnet_probe(struct tb_service *svc) INIT_DELAYED_WORK(&net->login_work, tbnet_login_work); INIT_DELAYED_WORK(&net->rx_refill_work, tbnet_rx_refill_work); INIT_WORK(&net->connected_work, tbnet_connected_work); + INIT_WORK(&net->login_rx_work, tbnet_login_rx_work); + spin_lock_init(&net->login_rx_lock); INIT_WORK(&net->disconnect_work, tbnet_disconnect_work); mutex_init(&net->connection_lock); spin_lock_init(&net->poll_lock); @@ -1531,6 +1604,7 @@ static void tbnet_remove(struct tb_service *svc) unregister_netdev(net->dev); /* A logout received while the interface was down queued this. */ cancel_work_sync(&net->disconnect_work); + cancel_work_sync(&net->login_rx_work); free_netdev(net->dev); } @@ -1551,6 +1625,8 @@ static int tbnet_suspend(struct device *dev) } tb_unregister_protocol_handler(&net->handler); + /* A login request may have arrived before the handler went away */ + cancel_work_sync(&net->login_rx_work); return 0; } From 9251a1cf428ec4c51217f725996aa3e4170457da Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:06:38 +1100 Subject: [PATCH 278/392] net: thunderbolt: Log in again when the DMA paths could not be enabled When tb_xdomain_enable_paths() fails, tbnet_connected_work() stops the rings and clears login_sent, and nothing else happens. If the remote end got its own paths up it keeps its carrier on and does not send another login request, so the link stays down until one of the ends is restarted. Queue a new login after TBNET_LOGIN_DELAY when connecting fails, up to three times per login round, so that a transient failure recovers on its own. A successful connection and start_login() reset the count. stop_login() cancels the login work a second time after the connect work is gone, since a failing connect attempt can queue it again. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/thunderbolt/main.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index dcac37524056b..f4d63584e6efe 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -34,6 +34,7 @@ #define TBNET_RING_SIZE 256 #define TBNET_LOGIN_RETRIES 60 +#define TBNET_CONNECT_RETRIES 3 #define TBNET_LOGOUT_RETRIES 10 #define TBNET_THROTTLING 128000 #define TBNET_E2E BIT(0) @@ -187,6 +188,8 @@ struct tbnet_login_rx { * @connection_lock: Lock serializing login state, connection setup, * teardown and NAPI enable state. * @login_retries: Number of login retries currently done + * @connect_retries: Number of new logins started after the DMA paths + * could not be enabled * @login_work: Worker to send ThunderboltIP login packets * @login_rx_lock: Protects @login_rx * @login_rx: Latest remote login request that has not been answered yet @@ -223,6 +226,7 @@ struct tbnet { int remote_transmit_path; struct mutex connection_lock; int login_retries; + int connect_retries; struct delayed_work login_work; spinlock_t login_rx_lock; struct tbnet_login_rx login_rx; @@ -347,6 +351,7 @@ static void start_login(struct tbnet *net) mutex_lock(&net->connection_lock); net->login_sent = false; net->login_received = false; + net->connect_retries = 0; mutex_unlock(&net->connection_lock); queue_delayed_work(system_long_wq, &net->login_work, @@ -361,6 +366,8 @@ static void stop_login(struct tbnet *net) spin_unlock(&net->login_rx_lock); cancel_delayed_work_sync(&net->login_work); cancel_work_sync(&net->connected_work); + /* A failed connection attempt may have started another login */ + cancel_delayed_work_sync(&net->login_work); netdev_dbg(net->dev, "login stopped\n"); } @@ -679,6 +686,21 @@ static void tbnet_connect_failed_locked(struct tbnet *net) /* Leave login_received set: only the peer can make it true again. */ lockdep_assert_held(&net->connection_lock); net->login_sent = false; + + /* + * The remote end may already consider the connection up and then + * never logs in again, so nothing would retry. Start a new login + * after a delay, a few times. + */ + if (netif_running(net->dev) && + net->connect_retries < TBNET_CONNECT_RETRIES) { + net->connect_retries++; + netdev_dbg(net->dev, "connection failed, new login in %u ms (%d/%d)\n", + TBNET_LOGIN_DELAY, net->connect_retries, + TBNET_CONNECT_RETRIES); + queue_delayed_work(system_long_wq, &net->login_work, + msecs_to_jiffies(TBNET_LOGIN_DELAY)); + } } static void tbnet_connected_work(struct work_struct *work) @@ -738,6 +760,7 @@ static void tbnet_connected_work(struct work_struct *work) goto err_stop_rings; } + net->connect_retries = 0; netif_carrier_on(net->dev); netif_start_queue(net->dev); From cf616e5f2ce8eb5ebca0711144eb1626bf2d5562 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 9 Nov 2026 11:11:46 -0300 Subject: [PATCH 279/392] drm/asahi: g17: keep the timestamp window base a literal in initdata The host tests g17-t6041-initdata-tests and t6040-calibration-tests compile fw/initdata.rs and v25g83.rs on their own, without crate::hw, so naming hw::t8140::dynamic::TIMESTAMP_WINDOW in HwData::init broke both. Write the base as a literal again, and assert in hw/t8140.rs that TIMESTAMP_WINDOW still starts there, so the two cannot drift apart. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3 ++- drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 3 ++- drivers/gpu/drm/asahi/hw/t8140.rs | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 6cf84c744eebd..caf2ba6b658e6 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1097,7 +1097,8 @@ impl HwData { self.unk_10 = 0x10_0000_0000; self.unk_18 = 0x10_0000_0000; self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = crate::hw::t8140::dynamic::TIMESTAMP_WINDOW.start; + // hw::t8140::dynamic::TIMESTAMP_WINDOW.start; a literal so the host tests build this file alone. + self.unk_28 = 0xffff_fc21_8140_0000; self.unk_e0 = Self::UNK_E0; self.unk_2d8 = Self::UNK_2D8; self.unk_3e0 = Self::UNK_3E0; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index af0cf7d55fe72..18ff53cb4eafc 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -291,7 +291,8 @@ impl HwData25G83 { self.unk_10 = 0x10_0000_0000; self.unk_18 = 0x10_0000_0000; self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = crate::hw::t8140::dynamic::TIMESTAMP_WINDOW.start; + // hw::t8140::dynamic::TIMESTAMP_WINDOW.start; a literal so the host tests build this file alone. + self.unk_28 = 0xffff_fc21_8140_0000; let mut used = [false; REGISTER_MAP_SLOTS]; for window in args.windows { diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 7a9591ea65aba..72c172e14b818 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -197,6 +197,8 @@ pub(crate) mod dynamic { /// are aliased here so that the firmware writes their stamps itself. pub(crate) const TIMESTAMP_WINDOW: core::ops::Range = 0xffff_fc21_8140_0000..0xffff_fc21_8540_0000; + // `HwData::init` (initdata.rs and v25g83.rs) writes this start as a literal. + const _: () = assert!(TIMESTAMP_WINDOW.start == 0xffff_fc21_8140_0000); /// Dynamic kernel objects in the upper window: from `KERNEL_OFFSET` up to the firmware /// MMIO aperture. The firmware gives the addresses from there on a fixed meaning. From 16ddd156cf90ce3161a7be5ff82c468b142741af Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 9 Nov 2026 11:11:46 -0300 Subject: [PATCH 280/392] drm/asahi: g17: stop handing out window stamps the firmware does not write Destination::write_window() waits up to 40 x 5 us for the firmware's fragment end stamp before the host writes it. The stamp has always landed by the first read, but if the firmware ever stops writing through the window (the mapping refused, another firmware), every render with a fragment end stamp would pay the full 200 us in retirement. After 8 host fallbacks in a row, hand no render a window address again; its stamps then go the host path with no wait. A stamp the firmware writes resets the count. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/gpu/drm/asahi/g17/completion.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 5116e4a3b08d0..97d4a6ea98cc2 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -19,7 +19,9 @@ use crate::{ hw::t8140::{dynamic, resources as cfg}, mmu, }; -use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU64, AtomicU8, Ordering}; +use core::sync::atomic::{ + fence as barrier, AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, Ordering, +}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, new_mutex, @@ -77,6 +79,7 @@ impl Destination { while self.buffer.read_timestamp(self.offset) != Some(ns) { if polls == WINDOW_POLLS { WINDOW_FALLBACKS.fetch_add(1, Ordering::Relaxed); + WINDOW_MISSES.fetch_add(1, Ordering::Relaxed); self.write(ticks); return; } @@ -84,6 +87,7 @@ impl Destination { polls += 1; } WINDOW_KEPT.fetch_add(1, Ordering::Relaxed); + WINDOW_MISSES.store(0, Ordering::Relaxed); } } @@ -100,6 +104,11 @@ const WINDOW_POLLS: u32 = 40; const WINDOW_POLL_US: i64 = 5; static WINDOW_KEPT: AtomicU64 = AtomicU64::new(0); static WINDOW_FALLBACKS: AtomicU64 = AtomicU64::new(0); +/// Host fallbacks in a row. At [`WINDOW_GIVE_UP`] the firmware is taken not to write through +/// the window at all (mapping refused, other firmware), and no render is handed a window +/// address again, so none pays the full wait. +static WINDOW_MISSES: AtomicU32 = AtomicU32::new(0); +const WINDOW_GIVE_UP: u32 = 8; /// `capacity`: firmware-written end stamps kept, and those the host had to write. pub(crate) fn window_counters() -> [u64; 2] { @@ -149,7 +158,7 @@ impl Destinations { /// Window addresses for a render's vertex and fragment start/end destinations: the fragment /// end's if it is mapped in the window, 0 for the others. fn window_vas(&self) -> [u64; 4] { - if !self.render { + if !self.render || WINDOW_MISSES.load(Ordering::Relaxed) >= WINDOW_GIVE_UP { return [0; 4]; } let vas: [u64; 4] = core::array::from_fn(|i| { From e847be17c139b0aff6f272fd5a741a4bb8de4d57 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:11:48 +1100 Subject: [PATCH 281/392] thunderbolt: apple: Restart the port when the peer link is lost When the peer on the USB4 port drops the link but the cable stays attached, the ACIO firmware puts the port group into its dead end state and waits for the host to power the block off: ... lane_init.c:1110: SBX disconnected! ... port_group.c:1528: Port Group - entering Dead End state thunderbolt-apple-nhi 381f00000.nhi: 0:1: xdomain unplugged ... port_group.c:1511: ACIO was not powered off after 2 seconds in Dead-End state ... (repeated every 2 s) A peer that reboots or powers off is reported by the Type-C controller as a cable change about 0.5 s later, which stops and restarts the ACIO, so the port recovers. When no cable change follows, for example when the peer resets its own port right after the lanes were bonded, nothing restarts it and the port stays down until a reboot. This was seen on an M4 Pro (T6040) connected to an M2 in 1 of 17 M2 reboots: the link came up while the M2 was booting, the M2 reset its port 0.5 s after bonding, and the M4 Pro port never trained again. Add a link_lost hook to struct tb_nhi_ops that the software connection manager calls when the router or host on a host router lane adapter is unplugged. The Apple NHI arms a 3 s check. If the cable is still attached, the port was not restarted meanwhile and no peer came back, it asks CD321x to revalidate the cable with a failed connection check, which cycles the ACIO and the PHY. Do this at most three times until a peer is found again. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/thunderbolt/apple.c | 113 +++++++++++++++++++++++++++++++++++- drivers/thunderbolt/nhi.h | 5 ++ drivers/thunderbolt/tb.c | 11 ++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 3374e0d3e794f..d026801b283f9 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -217,6 +217,14 @@ static const struct apple_cio_hw apple_cio_t6040_hw = { #define APPLE_CIO_SLEEP_RESTART_FALLBACK_MS 10000 /* How long a forced cable cycle may take before it no longer counts. */ #define APPLE_CIO_RESUME_RECOVERY_WINDOW_MS 30000 +/* + * After a peer drops the link without the cable going away, the ACIO + * firmware parks the port and waits for the host to power it off. A peer + * that reboots or goes away electrically is reported as a cable change + * within about 0.5 s and restarts the port anyway. + */ +#define APPLE_CIO_LINK_LOST_MS 3000 +#define APPLE_CIO_LINK_LOST_CYCLES 3 static bool resume_recovery; module_param(resume_recovery, bool, 0644); @@ -325,6 +333,12 @@ struct apple_cio { bool resume_recovery_failed; /* terminal recovery failure */ bool resume_link_failed; /* protected by lock */ bool resume_link_pending; /* set by NHI resume, consumed by ACIO complete */ + struct delayed_work link_lost_work; + unsigned int start_gen; /* protected by lock: bumped by every start */ + unsigned int link_lost_gen; /* start_gen when the peer went away */ + unsigned int link_lost_cycles; /* protected by lock */ + u64 link_lost_cycle_activity; /* remote activity at the last cycle */ + bool link_lost_restart; /* protected by lock: cycle requested */ struct notifier_block pm_notifier; struct delayed_work sleep_restart_work; enum { @@ -2053,8 +2067,27 @@ static bool apple_nhi_add_links(struct tb_nhi *nhi) return true; } +/* + * Called with the domain lock held when the peer on a host router lane + * adapter went away: only (re)arm the check. + */ +static void apple_nhi_link_lost(struct tb_nhi *nhi) +{ + struct apple_nhi *anhi = nhi_to_anhi(nhi); + struct apple_cio *acio = anhi->acio; + u64 activity = atomic64_read(&anhi->tb->remote_activity); + + /* A peer was found since the last cycle: that cycle worked */ + if (activity != READ_ONCE(acio->link_lost_cycle_activity)) + WRITE_ONCE(acio->link_lost_cycles, 0); + WRITE_ONCE(acio->link_lost_gen, READ_ONCE(acio->start_gen)); + mod_delayed_work(system_freezable_wq, &acio->link_lost_work, + msecs_to_jiffies(APPLE_CIO_LINK_LOST_MS)); +} + static const struct tb_nhi_ops apple_nhi_ops = { .add_links = apple_nhi_add_links, + .link_lost = apple_nhi_link_lost, .request_ring_irq = apple_nhi_request_irq, .release_ring_irq = apple_nhi_release_irq, .ring_interrupt_active = apple_nhi_ring_interrupt_active, @@ -2689,6 +2722,7 @@ static void apple_cio_stop(struct apple_cio *acio) acio->current_cable_info = 0; acio->resume_link_failed = false; + acio->link_lost_restart = false; } /* Only called while the ACIO power domains are on. */ @@ -2905,6 +2939,7 @@ static int apple_cio_start(struct apple_cio *acio) } acio->current_cable_info = acio->target_cable_info; + acio->start_gen++; return 0; err_depopulate: @@ -2935,7 +2970,7 @@ static int apple_cio_check_connection(struct apple_cio *acio) lockdep_assert_held(&acio->lock); /* Ask CD321x to cycle ACIO and the PHY using a fresh cable snapshot. */ - if (acio->resume_link_failed) + if (acio->resume_link_failed || acio->link_lost_restart) return -ENOLINK; guard(mutex)(&acio->pcie_tunnel_lock); if (!READ_ONCE(acio->nhi_pdev) || acio->pcie_tunnel_stopping || @@ -3000,6 +3035,79 @@ static int apple_cio_link_connected(struct apple_cio *acio) return verdict; } +/* Whether a router or host is connected to a lane adapter of the host router */ +static bool apple_cio_has_peer(struct apple_cio *acio, u64 *activity) +{ + struct platform_device *pdev = READ_ONCE(acio->nhi_pdev); + struct apple_nhi *anhi; + struct tb_port *port; + bool peer = true; + + lockdep_assert_held(&acio->lock); + if (!pdev) + return true; + device_lock(&pdev->dev); + anhi = platform_get_drvdata(pdev); + if (!pdev->dev.driver || !anhi || !anhi->tb) + goto unlock; + + scoped_guard(mutex, &anhi->tb->lock) { + *activity = atomic64_read(&anhi->tb->remote_activity); + peer = false; + tb_switch_for_each_port(anhi->tb->root_switch, port) { + if (!tb_port_is_null(port)) + continue; + if ((port->xdomain && !port->xdomain->is_unplugged) || + (port->remote && !port->remote->sw->is_unplugged)) + peer = true; + } + } +unlock: + device_unlock(&pdev->dev); + return peer; +} + +/* + * The peer dropped the link but the cable is still attached and no cable + * change followed. The firmware leaves the port down until it is powered + * off, so have CD321x cycle it with a fresh cable snapshot, a few times. + */ +static void apple_cio_link_lost_work(struct work_struct *work) +{ + struct apple_cio *acio = container_of(to_delayed_work(work), + struct apple_cio, link_lost_work); + u64 activity = 0; + bool notify = false; + + scoped_guard(mutex, &acio->lock) { + if (acio->removing || !acio->current_cable_info || + !acio->target_cable_info || + acio->sleep_phase != APPLE_CIO_SLEEP_OPEN || + acio->resume_recovery_failed || + acio->start_gen != READ_ONCE(acio->link_lost_gen)) + return; + if (apple_cio_has_peer(acio, &activity)) + return; + if (acio->link_lost_cycles >= APPLE_CIO_LINK_LOST_CYCLES) { + dev_warn(acio->dev, + "peer link still down after %u port restarts, leaving the port alone\n", + acio->link_lost_cycles); + return; + } + acio->link_lost_cycles++; + WRITE_ONCE(acio->link_lost_cycle_activity, activity); + acio->link_lost_restart = true; + dev_info(acio->dev, + "peer link lost with the cable attached, restarting the port (%u/%u)\n", + acio->link_lost_cycles, APPLE_CIO_LINK_LOST_CYCLES); + notify = true; + } + + /* CD321x may tear down the NHI: never notify under either driver lock. */ + if (notify) + typec_thunderbolt_switch_notify(acio->tbt_switch); +} + static void apple_cio_resume_link_work(struct work_struct *work) { struct apple_cio *acio = container_of(to_delayed_work(work), @@ -3043,6 +3151,7 @@ static int apple_cio_prepare(struct device *dev) /* Drain before taking lock; the worker acquires it to check the link. */ cancel_delayed_work_sync(&acio->resume_link_work); + cancel_delayed_work_sync(&acio->link_lost_work); scoped_guard(mutex, &acio->lock) { WRITE_ONCE(acio->resume_link_pending, false); acio->resume_link_failed = false; @@ -3331,6 +3440,7 @@ static int apple_cio_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&acio->pcie_tunnel_work, apple_cio_pcie_tunnel_work); INIT_DELAYED_WORK(&acio->resume_link_work, apple_cio_resume_link_work); + INIT_DELAYED_WORK(&acio->link_lost_work, apple_cio_link_lost_work); INIT_DELAYED_WORK(&acio->sleep_restart_work, apple_cio_sleep_restart_work); init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; @@ -3531,6 +3641,7 @@ static void apple_cio_remove(struct platform_device *pdev) disable_delayed_work_sync(&acio->sleep_restart_work); disable_delayed_work_sync(&acio->resume_link_work); + disable_delayed_work_sync(&acio->link_lost_work); /* The worker can notify this switch; stop current and future enqueues. */ disable_delayed_work_sync(&acio->pcie_tunnel_work); /* A set may hold the switch read lock while waiting for acio->lock. */ diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 75795fbf865a1..dbf1e4b407c52 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -106,6 +106,10 @@ struct tb_nhi_ring_layout { * %true if the host orders all of its tunneled native ports, so * none is left without a link. * @is_present: Whether the device is currently present on the parent bus + * @link_lost: Called with the domain lock held when the router or host + * connected to a lane adapter of the host router is unplugged. + * Lets the NHI recover a port whose link went down while the + * cable stays attached. * @init_interrupts: NHI specific interrupt initialization hook * @reset_interface: Resets the host interface */ @@ -134,6 +138,7 @@ struct tb_nhi_ops { struct tb_port *out, u64 generation); bool (*add_links)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); + void (*link_lost)(struct tb_nhi *nhi); void (*dp_tunnel_changed)(struct tb_nhi *nhi, u8 in_port, u64 generation, bool active); bool (*dp_tunnel_awaits_display)(struct tb_nhi *nhi, struct tb_port *in); int (*init_interrupts)(struct tb_nhi *nhi); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 19c1389c63d87..ad59f4ede6bfc 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -3258,6 +3258,15 @@ static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, /* hotplug handling */ +/* The partner on a host router lane adapter went away */ +static void tb_host_link_lost(struct tb *tb, struct tb_port *port) +{ + if (tb_route(port->sw) || !tb_port_is_null(port)) + return; + if (tb->nhi->ops && tb->nhi->ops->link_lost) + tb->nhi->ops->link_lost(tb->nhi); +} + /* * tb_handle_hotplug() - handle hotplug event * @@ -3322,6 +3331,7 @@ static void tb_handle_hotplug(struct work_struct *work) tb_recalc_estimated_bandwidth(tb); tb_tunnel_dp(tb); tb_dp_host_tunnel_gone(tb, dp_up); + tb_host_link_lost(tb, port); } else if (port->xdomain) { struct tb_xdomain *xd = tb_xdomain_get(port->xdomain); @@ -3339,6 +3349,7 @@ static void tb_handle_hotplug(struct work_struct *work) __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1); tb_xdomain_put(xd); tb_port_unconfigure_xdomain(port); + tb_host_link_lost(tb, port); } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) { tb_dp_resource_unavailable(tb, port, "adapter unplug"); /* A display went: the DP tunnel left may take a higher rate. */ From 688c1343d5f88184586aab46df1848a41194876d Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Sun, 8 Nov 2026 17:09:37 -0300 Subject: [PATCH 282/392] soc: apple: sep: open the xART guard's list per access The xART guard opens /var/lib/aurora-sep-xart-owned.bin read-write when the xART store is set up and keeps it in the guard, which lives as long as the store: until the device is unbound. Nothing unbinds it before a reboot, so the kernel holds a writer on the file system that holds /var/lib until power-off. systemd-shutdown's read-only remount of that file system then fails with EBUSY and it stays read-write. The host-state store has the same problem, and #583 fixes that one. Both files have to be closed before the file system unmounts cleanly: on a J616s with the host-state store fixed, the root still was not remounted read-only at shutdown, and every boot still replayed the ext4 journal. Open the list for each access instead: read-only for size, reads and syncs, existing-only for writes. Only Guard::open() creates it. The create-and-adopt rule in with_file() is unchanged: the directory is still flushed before a list is adopted. The list format and the KUnit tests, which use an in-memory file, are unchanged. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- drivers/soc/apple/xart_guard.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/xart_guard.rs b/drivers/soc/apple/xart_guard.rs index 5fe73719e3e33..ee60f63da5e51 100644 --- a/drivers/soc/apple/xart_guard.rs +++ b/drivers/soc/apple/xart_guard.rs @@ -399,29 +399,40 @@ pub(crate) trait ListFile { fn sync_parent(&self) -> Result<()>; } -impl ListFile for StoreFile { +/// Linux's list at [`OWNED_PATH`], opened for each access and closed again. +/// +/// The guard lives as long as the xART store, which the driver keeps until it +/// is unbound, and nothing unbinds it before a reboot. Holding the list open +/// for writing that long keeps a writer on the root file system, so the +/// read-only remount of `/` at shutdown fails with EBUSY. Writes never create +/// the file: only [`Guard::open`] does that. +pub(crate) struct HostListFile; + +impl ListFile for HostListFile { fn size(&self) -> Result { - StoreFile::size(self) + StoreFile::open_readonly(OWNED_PATH)?.size() } fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { - StoreFile::read_exact(self, off, buf) + StoreFile::open_readonly(OWNED_PATH)?.read_exact(off, buf) } fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { - StoreFile::write_all(self, off, buf) + StoreFile::open_writable_existing(OWNED_PATH)?.write_all(off, buf) } + // fsync on a new descriptor still flushes the writes made through earlier + // ones, and reports a writeback error that no descriptor has seen yet. fn sync(&self) -> Result<()> { - StoreFile::sync(self) + StoreFile::open_readonly(OWNED_PATH)?.sync() } fn sync_parent(&self) -> Result<()> { - StoreFile::sync_parent(self) + StoreFile::open_readonly(OWNED_PATH)?.sync_parent() } } -pub(crate) struct Guard { +pub(crate) struct Guard { level: u8, /// `None` when the list could not be opened: nothing can be adopted, so /// every session write that is not already listed is refused. @@ -452,7 +463,8 @@ impl Guard { let mut guard = Guard::with_file( level, existing, - StoreFile::open_host(OWNED_PATH).map(|(file, _created)| file), + // Only to create the list if absent; the handle closes here. + StoreFile::open_host(OWNED_PATH).map(|_| HostListFile), ); guard.allowance = take_allowance(); guard From d39547b6a405c448d20a9e09c35307ce1957eb5e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:01:34 +1100 Subject: [PATCH 283/392] drm/asahi: agx_stats: skip the idle gap in the first busy window busy_ns integrates Utilization windows: each message closes the window that started at the previous one and counts it at its busy percentage. The firmware sends no Utilization while the GPU is powered off, so the first message after power-up spanned the whole idle gap and counted it at the new load. On T6000 and T8112 busy_ns rose 2.4-2.6 s above elapsed time in the first window of a load that followed idle. Drop the window baseline when PowerOff or PowerOn arrives, so the first Utilization after power-up only starts a window. Move the integration into a small BusyIntegrator (stats_busy.rs) with KUnit cases for the weighting, the whole-second split, the power gap, a timestamp that goes back and saturation; tools/asahi/agx-stats-busy-tests.py runs them on the host, with a negative control that removes the baseline drop. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/channel.rs | 56 +++++--------- drivers/gpu/drm/asahi/stats.rs | 6 +- drivers/gpu/drm/asahi/stats_busy.rs | 114 ++++++++++++++++++++++++++++ tools/asahi/agx-stats-busy-tests.py | 64 ++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 6 files changed, 203 insertions(+), 39 deletions(-) create mode 100644 drivers/gpu/drm/asahi/stats_busy.rs create mode 100755 tools/asahi/agx-stats-busy-tests.py diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 1a5cf1ba9a7c1..a9ed7a204b3b8 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -40,6 +40,7 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_busy; mod stats_raw; mod sysfs_exports; mod timestamp; diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 5a14982dca053..dbb84f288087b 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -595,13 +595,11 @@ pub(crate) struct StatsChannel { /// Last raw sample of every statistics kind, shared with the export /// through `crate::stats_raw`. raw: Arc>, - /// Rate of firmware stats-message timestamps (Hz). All stats timestamps - /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every - /// supported SoC), not nanoseconds. - ts_hz: u64, - /// Last `Utilization` message timestamp (raw base-clock ticks), for - /// utilization-weighted `busy_ns` integration. - last_util_ts: u64, + /// Utilization-weighted `busy_ns` integration across `Utilization` + /// windows. All stats timestamps run on the chip base clock + /// (`HwConfig::base_clock_hz`, 24 MHz on every supported SoC), not + /// nanoseconds. + busy: crate::stats_busy::BusyIntegrator, } #[versions(AGX)] @@ -619,8 +617,7 @@ impl StatsChannel::ver { ch: RxChannel::::new(alloc, 0x100)?, snap, raw, - ts_hz, - last_util_ts: 0, + busy: crate::stats_busy::BusyIntegrator::new(ts_hz), }) } @@ -666,37 +663,20 @@ impl StatsChannel::ver { // across a saturated 30 s matmul and 0-2 on // an idle desktop, while FwBusy timestamp // deltas (the prior producer) covered 0.17% - // of that matmul. The timestamps are base- - // clock ticks (24 MHz), so convert to ns - // once here. The kernel runtime has no - // 128-bit division, so split whole seconds - // from the tick remainder: whole seconds - // contribute util * 1e7 ns each (1e9 / 100 - // exactly), the remainder contributes - // rem * util * 1e9 / (100 * ts_hz), which - // cannot overflow u64 (rem < ts_hz <= 24e6 - // on supported SoCs). - let util = u64::from( - [*util1, *util2, *util3, *util4] - .into_iter() - .max() - .unwrap_or(0) - .min(100), - ); - let ts = timestamp.0; - if self.last_util_ts != 0 && ts > self.last_util_ts { - let delta = ts - self.last_util_ts; - let whole_s = delta / self.ts_hz; - let rem = delta % self.ts_hz; - let busy = whole_s - .saturating_mul(util) - .saturating_mul(10_000_000) - .saturating_add( - rem * util * 1_000_000_000 / (100 * self.ts_hz), - ); + // of that matmul. + let util = [*util1, *util2, *util3, *util4] + .into_iter() + .max() + .unwrap_or(0); + let busy = self.busy.utilization(timestamp.0, util); + if busy != 0 { self.snap.busy_ns.fetch_add(busy, Ordering::Relaxed); } - self.last_util_ts = ts; + } + // No Utilization is sent while the GPU is off: + // the next window must not span the gap. + StatsMsg::ver::PowerOn { .. } | StatsMsg::ver::PowerOff { .. } => { + self.busy.power_changed(); } StatsMsg::ver::PowerState { pstate, .. } => { self.snap.pstate.store(*pstate, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 387ff578369a8..b8636d4c5eb70 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -26,7 +26,11 @@ //! derived from successive `Utilization` windows (busiest subqueue //! percentage x window duration; the firmware timestamps are base-clock //! ticks, 24 MHz, converted with `HwConfig::base_clock_hz`). Monotonic and -//! bounded by elapsed time. Saturates at u64::MAX. +//! bounded by elapsed time. Saturates at u64::MAX. A window only counts +//! while the GPU stayed powered: `PowerOff`/`PowerOn` drop the baseline, +//! so the idle gap before the first window after power-up is not credited +//! at that window's (busy) percentage. See +//! [`crate::stats_busy::BusyIntegrator`]. //! - `jobs`: completed submissions since boot, counted at submission //! completion (`JobFence::command_complete`). //! diff --git a/drivers/gpu/drm/asahi/stats_busy.rs b/drivers/gpu/drm/asahi/stats_busy.rs new file mode 100644 index 0000000000000..b354d3d0ca5e8 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats_busy.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Utilization-weighted busy time for the legacy `agx_stats` `busy_ns` field. + +/// Integrates `Utilization` windows into utilization-weighted busy time. +/// +/// Each `Utilization` message carries a base-clock timestamp and the busy +/// percentage of the window that ends there; the window starts at the +/// previous message. The firmware sends no `Utilization` while the GPU is +/// powered off, so the first message after power-up would otherwise span +/// the whole idle gap and credit it at the new, typically 100%, load (the +/// first window after idle read 2.4-2.6 s too high on T6000 and T8112). +/// `PowerOff` and `PowerOn` therefore drop the baseline: the first window +/// after power-up only sets it. That loses at most the one short window in +/// which the GPU powered up, instead of crediting the idle gap. +pub(crate) struct BusyIntegrator { + ts_hz: u64, + last_ts: Option, +} + +impl BusyIntegrator { + /// `ts_hz` is the rate of the firmware timestamps (the chip base clock). + pub(crate) fn new(ts_hz: u64) -> Self { + Self { + ts_hz: ts_hz.max(1), + last_ts: None, + } + } + + /// The GPU powered off or on: the next window has no valid start. + pub(crate) fn power_changed(&mut self) { + self.last_ts = None; + } + + /// Account one `Utilization` message at `ts` (base-clock ticks) with + /// busiest-subqueue percentage `util`; returns the busy nanoseconds of + /// the window it closes. + /// + /// The kernel runtime has no 128-bit division, so whole seconds + /// contribute `util * 1e7` ns each (1e9 / 100 exactly) and the tick + /// remainder `rem * util * 1e9 / (100 * ts_hz)`, which cannot overflow + /// u64 (rem < ts_hz <= 24e6 on supported SoCs). + pub(crate) fn utilization(&mut self, ts: u64, util: u32) -> u64 { + let util = u64::from(util.min(100)); + let busy = match self.last_ts { + Some(last) if ts > last => { + let delta = ts - last; + let whole_s = delta / self.ts_hz; + let rem = delta % self.ts_hz; + whole_s + .saturating_mul(util) + .saturating_mul(10_000_000) + .saturating_add(rem * util * 1_000_000_000 / (100 * self.ts_hz)) + } + _ => 0, + }; + self.last_ts = Some(ts); + busy + } +} + +#[kernel::macros::kunit_tests(asahi_stats_busy)] +mod tests { + use super::*; + + const HZ: u64 = 24_000_000; + + #[test] + fn windows_are_weighted_by_utilization() { + let mut b = BusyIntegrator::new(HZ); + assert_eq!(b.utilization(1_000, 100), 0); + // 10 ms at 100%, then 10 ms at 50%. + assert_eq!(b.utilization(1_000 + 240_000, 100), 10_000_000); + assert_eq!(b.utilization(1_000 + 480_000, 50), 5_000_000); + // Over 100% is clamped. + assert_eq!(b.utilization(1_000 + 720_000, 250), 10_000_000); + } + + #[test] + fn long_windows_split_whole_seconds() { + let mut b = BusyIntegrator::new(HZ); + b.utilization(0, 0); + // 2.5 s at 100%. A first timestamp of 0 is a valid baseline. + assert_eq!(b.utilization(60_000_000, 100), 2_500_000_000); + } + + #[test] + fn idle_gap_after_power_off_is_not_credited() { + let mut b = BusyIntegrator::new(HZ); + b.utilization(HZ, 0); + // PowerOff, then PowerOn 2.5 s later. + b.power_changed(); + b.power_changed(); + // First window after power-up: baseline only. + assert_eq!(b.utilization(HZ + 60_000_000 + 240_000, 100), 0); + assert_eq!(b.utilization(HZ + 60_000_000 + 480_000, 100), 10_000_000); + } + + #[test] + fn a_timestamp_that_goes_back_only_rebases() { + let mut b = BusyIntegrator::new(HZ); + b.utilization(5 * HZ, 100); + assert_eq!(b.utilization(HZ, 100), 0); + assert_eq!(b.utilization(HZ + 240_000, 100), 10_000_000); + assert_eq!(b.utilization(HZ + 240_000, 100), 0); + } + + #[test] + fn huge_windows_saturate() { + let mut b = BusyIntegrator::new(1); + b.utilization(1, 100); + assert_eq!(b.utilization(u64::MAX, 100), u64::MAX); + } +} diff --git a/tools/asahi/agx-stats-busy-tests.py b/tools/asahi/agx-stats-busy-tests.py new file mode 100755 index 0000000000000..8854add06bc0c --- /dev/null +++ b/tools/asahi/agx-stats-busy-tests.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the agx_stats busy_ns integrator's KUnit cases on the host. + +BusyIntegrator (drivers/gpu/drm/asahi/stats_busy.rs) turns the firmware's +Utilization windows into busy_ns. The cases pin that a window is weighted by +its utilization, that whole seconds are split from the tick remainder, and +that a PowerOff/PowerOn drops the baseline so the idle gap is never credited. + +--negative-control turns power_changed() into a no-op and requires the cases +to fail, so the harness is shown to catch the over-count it guards against. +""" + +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +STATS = ROOT / "drivers/gpu/drm/asahi/stats_busy.rs" +ATTR = "#[kernel::macros::kunit_tests(asahi_stats_busy)]" + + +def source(negative): + text = STATS.read_text() + start = text.index("pub(crate) struct BusyIntegrator") + body = text[start:] + assert ATTR in body, "KUnit suite attribute moved" + body = body.replace(ATTR, "#[cfg(test)]") + if negative: + old = " self.last_ts = None;\n" + assert body.count(old) == 1 + body = body.replace(old, "") + return "#![allow(dead_code)]\n" + body + + +def run(negative): + with tempfile.TemporaryDirectory(prefix="asahi-busy-") as tmp: + main = Path(tmp) / "main.rs" + main.write_text(source(negative)) + binary = Path(tmp) / "test" + subprocess.run(["rustc", "--edition=2021", "--test", "-Dwarnings", str(main), "-o", str(binary)], + check=True) + return subprocess.run([str(binary), "--test-threads=1"], + capture_output=negative).returncode + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--negative-control", action="store_true") + args = parser.parse_args() + rc = run(args.negative_control) + if args.negative_control: + if rc == 0: + print("negative control: the cases passed without the baseline drop") + return 1 + print("negative control: the cases fail without the baseline drop (expected)") + return 0 + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index b878404b2d9fa..1937bd816f795 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -71,6 +71,7 @@ def mutations(flag, names, base, expect=KILL): 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'asahi-stats-raw-tests.py': [([], PASS)], + 'agx-stats-busy-tests.py': [([], PASS), (['--negative-control'], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], 'avd-av1-tail-tests.py': [(out(), PASS)] + [(out() + ['--case', c], PASS) for c in ('copy', 'seed', 'aligned')] From 6c8b0f71ed35db01b335b82d3e31c76d3b9a44e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:13:52 +1100 Subject: [PATCH 284/392] usb: typec: tipd: Take the USB4 router again after it rebinds CD321x looks up the USB4/Thunderbolt switch of its router once, at probe (or when the router binds late). When the router driver unbinds and binds again, for example "echo 381ac0000.cio > .../unbind" and "bind" for the Apple ACIO, the old switch is unregistered and the router registers a new one. CD321x keeps the old handle, every cable update then fails with -ENODEV, it ends with "cable revalidation failed", and the port never comes back: the ACIO is not started again, there is no NHI and no link until a reboot. Add typec_thunderbolt_switch_is_live() so that a consumer can tell that the provider of its handle went away. Watch for switch registration on every port with a router, not only on a port that probed without it. When the router registers a switch while the one in use is gone, move the notifier to the new switch, drop the old handle and reapply the cable state as for a router that binds late. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/usb/typec/mux.c | 20 ++++++++++++ drivers/usb/typec/tipd/cd321x-tbt-test.c | 40 ++++++++++++++++++++++++ drivers/usb/typec/tipd/core.c | 33 ++++++++++++++----- include/linux/usb/typec_mux.h | 7 +++++ 4 files changed, 93 insertions(+), 7 deletions(-) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index db28d50cf61b6..71293c645c51b 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -866,6 +866,26 @@ void typec_thunderbolt_switch_put(struct typec_thunderbolt_switch *sw) } EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_put); +/** + * typec_thunderbolt_switch_is_live - Whether the switch provider is still bound + * @sw: USB4/Thunderbolt switch + * + * A handle outlives the provider: once the provider unregisters its switch, + * typec_thunderbolt_switch_set() fails with -ENODEV. A provider that binds + * again registers a new switch, which must be looked up again. + * + * Return: %true if @sw is valid and its provider is still registered. + */ +bool typec_thunderbolt_switch_is_live(struct typec_thunderbolt_switch *sw) +{ + if (IS_ERR_OR_NULL(sw)) + return false; + + guard(rwsem_read)(&sw->sw_dev->set_lock); + return sw->sw_dev->set; +} +EXPORT_SYMBOL_GPL(typec_thunderbolt_switch_is_live); + /** * typec_thunderbolt_switch_set - Forward cable details to USB4/Thunderbolt switch * @sw: USB4/Thunderbolt switch diff --git a/drivers/usb/typec/tipd/cd321x-tbt-test.c b/drivers/usb/typec/tipd/cd321x-tbt-test.c index ee46db0734c41..5ed8a2ab136ce 100644 --- a/drivers/usb/typec/tipd/cd321x-tbt-test.c +++ b/drivers/usb/typec/tipd/cd321x-tbt-test.c @@ -367,6 +367,45 @@ static void cd321x_tbt_router_binds_after_removal_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd->tbt_notifier_registered); } +/* A router that unbinds and binds again is taken again; the cable is reread. */ +static void cd321x_tbt_router_rebinds_test(struct kunit *test) +{ + struct cd321x *cd = cd321x_tbt_test_late_port(test); + struct typec_thunderbolt_switch_desc desc = { + .fwnode = software_node_fwnode(&cd321x_test_router), + .set = cd321x_tbt_test_set, + .name = "router-first", + }; + struct typec_thunderbolt_switch_dev *sw_dev; + struct device *router; + int ret; + + router = kunit_device_register(test, "router-first"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, router); + sw_dev = typec_thunderbolt_switch_register(router, &desc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sw_dev); + + ret = cd321x_tbt_watch_late(cd, software_node_fwnode(&cd321x_test_connector)); + KUNIT_ASSERT_EQ(test, ret, 0); + flush_work(&cd->tbt_attach_work); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cd->tbt_switch); + KUNIT_EXPECT_TRUE(test, typec_thunderbolt_switch_is_live(cd->tbt_switch)); + KUNIT_EXPECT_EQ(test, atomic_read(&cd->restore_event), 1); + atomic_set(&cd->restore_event, 0); + + typec_thunderbolt_switch_unregister(sw_dev); + KUNIT_EXPECT_FALSE(test, typec_thunderbolt_switch_is_live(cd->tbt_switch)); + + cd321x_tbt_test_add_router(test, &cd321x_test_router); + flush_work(&cd->tbt_attach_work); + KUNIT_EXPECT_TRUE(test, typec_thunderbolt_switch_is_live(cd->tbt_switch)); + KUNIT_EXPECT_TRUE(test, cd->tbt_notifier_registered); + KUNIT_EXPECT_EQ(test, atomic_read(&cd->restore_event), 1); + + cd321x_tbt_test_release_port(cd); + KUNIT_EXPECT_FALSE(test, cd->tbt_notifier_registered); +} + static struct kunit_case cd321x_tbt_cases[] = { KUNIT_CASE(cd321x_tbt_no_router_test), KUNIT_CASE(cd321x_tbt_router_bound_test), @@ -376,6 +415,7 @@ static struct kunit_case cd321x_tbt_cases[] = { KUNIT_CASE(cd321x_tbt_router_bound_before_watch_test), KUNIT_CASE(cd321x_tbt_other_router_binds_test), KUNIT_CASE(cd321x_tbt_router_binds_after_removal_test), + KUNIT_CASE(cd321x_tbt_router_rebinds_test), {} }; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 74c820dd0be24..6d249a0919ac5 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -2235,20 +2235,25 @@ cd321x_tbt_switch_get(struct device *dev, struct fwnode_handle *fwnode, EXPORT_SYMBOL_IF_KUNIT(cd321x_tbt_switch_get); /* - * Take a router that bound after the port gave up on it. An attached cable - * raises no new event, so reapply it through the router as after system sleep. + * Take a router that bound after the port gave up on it, or that unbound and + * bound again. An attached cable raises no new event, so reapply it through + * the router as after system sleep. */ static void cd321x_tbt_attach_work(struct work_struct *work) { struct cd321x *cd321x = container_of(work, struct cd321x, tbt_attach_work); struct tps6598x *tps = &cd321x->tps; - struct typec_thunderbolt_switch *sw; + struct typec_thunderbolt_switch *sw, *old; int ret; scoped_guard(mutex, &tps->lock) { - if (cd321x->tbt_switch || cd321x->pm.phase == CD321X_PM_REMOVED) + if (cd321x->pm.phase == CD321X_PM_REMOVED) + return; + /* The router this port uses is still bound. */ + if (typec_thunderbolt_switch_is_live(cd321x->tbt_switch)) return; + old = cd321x->tbt_switch; } /* Another port's router, or this one is not usable yet: keep watching. */ @@ -2256,6 +2261,13 @@ static void cd321x_tbt_attach_work(struct work_struct *work) if (IS_ERR_OR_NULL(sw)) return; + /* The router went away: its switch sends nothing more, stop listening. */ + if (old && cd321x->tbt_notifier_registered) { + typec_thunderbolt_switch_unregister_notifier(old, + &cd321x->tbt_notifier); + cd321x->tbt_notifier_registered = false; + } + ret = typec_thunderbolt_switch_register_notifier(sw, &cd321x->tbt_notifier); if (ret) { @@ -2269,7 +2281,9 @@ static void cd321x_tbt_attach_work(struct work_struct *work) cd321x->tbt_switch = sw; cd321x->tbt_notifier_registered = true; } - dev_info(tps->dev, "USB4/Thunderbolt router bound, reapplying the cable state\n"); + typec_thunderbolt_switch_put(old); + dev_info(tps->dev, "USB4/Thunderbolt router bound%s, reapplying the cable state\n", + old ? " again" : ""); atomic_set(&cd321x->restore_event, 1); mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -2396,8 +2410,13 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_put_switch; cd321x->tbt_notifier_registered = true; - /* Keep the port without the router rather than fail it: see above. */ - if (tbt_late && cd321x_tbt_watch_late(cd321x, fwnode)) + /* + * Keep the port without the router rather than fail it: see above. A + * router that is there can also unbind and bind again, which gives it + * a new switch: watch for that as well. + */ + if ((tbt_late || cd321x->tbt_switch) && + cd321x_tbt_watch_late(cd321x, fwnode)) dev_warn(tps->dev, "cannot watch for the USB4/Thunderbolt router\n"); return 0; diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index ea0d697629611..9d1df67201239 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -253,6 +253,7 @@ struct typec_thunderbolt_switch_desc { struct typec_thunderbolt_switch * fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *fwnode); void typec_thunderbolt_switch_put(struct typec_thunderbolt_switch *sw); +bool typec_thunderbolt_switch_is_live(struct typec_thunderbolt_switch *sw); int typec_thunderbolt_switch_set(struct typec_thunderbolt_switch *sw, const struct typec_thunderbolt_switch_data *data); int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, @@ -302,6 +303,12 @@ typec_thunderbolt_switch_register(struct device *parent, return ERR_PTR(-EOPNOTSUPP); } +static inline bool +typec_thunderbolt_switch_is_live(struct typec_thunderbolt_switch *sw) +{ + return false; +} + static inline int typec_thunderbolt_switch_register_notifier(struct typec_thunderbolt_switch *sw, struct notifier_block *nb) From 44230fbda0585b47bc74dbc0dbd0bcea7585fe0e Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 09:26:22 -0500 Subject: [PATCH 285/392] soc: apple: display-gate: adopt the M4 Max internal display handoff aura hands the M4 Max's internal display over the way it hands over the M4 Pro's: the DCP, the display subsystem and the DCP's piodma carry apple,t6041-handoff, the DCP names the 26.6.2 firmware, and the streams sit on the same DARTs. The M4 Max gate only enabled the external chains, so a handoff there was ignored and the panel stayed on simpledrm. Run the M4 Pro's internal display check on the M4 Max first. With the handoff and a running DCP it enables the DARTs, the mailbox, the display subsystem, the DCP and the panel; without it nothing changes. The external chains still need an enabled display subsystem, now the one this handoff enables, and still only add their DARTs to it. Link: https://github.com/aurora-silicon/linux/issues/1332 Signed-off-by: Diljit Singh --- drivers/soc/apple/display-gate-test.c | 5 ++-- drivers/soc/apple/display-gate.c | 39 +++++++++++++++++--------- drivers/soc/apple/t6041-gate-test.dtso | 5 ++-- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/drivers/soc/apple/display-gate-test.c b/drivers/soc/apple/display-gate-test.c index c767a064e9280..a732f844683b8 100644 --- a/drivers/soc/apple/display-gate-test.c +++ b/drivers/soc/apple/display-gate-test.c @@ -20,8 +20,9 @@ * leaves the internal panel alone. * - T6041 on the J616c (t6041-gate-test.dtso): no PMP; each external chain * is enabled once its processor has firmware memory, one without it or - * already enabled stays as it is, and the display subsystem is never - * written. + * already enabled stays as it is, and the chains never write the display + * subsystem's status. The tree has no dcp alias, so the internal display + * handoff finds nothing to adopt. * Included at the end of display-gate.c. * * The test trees are applied once and stay: the gate gives some properties diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index bdb4b6dac2031..73c56cc29ca21 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -78,11 +78,12 @@ * driver of its own, t6041-display-gate.c, by Ryan Murray and FrantiÅ¡ek * Balázsy. * - * On the M4 Max (J616c, apple,t6041) the PMP never runs: iBoot leaves the - * external chains powered and the boot loader fills each external display - * processor's firmware memory. The gate enables a chain once its processor has - * that memory, before any device exists (see gate_t6041_max_run()), and never - * touches the internal display. + * On the M4 Max (J614c and J616c, apple,t6041) the internal display is + * handed over and enabled as on the M4 Pro. The PMP never runs there: iBoot + * leaves the external chains powered and the boot loader fills each external + * display processor's firmware memory. The gate enables a chain once its + * processor has that memory and the display subsystem is enabled, before any + * device exists (see gate_t6041_max_run()). */ #define pr_fmt(fmt) fmt @@ -336,11 +337,17 @@ static const struct gate_soc gate_t6041 __initconst = { static void __init gate_t6041_max_run(void); -/* T6041 (M4 Max): no PMP; the external chains follow the boot loader's firmware memory. */ +/* + * T6041 (M4 Max): the M4 Pro's internal display handoff; no PMP, so the + * external chains follow the boot loader's firmware memory. + */ static const struct gate_soc gate_t6041_max __initconst = { .machine = "apple,t6041", .prefix = "apple-t6041-display: ", .name = "T6041", + .dcp_compat = "apple,t6041-dcp", + .display_compat = "apple,display-subsystem", + .marker = "apple,t6041-handoff", .run = gate_t6041_max_run, }; @@ -3030,12 +3037,13 @@ static int __init t6041_max_apply(unsigned int i, struct device_node *display) } /* - * Each chain is enabled in one changeset, once its processor has firmware - * memory. A chain that has none, or is not all there and disabled (a boot - * loader that enabled it itself), stays as it is, and the other chain does not - * depend on it. The display subsystem only gains the DART of an enabled chain: - * its status is never written, so the internal display stays as the boot - * loader left it whatever happens here. + * The internal display first, as on the M4 Pro: an accepted handoff enables + * the display subsystem with the DCP. Then each external chain is enabled in + * one changeset, once its processor has firmware memory. A chain that has + * none, or is not all there and disabled (a boot loader that enabled it + * itself), stays as it is, and the other chain does not depend on it. The + * chains need the display subsystem enabled, by the handoff or by the boot + * loader, and only add their DARTs to it: its status is not written here. */ static void __init gate_t6041_max_run(void) { @@ -3046,10 +3054,15 @@ static void __init gate_t6041_max_run(void) if (IS_ENABLED(CONFIG_VIDEO) && video_firmware_drivers_only()) return; + gate_t6041_internal(); staged_prefix = gate_soc->prefix; display = of_find_node_by_path("/display-subsystem"); - if (!display || !of_device_is_available(display)) { + if (!display) { staged_err("no display subsystem, nothing enabled\n"); + return; + } + if (!of_device_is_available(display)) { + staged_info("display subsystem not enabled, external displays left off\n"); of_node_put(display); return; } diff --git a/drivers/soc/apple/t6041-gate-test.dtso b/drivers/soc/apple/t6041-gate-test.dtso index 96c0eaa541ced..d12929a73c0c9 100644 --- a/drivers/soc/apple/t6041-gate-test.dtso +++ b/drivers/soc/apple/t6041-gate-test.dtso @@ -7,8 +7,9 @@ * heap regions of both processors are the boot loader's disabled * placeholders, the text region the one it adds. * - * The display subsystem is the only node that is enabled. It sits at the - * root, as on the J616c, so a tree that has one does not get this applied. + * The display subsystem is the only node that is enabled, as the internal + * display handoff leaves it. It sits at the root, as on the J616c, so a tree + * that has one does not get this applied. */ /dts-v1/; From ffcc24f462200bccd83b0b3ff046ffc6a00c4caa Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 09:26:24 -0500 Subject: [PATCH 286/392] arm64: dts: apple: t6041-j614c-j616c: leave the internal display to the handoff The J614c/J616c tree enabled /display-subsystem itself. aura refuses to hand over a display whose nodes are already enabled ("display is not status "disabled"", linux#1332), so the panel stayed on simpledrm. Keep the display subsystem disabled, and the panel with it, as the J616s tree does. The display gate enables them with the DCP once the boot loader has handed the display over. Without a handoff the display subsystem now stays disabled, and the external chains, which need it, stay off too. aura publishes the external processors' firmware memory only together with the panel handoff, so nothing that worked with aura before is lost. A loader that fills that memory without handing the panel over loses the external displays until it hands the panel over as well. Link: https://github.com/aurora-silicon/linux/issues/1332 Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi b/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi index de5eed04aed63..98337a09a596f 100644 --- a/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi +++ b/arch/arm64/boot/dts/apple/t6041-j614c-j616c.dtsi @@ -448,9 +448,13 @@ status = "okay"; }; +/* + * As on the J616s, the display gate enables the display subsystem, the DCP, + * its DARTs, mailbox and panel once the boot loader has handed the internal + * display over (apple,t6041-handoff); otherwise the boot framebuffer stays. + */ &display { phandle = <&display>; - status = "okay"; }; &dcp { @@ -459,6 +463,7 @@ panel: panel { apple,max-brightness = <500>; + status = "disabled"; }; }; From 833be9cc6af4b54a298014cc97139116c58ac0d5 Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:35:40 +0100 Subject: [PATCH 287/392] drm/asahi: g16: accept the J604's ADT performance layout The G16 backend folds the original ADT's sixteen GPU performance states into the firmware's ten rows with one fixed index map, the J713's: four low states followed by six equal-voltage pairs. The base M4 MacBook Pro 14-inch (J604) has the same sixteen states in another shape: five low states, five equal-voltage pairs (5/6 to 13/14) and a single top state. The J713 map pairs states of different voltage there, so admission fails and the GPU never starts: aurora-g16 302600000.gpu: M4 original ADT performance layout rejected: Shape aurora-g16 302600000.gpu: G16 start failed: EINVAL; session retained until reboot Keep a short list of qualified layouts and take the first one whose rows all pair states of equal frequency, SRAM frequency and voltages. The J604 layout drops the fifth low state (4), since eleven distinct rows do not fit in ten; the four lowest states, every pair and the top state keep their own ADT values. The J713 keeps its current map and is tried first, and each layout's pairing check rejects the other shape, so nothing changes for it. select() keeps its signature for the host tests; select_named() also returns the layout's name, which the boot log prints. test-perf.py gains a J604-shaped case (its selected rows, the J713 fixture still choosing the J713 layout, and an unequal pair matching neither). The J604 layout was read from the machine's own ADT (16 states, 1578 MHz @ 1045 mV at the top). Tested on a J604 (25G83 firmware) on integration/aurora 8f484810fc: the J604 layout is chosen, the firmware reaches READY, the DRM device registers, and gpubench's core-a chunk (submit.empty, compute.light, render.light; 23 results) passes on Aurora's Mesa with no GPU faults. Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- drivers/gpu/drm/asahi/g16/board.rs | 7 ++- drivers/gpu/drm/asahi/g16/board/data.rs | 60 +++++++++++++++++-- .../drivers/gpu/asahi-m4/test-perf.py | 24 +++++++- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g16/board.rs b/drivers/gpu/drm/asahi/g16/board.rs index 0ee62758e8ad0..9d005164e6de8 100644 --- a/drivers/gpu/drm/asahi/g16/board.rs +++ b/drivers/gpu/drm/asahi/g16/board.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! J713 board admission for the qualified M4 firmware graph. +//! M4 board admission for the qualified firmware graph. //! //! M4 uses ten firmware rows selected from sixteen source states, including off, and sixteen //! voltage lanes. The source values belong to this machine's preserved original ADT. @@ -14,7 +14,7 @@ use kernel::{device, prelude::*}; /// Admit the qualified table layout while preserving every machine-specific source value. /// Differently shaped tables and inconsistent voltage pairs fail before publication. pub(crate) fn qualify(dev: &device::Device, adt: &GpuAdt) -> Result { - let performance = data::select(&adt.performance).map_err(|error| { + let (performance, layout) = data::select_named(&adt.performance).map_err(|error| { dev_err!( dev, "M4 original ADT performance layout rejected: {:?}\n", @@ -24,7 +24,8 @@ pub(crate) fn qualify(dev: &device::Device, adt: &GpuAdt) -> Result })?; dev_info!( dev, - "M4 ADT DVFS: 16 source states -> 10 firmware rows, top {} MHz\n", + "M4 ADT DVFS: 16 source states -> 10 firmware rows ({} layout), top {} MHz\n", + layout, performance.frequency[PERF_STATES - 1] ); for i in 0..PERF_STATES { diff --git a/drivers/gpu/drm/asahi/g16/board/data.rs b/drivers/gpu/drm/asahi/g16/board/data.rs index a67dfa3a77a1e..6d18386727a7f 100644 --- a/drivers/gpu/drm/asahi/g16/board/data.rs +++ b/drivers/gpu/drm/asahi/g16/board/data.rs @@ -1,13 +1,39 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Qualified J713/25G83 selection of the original ADT's sixteen performance states. +//! Qualified selections of the original ADT's sixteen performance states for 25G83. //! -//! Four common low states are followed by six equal-voltage pairs. These are indices into -//! the machine's original ADT, not frequencies or voltages copied from a reference machine. +//! The firmware takes ten rows, each a main and an auxiliary frequency at one voltage. Every +//! qualified layout maps the sixteen ADT states onto those rows by index; the values are always +//! this machine's own, never frequencies or voltages copied from a reference machine. pub(crate) const SOURCE_STATES: usize = 16; -pub(crate) const MAIN: [usize; 10] = [0, 1, 2, 3, 5, 7, 9, 11, 13, 15]; -pub(crate) const AUXILIARY: [usize; 10] = [0, 1, 2, 3, 4, 6, 8, 10, 12, 14]; + +/// One qualified mapping of the ADT states onto the firmware rows. +pub(crate) struct Layout { + pub(crate) name: &'static str, + pub(crate) main: [usize; PERF_STATES], + pub(crate) auxiliary: [usize; PERF_STATES], +} + +/// Tried in order; a layout matches only when each row's two states share frequency, SRAM +/// frequency and both voltages, so the shapes below cannot be mistaken for each other. +pub(crate) const LAYOUTS: [Layout; 2] = [ + // J713: four common low states followed by six equal-voltage pairs. + Layout { + name: "J713", + main: [0, 1, 2, 3, 5, 7, 9, 11, 13, 15], + auxiliary: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14], + }, + // J604 (MacBook Pro 14-inch, M4): five low states, five equal-voltage pairs (5/6 to + // 13/14) and a single top state 15. Eleven distinct rows do not fit in ten, so the fifth + // low state (4) is left out; the four lowest states, every pair and the top state keep + // their own ADT values. + Layout { + name: "J604", + main: [0, 1, 2, 3, 6, 8, 10, 12, 14, 15], + auxiliary: [0, 1, 2, 3, 5, 7, 9, 11, 13, 15], + }, +]; use super::super::fw::initdata::{Performance, PERF_STATES}; @@ -15,10 +41,32 @@ use super::super::fw::initdata::{Performance, PERF_STATES}; pub(crate) fn select( source: &super::super::adt::perf::Source, ) -> core::result::Result { + select_named(source).map(|(performance, _)| performance) +} + +/// As [`select`], with the name of the layout that matched. +pub(crate) fn select_named( + source: &super::super::adt::perf::Source, +) -> core::result::Result<(Performance, &'static str), super::super::adt::perf::Error> { use super::super::adt::perf::Error; if source.count != SOURCE_STATES { return Err(Error::Shape); } + for layout in LAYOUTS.iter() { + match select_layout(source, layout) { + Err(Error::Shape) => continue, + Err(error) => return Err(error), + Ok(performance) => return Ok((performance, layout.name)), + } + } + Err(Error::Shape) +} + +fn select_layout( + source: &super::super::adt::perf::Source, + layout: &Layout, +) -> core::result::Result { + use super::super::adt::perf::Error; let mut performance = Performance { frequency: [0; PERF_STATES], auxiliary: [0; PERF_STATES], @@ -26,7 +74,7 @@ pub(crate) fn select( sram: [0; PERF_STATES], }; for i in 0..PERF_STATES { - let (main, aux) = (MAIN[i], AUXILIARY[i]); + let (main, aux) = (layout.main[i], layout.auxiliary[i]); if source.frequency_mhz[main] != source.sram_frequency_mhz[main] || source.frequency_mhz[aux] != source.sram_frequency_mhz[aux] || source.core_mv[main] != source.core_mv[aux] diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-perf.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-perf.py index 32f290d9fd423..36574512d4298 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-perf.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-perf.py @@ -135,6 +135,28 @@ def node(props, children=()): let mut s=source();s.frequency_mhz[7]=s.frequency_mhz[5];s.sram_frequency_mhz[7]=s.frequency_mhz[7];assert!(data::select(&s).is_err()); let mut s=source();s.core_mv[14]=600;s.core_mv[15]=600;assert!(data::select(&s).is_err()); } +#[test] fn j604_shape_selects_its_own_layout_and_the_j713_keeps_its_own() { + assert_eq!(data::select_named(&source()).unwrap().1,"J713"); + // J604 shape: five low states, equal-voltage pairs 5/6 to 13/14, a single top state 15. + // Rows 0-3 and 5-15 are the J604's selected values; state 4 (left out) is synthetic. + let mhz=[0,338,618,796,928,952,1056,1053,1170,1152,1278,1204,1338,1326,1470,1578]; + let mv=[125,625,640,680,725,780,780,825,825,865,865,895,895,970,970,1045]; + let mut s=source(); + for i in 0..16 { + s.frequency_mhz[i]=mhz[i];s.sram_frequency_mhz[i]=mhz[i]; + s.core_mv[i]=mv[i];s.sram_mv[i]=mv[i].max(780); + } + let (p,name)=data::select_named(&s).unwrap(); + assert_eq!(name,"J604"); + assert_eq!(p.frequency,[0,338,618,796,1056,1170,1278,1338,1470,1578]); + assert_eq!(p.auxiliary,[0,338,618,796,952,1053,1152,1204,1326,1578]); + assert_eq!(p.voltage,[125,625,640,680,780,825,865,895,970,1045]); + assert_eq!(p.sram,[780,780,780,780,780,825,865,895,970,1045]); + assert!(encode(p).is_ok()); + // A J604 shape with one unequal pair matches neither layout. + let mut bad=s;bad.core_mv[9]+=1;bad.sram_mv[9]+=1; + assert_eq!(data::select(&bad),Err(perf::Error::Shape)); +} #[test] fn invalid_encoder_input_is_refused_without_mutating_output() { for variant in 0..5 { let mut p=tables(); match variant {0=>p.frequency[1]=0,1=>p.auxiliary[5]=p.frequency[5]+1,2=>p.voltage[9]=1101,3=>p.sram[9]=100,_=>p.frequency[0]=1}; @@ -155,7 +177,7 @@ def node(props, children=()): print(run.stdout, end='') print(run.stderr, end='') run.check_returncode() -report = {'passed': True, 'test_groups': 7, 'hardware_validated': False, +report = {'passed': True, 'test_groups': 8, 'hardware_validated': False, 'fixture_sha256': sha(fixture), 'original_adt_sha256': metadata['provenance']['source_sha256'], 'production_sha256': {str(p.relative_to(root)): sha(p) for p in [backend / 'adt.rs', backend / 'adt/perf.rs', backend / 'board.rs', backend / 'board/data.rs', backend / 'initdata.rs', backend / 'fw/initdata.rs']}} From 579bcc38d686758d9f18470bd404a27f82786df1 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 09:45:40 -0500 Subject: [PATCH 288/392] drm/asahi: run the 25G83 build on the 40-core M4 Max The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware on the M4 Max as well. integration/aurora admits only 25F84 there, so a 40-core J614c booting through a 26.6.2 stub has no GPU (linux#1332). A firmware build can now carry the SoC configuration it runs with. Once the build is identified, that configuration replaces the probed one. This is the mechanism linux#1050 proposes, so its 32-core profile can build on it. T6041 gains a 25G83 build. Its configuration is the 40-core 25F84 profile with two changes: - the 25G83 variant, with the four-cluster tiler values (TA mode 1) that 25F84 uses on this machine and #1050 uses for 25G83; - the 25G83 register map. It has the same 25 windows (slot 3 holds the 40-core part's eight memory controllers), laid out the way 25G83 lays them out on the M4 Pro: from +0x200000, each window followed by a 16K guard page, with no reserved slots. This rule reproduces both known 25G83 maps, the M4 Pro's (four controllers) and #1050's 32-core M4 Max map (six). A compile-time check holds the M4 Pro's map and the new one to the rule, and the new one to 25F84's windows. The rest comes from the 25F84 profile: 40 cores in four clusters, the operand-state and deflake layouts, the 160-block compute pool, the hardware data and power words, and region C. The core count check stays, so a 32-core part is refused with this configuration. The build admits one loaded image: the only T6041 25G83 image measured so far (sha256 aa113ed6...effc5, measured on #1050's 32-core J616C). iBoot fills part of the loaded TEXT for each Mac (three J616s loaded three different T6040 images), so a 40-core Mac most likely loads a different one. If so, its "unsupported GPU firmware build" line gives the sha256 to add. KUnit tests cover the identities of both T6041 builds, the 25G83 configuration, and the render policy of both builds. The host model of the build selector now returns the build's configuration, and the T6041 initdata host test also builds the 25G83 objects. Not run on hardware. Link: https://github.com/aurora-silicon/linux/issues/1332 Link: https://github.com/aurora-silicon/linux/pull/1050 Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/fwid.rs | 32 ++++ drivers/gpu/drm/asahi/g17/fw/render.rs | 38 ++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 3 + drivers/gpu/drm/asahi/g17/mod.rs | 15 +- drivers/gpu/drm/asahi/hw/t6040.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 158 +++++++++++++++++--- drivers/gpu/drm/asahi/hw/t604x.rs | 18 +++ tools/asahi/g17-firmware-selection-tests.py | 24 ++- tools/asahi/g17-t6041-initdata-tests.py | 18 ++- 9 files changed, 264 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 34c65a0d8a097..514f09539dc73 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -247,6 +247,38 @@ mod tests { assert!(measured(build.images, 0x138000, [0x59839, 0x5991c], [0; 32]).is_none()); } + #[test] + fn t6041_builds_admit_only_their_own_images() { + let crate::g17::fw::variant::Firmware::Identified(builds) = + crate::hw::t6041::CONFIG.firmware + else { + panic!("T6041 must identify its loaded firmware"); + }; + assert_eq!(builds.len(), 2); + for (i, build) in builds.iter().enumerate() { + let other = &builds[1 - i]; + for image in build.images { + let (entropy, digest) = (image.entropy, image.text_sha256); + let found = measured(build.images, image.data_size, entropy, digest); + assert!(core::ptr::eq(found.unwrap(), image)); + assert!(measured(other.images, image.data_size, entropy, digest).is_none()); + assert!(measured(build.images, 0x13c000, entropy, digest).is_none()); + } + } + // Only the 25G83 build runs its own configuration: the 25G83 register map, 40 cores. + assert!(builds[0].config.is_none()); + let g83 = builds[1].config.unwrap(); + let map = &crate::hw::t6041::REGISTER_WINDOWS_25G83; + assert_eq!(g83.windows.len(), map.len()); + for (window, expected) in g83.windows.iter().zip(map.iter()) { + assert_eq!((window.slot, window.va), (expected.slot, expected.va)); + } + assert!(g83.reserved_slots.is_empty()); + let cores = crate::hw::t6041::CONFIG.core_layout.cores; + assert_eq!(g83.core_layout.cores, cores); + assert_eq!(cores, Some((4, 40))); + } + #[cfg(CONFIG_DRM_ASAHI_G16)] #[test] fn j713_accepts_both_loaded_images() { diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 8fcc25500ab2d..f63821c904fd2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1629,14 +1629,38 @@ mod ta_mode_tests { } } + /// The variants of the builds T6041 admits: 25F84, then 25G83. + fn t6041_variants() -> [&'static Variant; 2] { + let Firmware::Identified(builds) = crate::hw::t6041::CONFIG.firmware else { + panic!("T6041 must identify its loaded firmware"); + }; + assert_eq!(builds.len(), 2); + [builds[0].variant, builds[1].variant] + } + + /// The TA mode of every admitted T6041 build. + fn t6041_ta_mode_1() -> bool { + let [f84, g83] = t6041_variants(); + assert_eq!(f84.ta_mode_1, g83.ta_mode_1); + f84.ta_mode_1 + } + #[test] - fn policy_is_local_to_the_admitted_t6041_build() { - let max = selected(&crate::hw::t6041::CONFIG.firmware); + fn policy_is_local_to_the_admitted_t6041_builds() { + let [f84, g83] = t6041_variants(); let pro = selected(&crate::hw::t6040::CONFIG.firmware); - assert_eq!(max.compute, crate::g17::fw::compute::ComputeProgram::V25F84); - assert!(max.split_firmware_area); - assert_eq!(max.programs, RegisterPrograms::V25G83); - assert!(max.ta_mode_1); + assert_eq!(f84.compute, crate::g17::fw::compute::ComputeProgram::V25F84); + assert!(f84.split_firmware_area); + // 25G83 keeps the M4 Pro's queue and compute rules on the M4 Max. + assert_eq!(g83.compute, pro.compute); + assert_eq!(g83.split_firmware_area, pro.split_firmware_area); + assert_eq!(g83.opening_record, pro.opening_record); + assert_eq!(g83.qid_floor, pro.qid_floor); + assert_eq!(g83.registration, pro.registration); + for max in [f84, g83] { + assert_eq!(max.programs, RegisterPrograms::V25G83); + assert!(max.ta_mode_1); + } assert!(!pro.ta_mode_1); assert!(!V25F84.ta_mode_1); assert!(!V25G83.ta_mode_1); @@ -1704,7 +1728,7 @@ mod ta_mode_tests { for scs in [false, true] { let mut args = args(&pass, &resources, scs); let (before, count) = ta_registers_25g83(&args, &geometry, &words).unwrap(); - args.ta_mode_1 = selected(&crate::hw::t6041::CONFIG.firmware).ta_mode_1; + args.ta_mode_1 = t6041_ta_mode_1(); let (after, new_count) = ta_registers_25g83(&args, &geometry, &words).unwrap(); assert_eq!(count, 83 + usize::from(scs)); assert_eq!(new_count, count); diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index c5287e66a641d..222473a8b8c36 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -89,6 +89,9 @@ pub(crate) struct Build { pub(crate) images: &'static [crate::fwid::FirmwareBuild], /// Layouts and rules of the build. pub(crate) variant: &'static Variant, + /// The SoC configuration this build runs with, which replaces the probed one once the + /// build is identified; `None` keeps the probed configuration. + pub(crate) config: Option<&'static crate::g17::Config>, } /// How a SoC's firmware build is determined. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ffb0c20d0267f..df5a31f621a3c 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1173,7 +1173,9 @@ impl Gpu { } else { None }; - let variant = Self::firmware_variant(dev, &cfg.firmware)?; + let (variant, build_cfg) = Self::firmware_variant(dev, &cfg.firmware)?; + // A build with its own configuration of the SoC runs with that one. + let cfg = build_cfg.unwrap_or(cfg); let id = regs.get_g17_id()?; if let Some(calibration) = calibration { calibration.check_cores(&id.core_masks)?; @@ -1324,14 +1326,15 @@ impl Gpu { Ok(gpu) } - /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it - /// does not know, before anything is written to the GPU. + /// Returns the variant of the loaded firmware build and the configuration it runs with, if + /// it has its own. An identified SoC refuses a build it does not know, before anything is + /// written to the GPU. fn firmware_variant( dev: &AsahiDrmDevice, firmware: &fw::variant::Firmware, - ) -> Result<&'static fw::Variant> { + ) -> Result<(&'static fw::Variant, Option<&'static Config>)> { let builds = match firmware { - fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Fixed(variant) => return Ok((variant, None)), fw::variant::Firmware::Identified(builds) => builds, }; let version = Self::os_firmware_version(dev); @@ -1346,7 +1349,7 @@ impl Gpu { for build in Iterator::chain(preferred, others) { if Self::identify_image(dev, build).is_ok() { dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok(build.variant); + return Ok((build.variant, build.config)); } } dev_err!( diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index d531e20c7e9f9..5f6b9944a811e 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -100,6 +100,7 @@ pub(crate) const CONFIG: Config = Config { }, ], variant: &fw::variant::V25G83, + config: None, }]), // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 8be466f7842ab..956aa0cc7728b 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -9,7 +9,9 @@ use crate::hw::{GpuGen, GpuVariant}; use super::t604x::{instances, window}; use crate::mmu::UatQuirks; -pub(crate) const CONFIG: Config = Config { +/// What the configurations of both firmware builds share: the 40-core M4 Max measured with +/// 25F84. Only the register map is the build's own. +const BASE: Config = Config { chip_id: CHIP_ID, identity: Identity { family: 0xa, @@ -46,26 +48,7 @@ pub(crate) const CONFIG: Config = Config { fabric_bridge: false, recovery_slot_keys: None, gpu_generation: GpuGen::G16, - firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { - os_version: [26, 5, 2], - name: "macOS 26.5.2 (25F84, iBoot mBoot-18000.121.3)", - images: &[crate::fwid::FirmwareBuild { - name: "Apple GFX 25F84 for T6041 (normalized loaded image)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x5983a, 0x5991d], - text_sha256: [ - 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, - 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, - 0xa3, 0xc1, 0x83, 0xe8, - ], - }], - // macOS writes these tiler segment values on this machine, not the two-cluster ones. - variant: &fw::variant::Variant { - ta_mode_1: true, - ..fw::variant::V25F84 - }, - }]), + firmware: fw::variant::Firmware::Identified(&[]), core_layout: CoreLayout { cores: Some((4, 40)), alternate_core_masks: None, @@ -75,6 +58,76 @@ pub(crate) const CONFIG: Config = Config { }, }; +/// The 40-core M4 Max. 25F84 runs this configuration; 25G83 runs [`CONFIG_25G83`]. +pub(crate) const CONFIG: Config = Config { + firmware: fw::variant::Firmware::Identified(&BUILDS), + ..BASE +}; + +const BUILDS: [fw::variant::Build; 2] = [ + BUILD_25F84, + fw::variant::Build { + config: Some(&CONFIG_25G83), + ..BUILD_25G83 + }, +]; + +const BUILD_25F84: fw::variant::Build = fw::variant::Build { + os_version: [26, 5, 2], + name: "macOS 26.5.2 (25F84, iBoot mBoot-18000.121.3)", + images: &[crate::fwid::FirmwareBuild { + name: "Apple GFX 25F84 for T6041 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x5983a, 0x5991d], + text_sha256: [ + 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, + 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, + 0xa3, 0xc1, 0x83, 0xe8, + ], + }], + // macOS writes these tiler segment values on this machine, not the two-cluster ones. + variant: &fw::variant::Variant { + ta_mode_1: true, + ..fw::variant::V25F84 + }, + config: None, +}; + +/// The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 build the M4 Pro +/// runs. iBoot fills part of the loaded TEXT for each Mac, so each admitted image is a +/// measurement. The one listed is the only T6041 image measured so far, on a 32-core J616C +/// (linux#1050); there the core count check of [`CONFIG_25G83`] refuses the GPU. +const BUILD_25G83: fw::variant::Build = fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + images: &[crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6041 (normalized loaded image, 32-core J616C)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0xaa, 0x11, 0x3e, 0xd6, 0x20, 0x79, 0xbc, 0xfd, 0xee, 0xf0, 0xf8, 0x6c, 0x2a, 0x8d, + 0x7e, 0xae, 0x9f, 0x90, 0x7b, 0x03, 0xbe, 0x28, 0xcc, 0xf2, 0x98, 0xc9, 0x43, 0x95, + 0x1d, 0x0e, 0xff, 0xc5, + ], + }], + // Four clusters: the tiler segment values of 25F84 on this machine. + variant: &fw::variant::Variant { + ta_mode_1: true, + ..fw::variant::V25G83 + }, + config: None, +}; + +/// The 40-core M4 Max on 25G83: the 25F84 configuration with the 25G83 register map. +const CONFIG_25G83: Config = Config { + windows: ®ISTER_WINDOWS_25G83, + reserved_slots: &[], + firmware: fw::variant::Firmware::Identified(&[BUILD_25G83]), + ..BASE +}; + pub(crate) const CHIP_ID: u32 = 0x6041; pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ @@ -113,6 +166,69 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(43, 0x2_88e6_0000, 0x2f_8000, 0x58, 0xe6_0000, 2), ]; +/// The register map of the 25G83 build: the windows of [`REGISTER_WINDOWS`] (slot 3 holds the +/// eight memory controllers) laid out as 25G83 lays them out on the M4 Pro +/// ([`super::t6040::REGISTER_WINDOWS`]), from `+0x20_0000`, each window followed by a 16K +/// guard page, with no reserved slots. The windows after slot 3 lie 0x60000 further than the M4 +/// Pro's, for its four more controllers. +pub(crate) const REGISTER_WINDOWS_25G83: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances( + window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), + 8, + 0x200_0000, + ), + window(4, 0x5_0244_c000, 0x2c_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x2d_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x2d_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x2e_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x2e_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x2f_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x30_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x31_0000, 0x2_1400, 0, 2), + instances( + window(20, 0x2_8b00_0000, 0x33_8000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), + window(22, 0x2_8900_0000, 0x43_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x44_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x45_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x46_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x46_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x47_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x49_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x49_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x4a_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x4a_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x4b_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x4c_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x4c_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x4d_4000, 0x58, 0xe6_0000, 2), +]; + +// The 25G83 map holds the 25F84 windows, laid out as 25G83 lays out the M4 Pro's. +const _: () = { + assert!(super::t604x::laid_out_25g83( + &super::t6040::REGISTER_WINDOWS + )); + assert!(super::t604x::laid_out_25g83(®ISTER_WINDOWS_25G83)); + let mut index = 0; + while index < REGISTER_WINDOWS.len() { + let (f84, g83) = (®ISTER_WINDOWS[index], ®ISTER_WINDOWS_25G83[index]); + assert!( + f84.slot == g83.slot + && f84.phys == g83.phys + && f84.size == g83.size + && f84.unk_18 == g83.unk_18 + && f84.unk_20 == g83.unk_20 + && f84.instances == g83.instances + && f84.phys_stride == g83.phys_stride + ); + index += 1; + } +}; + const RESERVED_SLOTS: [usize; 14] = [2, 5, 6, 7, 8, 27, 30, 38, 42, 46, 47, 48, 49, 52]; pub(crate) const PERF: PerfConfig = PerfConfig { diff --git a/drivers/gpu/drm/asahi/hw/t604x.rs b/drivers/gpu/drm/asahi/hw/t604x.rs index 961cef76cf4aa..6021419eeb1dd 100644 --- a/drivers/gpu/drm/asahi/hw/t604x.rs +++ b/drivers/gpu/drm/asahi/hw/t604x.rs @@ -31,3 +31,21 @@ pub(super) const fn instances(window: RegisterWindow, instances: u32, phys_strid ..window } } + +/// Whether `windows` are laid out as the 25G83 build lays out its register map: in order from +/// `+0x20_0000`, the pages of each window's instances followed by a 16K guard page. +pub(super) const fn laid_out_25g83(windows: &[RegisterWindow]) -> bool { + let mut next = MMIO_APERTURE + 0x20_0000; + let mut index = 0; + while index < windows.len() { + let window = &windows[index]; + let page_offset = window.phys & 0x3fff; + if window.va != next + page_offset { + return false; + } + let pages = (page_offset + window.size as u64 + 0x3fff) & !0x3fff; + next += pages * window.instances as u64 + 0x4000; + index += 1; + } + true +} diff --git a/tools/asahi/g17-firmware-selection-tests.py b/tools/asahi/g17-firmware-selection-tests.py index 771c9c19c6e1e..2a3cae91fdb08 100644 --- a/tools/asahi/g17-firmware-selection-tests.py +++ b/tools/asahi/g17-firmware-selection-tests.py @@ -34,6 +34,7 @@ def method(source, signature): image: u32, checked: std::cell::RefCell>, } +struct Config(u32); mod fw { pub struct Variant(pub u32); pub mod variant { @@ -43,6 +44,7 @@ def method(source, signature): pub name: &'static str, pub image: u32, pub variant: &'static Variant, + pub config: Option<&'static crate::Config>, } pub enum Firmware { Fixed(&'static Variant), Identified(&'static [Build]) } } @@ -60,9 +62,11 @@ def method(source, signature): } static FIRST: fw::Variant = fw::Variant(1); static SECOND: fw::Variant = fw::Variant(2); +static OWN: Config = Config(7); static BUILDS: [fw::variant::Build; 2] = [ - fw::variant::Build { os_version: [26,6,2], name: "first", image: 1, variant: &FIRST }, - fw::variant::Build { os_version: [27,0,0], name: "second", image: 2, variant: &SECOND }, + fw::variant::Build { os_version: [26,6,2], name: "first", image: 1, variant: &FIRST, config: None }, + fw::variant::Build { os_version: [27,0,0], name: "second", image: 2, variant: &SECOND, + config: Some(&OWN) }, ]; fn device(version: Option<[u32; 3]>, image: u32) -> AsahiDrmDevice { AsahiDrmDevice { version, image, checked: std::cell::RefCell::new(Vec::new()) } @@ -71,18 +75,26 @@ def method(source, signature): use fw::variant::Firmware; for version in [None, Some([26,6,2]), Some([26,7,0]), Some([99,0,0])] { let dev = device(version, 1); - assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap().0, 1); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap(); + assert_eq!(variant.0, 1); + assert!(config.is_none(), "a build without its own configuration keeps the probed one"); } let dev = device(Some([27,0,0]), 1); - assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap().0, 1); + assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap().0.0, 1); assert_eq!(*dev.checked.borrow(), [2,1], "the OS version only orders the candidates"); + let dev = device(None, 2); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap(); + assert_eq!((variant.0, config.map(|c| c.0)), (2, Some(7)), "the identified build's configuration"); let dev = device(Some([26,6,2]), 9); assert!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).is_err()); assert_eq!(*dev.checked.borrow(), [1,2]); let dev = device(None, 9); - assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Fixed(&SECOND)).unwrap().0, 2); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Fixed(&SECOND)).unwrap(); + assert_eq!(variant.0, 2); + assert!(config.is_none()); assert!(dev.checked.borrow().is_empty(), "fixed Neo admission is unchanged"); - println!("PASS G17 selection: exact image required, OS hint optional, fixed Neo unchanged"); + println!("PASS G17 selection: exact image required, OS hint optional, build configuration \ + follows the image, fixed Neo unchanged"); } ''' diff --git a/tools/asahi/g17-t6041-initdata-tests.py b/tools/asahi/g17-t6041-initdata-tests.py index 05ddc6b145d9d..a48174df48547 100644 --- a/tools/asahi/g17-t6041-initdata-tests.py +++ b/tools/asahi/g17-t6041-initdata-tests.py @@ -102,6 +102,7 @@ def item(source, start): ] T6041_ITEMS = SOC_ITEMS + [ + "pub(crate) const REGISTER_WINDOWS_25G83", "const RESERVED_SLOTS", "const MAX_POWER_MW", "const CHIP: ", @@ -190,9 +191,20 @@ def aux(name, rows): &t6041::RESERVED_SLOTS, &t6041::PERF, &T6041_OPPS, &T6041_AUX, &t6041::CHIP, &t6041::REGION_C, 40); println!("T6041 initialization objects built"); - std::fs::write(format!("{}/t6041-hwdata.bin", out), hw).unwrap(); - std::fs::write(format!("{}/t6041-power-config.bin", out), power).unwrap(); - std::fs::write(format!("{}/t6041-region-c.bin", out), region).unwrap(); + std::fs::write(format!("{}/t6041-hwdata.bin", out), &hw).unwrap(); + std::fs::write(format!("{}/t6041-power-config.bin", out), &power).unwrap(); + std::fs::write(format!("{}/t6041-region-c.bin", out), ®ion).unwrap(); + // 25G83: the same configuration with its own register map. + let (hw_g83, power_g83, region_g83) = build(t6041::CHIP_ID, &t6041::REGISTER_WINDOWS_25G83, + &[], &t6041::PERF, &T6041_OPPS, &T6041_AUX, &t6041::CHIP, &t6041::REGION_C, 40); + println!("T6041 25G83 initialization objects built"); + assert_eq!(power_g83, power, "the register map changes only the hardware data"); + assert_eq!(region_g83, region, "the register map changes only the hardware data"); + let map = 0x640..0x640 + REGISTER_MAP_SLOTS * core::mem::size_of::(); + let outside = |a: &[u8], b: &[u8]| a.iter().zip(b).enumerate() + .filter(|(i, (x, y))| x != y && !map.contains(i)).count(); + assert_eq!(outside(&hw_g83, &hw), 0, "the 25G83 hardware data differs only in its register map"); + std::fs::write(format!("{}/t6041-25g83-hwdata.bin", out), &hw_g83).unwrap(); } """ From 515ef4e38ac6d64408f8e1dc0816d6715ef7724f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:46:54 +1100 Subject: [PATCH 289/392] wifi: mt7932: add experimental AWDL interface Add the native second-BSS AWDL command records, peer and WTBL ownership, normal-ring RX routing, event dispatch, data framing, and hardware-free record/parser tests. Keep the feature disabled by default. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/wireless/mediatek/mt7932/Kconfig | 19 + drivers/net/wireless/mediatek/mt7932/Makefile | 2 + .../mediatek/mt7932/awdl-kunit.config | 13 + .../net/wireless/mediatek/mt7932/awdl-test.c | 216 ++++ drivers/net/wireless/mediatek/mt7932/awdl.c | 1026 +++++++++++++++++ drivers/net/wireless/mediatek/mt7932/awdl.h | 120 ++ drivers/net/wireless/mediatek/mt7932/data.c | 6 + .../net/wireless/mediatek/mt7932/frontend.c | 15 +- .../net/wireless/mediatek/mt7932/lifecycle.c | 6 + drivers/net/wireless/mediatek/mt7932/link.c | 36 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 5 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 5 +- drivers/net/wireless/mediatek/mt7932/packet.h | 3 +- .../net/wireless/mediatek/mt7932/startup.c | 9 + 14 files changed, 1467 insertions(+), 14 deletions(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/awdl-kunit.config create mode 100644 drivers/net/wireless/mediatek/mt7932/awdl-test.c create mode 100644 drivers/net/wireless/mediatek/mt7932/awdl.c create mode 100644 drivers/net/wireless/mediatek/mt7932/awdl.h diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 49ff2296003c1..50b3fcab2b6ca 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -27,6 +27,25 @@ config MT7932_FULLMAC unloaded. A function that fails to initialize and cannot be reset stays bound and keeps its DMA memory. +config MT7932_AWDL + bool "Experimental clean-room MT7932 AWDL interface" + depends on MT7932_FULLMAC + default n + help + Build the experimental second-BSS Apple Wireless Direct Link path. + It exposes an awdl0 NAN-style Ethernet interface, programs the native + firmware AWDL command records and routes BSS-1 RX records. The option + is deliberately off by default while firmware policy values and target + validation remain incomplete. + +config MT7932_AWDL_KUNIT_TEST + bool "KUnit tests for MT7932 AWDL records and RX parsing" if !KUNIT_ALL_TESTS + depends on KUNIT=y && MT7932_AWDL + default KUNIT_ALL_TESTS + help + Test every clean-room command-record builder, AWDL data framing and + synthetic CONNAC2/event receive records without radio hardware. + config MT7932_RF_KUNIT_TEST bool "KUnit tests for MT7932 terminal RF failure" if !KUNIT_ALL_TESTS depends on KUNIT=y && MT7932_FULLMAC diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index cf7b169dab786..4c60235a399f2 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o +mt7932-fullmac-$(CONFIG_MT7932_AWDL) += awdl.o +mt7932-fullmac-$(CONFIG_MT7932_AWDL_KUNIT_TEST) += awdl-test.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config new file mode 100644 index 0000000000000..b2c173277d03a --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config @@ -0,0 +1,13 @@ +CONFIG_KUNIT=y +CONFIG_PCI=y +CONFIG_OF=y +CONFIG_ARCH_APPLE=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_IOMMU_DMA=y +CONFIG_CFG80211=y +CONFIG_PCIE_APPLE=y +CONFIG_PCIEASPM=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_MT7932_FULLMAC=y +CONFIG_MT7932_AWDL=y +CONFIG_MT7932_AWDL_KUNIT_TEST=y diff --git a/drivers/net/wireless/mediatek/mt7932/awdl-test.c b/drivers/net/wireless/mediatek/mt7932/awdl-test.c new file mode 100644 index 0000000000000..9efe661abaa24 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/awdl-test.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "mt7932.h" + +static const u8 test_mac[ETH_ALEN] = { 0x02, 1, 2, 3, 4, 5 }; + +static void mt7932_awdl_queue_test(struct kunit *test) +{ + u8 body[32]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_queue(body, 0, false, true), 32); + KUNIT_EXPECT_MEMEQ(test, body, ((u8[]){ 2, 0, 1, 1 }), 4); + KUNIT_EXPECT_EQ(test, body[14], 1); + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_queue(body, 0, false, false), 32); + KUNIT_EXPECT_MEMEQ(test, body, ((u8[]){ 0, 0, 1, 1 }), 4); + KUNIT_EXPECT_MEMEQ(test, body + 4, ((u8[28]) {}), 28); + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_queue(body, 7, true, true), 32); + KUNIT_EXPECT_MEMEQ(test, body, ((u8[]){ 15, 0, 5, 0 }), 4); + KUNIT_EXPECT_EQ(test, body[14], 1); + KUNIT_EXPECT_EQ(test, body[15], 7); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 16), 15); + KUNIT_EXPECT_MEMEQ(test, body + 20, + ((u8[]){ 1, 0, 0, 1, 2, 2, 3, 3 }), 8); + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_queue(body, 7, true, false), 32); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 16), 2); +} + +static void mt7932_awdl_active_test(struct kunit *test) +{ + u8 body[12]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_bss_active(body, test_mac, 9, true), 12); + KUNIT_EXPECT_EQ(test, body[0], 1); + KUNIT_EXPECT_EQ(test, body[1], 1); + KUNIT_EXPECT_EQ(test, body[2], 5); + KUNIT_EXPECT_EQ(test, body[3], 0); + KUNIT_EXPECT_MEMEQ(test, body + 4, test_mac, ETH_ALEN); + KUNIT_EXPECT_EQ(test, body[10], 9); + KUNIT_EXPECT_EQ(test, body[11], 1); + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_bss_active(body, test_mac, 9, false), 12); + KUNIT_EXPECT_EQ(test, body[1], 0); + KUNIT_EXPECT_MEMEQ(test, body + 4, ((u8[6]) {}), ETH_ALEN); +} + +static void mt7932_awdl_bss_test(struct kunit *test) +{ + u8 body[MT7932_AWDL_BSS_RECORD_LEN]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_bss(body, 9, 1, 0x7a, 2), 116); + KUNIT_EXPECT_MEMEQ(test, body, ((u8[]){ 1, 1, 7, 0 }), 4); + KUNIT_EXPECT_EQ(test, body[42], 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 44), 0x3fc0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 46), 0x0540); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 50), 0x004b); + KUNIT_EXPECT_EQ(test, body[55], 0x7a); + KUNIT_EXPECT_EQ(test, body[58], 9); + KUNIT_EXPECT_MEMEQ(test, body + 68, ((u8[]){ 1, 2, 44, 0 }), 4); + KUNIT_EXPECT_EQ(test, body[81], 1); + KUNIT_EXPECT_EQ(test, body[82], 1); + KUNIT_EXPECT_EQ(test, body[83], 2); + KUNIT_EXPECT_EQ(test, body[84], 155); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 86), 0x0540); + KUNIT_EXPECT_EQ(test, body[88], 2); + KUNIT_EXPECT_EQ(test, body[89], 2); + KUNIT_EXPECT_MEMEQ(test, body + 90, ((u8[]){ 4, 1, 0, 2 }), 4); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 96), 0x3ff0); + KUNIT_EXPECT_MEMEQ(test, body + 104, ((u8[12]) {}), 12); +} + +static void mt7932_awdl_sta_test(struct kunit *test) +{ + struct mt7932_awdl_peer_caps caps = { + .phy = 0x7a, .rates = 0x3fc0, .ht_mcs = 0xff, .mcs32 = 1, + .rx_mcs = { 0xff, 0xff }, .rx_highest = 867, + .ht_cap = 0x1234, .ht_ext_cap = 0x5678, .tx_bf_cap = 0x11223344, + .ampdu = 0x1b, .asel = 2, .rcpi = 80, + .vht_cap = 0x55667788, .vht_rx_mcs = 0xfffa, + .vht_rx_highest = 866, .vht_tx_mcs = 0xfffa, + .vht_tx_highest = 866, .nss = 2, .he = { 0xaa, 0xbb }, + }; + u8 body[MT7932_AWDL_STA_RECORD_LEN]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_sta(body, 3, 4, test_mac, &caps, + true, true), 200); + KUNIT_EXPECT_MEMEQ(test, body, + ((u8[]){ 3, 0x10, 2, 1, 2, 3, 4, 5 }), 8); + KUNIT_EXPECT_EQ(test, body[12], 1); + KUNIT_EXPECT_EQ(test, body[13], 0x7a); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 14), 0x3fc0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 16), 0x0540); + KUNIT_EXPECT_EQ(test, body[18], 1); + KUNIT_EXPECT_EQ(test, body[20], 2); + KUNIT_EXPECT_EQ(test, body[23], 1); + KUNIT_EXPECT_MEMEQ(test, body + 24, caps.rx_mcs, 10); + KUNIT_EXPECT_EQ(test, body[51], 1); + KUNIT_EXPECT_EQ(test, body[54], 4); + KUNIT_EXPECT_EQ(test, body[55], 0xff); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 56), 0x55667788); + KUNIT_EXPECT_EQ(test, body[68], 2); + KUNIT_EXPECT_EQ(test, body[125], 1); + KUNIT_EXPECT_MEMEQ(test, body + 136, caps.he, 34); + KUNIT_EXPECT_MEMEQ(test, body + 170, ((u8[30]) {}), 30); + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_sta(body, 3, 4, test_mac, &caps, + false, true), 200); + KUNIT_EXPECT_EQ(test, body[20], 0); +} + +static void mt7932_awdl_tlv_test(struct kunit *test) +{ + u8 body[8] = { 1, 0, 0, 0, 0x44, 0x33, 0x22, 0x11 }; + u8 out[20]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_tlv(out, sizeof(out), 0x0e, + body, sizeof(body)), 20); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(out), 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(out + 2), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(out + 4), 0x0e); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(out + 8), 8); + KUNIT_EXPECT_MEMEQ(test, out + 12, body, sizeof(body)); +} + +static void mt7932_awdl_schedule_test(struct kunit *test) +{ + const u8 channels[] = { 149, 6 }; + const u8 classes[] = { 128, 81 }; + u8 body[134]; + + KUNIT_ASSERT_EQ(test, mt7932_awdl_build_channel_sequence(body, channels, + classes, 2), 134); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body), 1); + KUNIT_EXPECT_EQ(test, body[4], 4); + KUNIT_EXPECT_EQ(test, body[5], 2); + KUNIT_EXPECT_MEMEQ(test, body + 6, channels, 2); + KUNIT_EXPECT_MEMEQ(test, body + 70, classes, 2); + KUNIT_EXPECT_MEMEQ(test, body + 72, ((u8[62]) {}), 62); +} + +static void mt7932_awdl_data_test(struct kunit *test) +{ + u8 ethernet[64] = { 0 }, wrapped[80]; + size_t length; + int ret; + + memcpy(ethernet, "\x33\x33\x00\x00\x00\xfb\x02\x01\x02\x03\x04\x05", 12); + put_unaligned_be16(ETH_P_IPV6, ethernet + 12); + memset(ethernet + 14, 0xa5, 50); + ret = mt7932_awdl_encap(wrapped, sizeof(wrapped), ethernet, sizeof(ethernet), + 0x1234); + KUNIT_ASSERT_EQ(test, ret, 80); + KUNIT_EXPECT_EQ(test, get_unaligned_be16(wrapped + 12), 66); + KUNIT_EXPECT_MEMEQ(test, wrapped + 14, + ((u8[]){ 0xaa, 0xaa, 3, 0, 0x17, 0xf2, + 8, 0, 3, 4 }), 10); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(wrapped + 24), 0x1234); + KUNIT_EXPECT_EQ(test, get_unaligned_be16(wrapped + 28), ETH_P_IPV6); + length = ret; + KUNIT_ASSERT_EQ(test, mt7932_awdl_decap(wrapped, &length), 0); + KUNIT_EXPECT_EQ(test, length, sizeof(ethernet)); + KUNIT_EXPECT_MEMEQ(test, wrapped, ethernet, sizeof(ethernet)); +} + +static void mt7932_awdl_rx_test(struct kunit *test) +{ + u8 packet[24 + ETH_HLEN] = {}; + struct mt7932_rx_frame frame; + + put_unaligned_le32(sizeof(packet) | (2U << 27), packet); + put_unaligned_le32(4, packet + 4); + put_unaligned_le32(1 | BIT(13), packet + 8); + put_unaligned_le32((149U << 8) | BIT(16), packet + 12); + put_unaligned_be16(ETH_P_IP, packet + 24 + 12); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.translated); + KUNIT_EXPECT_EQ(test, frame.bss, 1); + KUNIT_EXPECT_EQ(test, frame.wtbl, 4); + KUNIT_EXPECT_EQ(test, frame.channel, 149); +} + +static void mt7932_awdl_event_test(struct kunit *test) +{ + u8 packet[52] = {}; + struct mt7932_event event = { + .eid = 0xf7, .packet = packet, .length = sizeof(packet), + }; + struct mt7932_awdl_event_info info; + + put_unaligned_le32(7, packet + 40); + KUNIT_ASSERT_EQ(test, mt7932_awdl_event_parse(&event, &info), 0); + KUNIT_EXPECT_EQ(test, info.subtype, 7U); + KUNIT_EXPECT_PTR_EQ(test, info.data, packet + 44); + KUNIT_EXPECT_EQ(test, info.length, (size_t)8); + event.length = 43; + KUNIT_EXPECT_EQ(test, mt7932_awdl_event_parse(&event, &info), -EMSGSIZE); +} + +static struct kunit_case mt7932_awdl_cases[] = { + KUNIT_CASE(mt7932_awdl_queue_test), + KUNIT_CASE(mt7932_awdl_active_test), + KUNIT_CASE(mt7932_awdl_bss_test), + KUNIT_CASE(mt7932_awdl_sta_test), + KUNIT_CASE(mt7932_awdl_tlv_test), + KUNIT_CASE(mt7932_awdl_schedule_test), + KUNIT_CASE(mt7932_awdl_data_test), + KUNIT_CASE(mt7932_awdl_rx_test), + KUNIT_CASE(mt7932_awdl_event_test), + {} +}; + +static struct kunit_suite mt7932_awdl_suite = { + .name = "mt7932-awdl", + .test_cases = mt7932_awdl_cases, +}; + +kunit_test_suite(mt7932_awdl_suite); + diff --git a/drivers/net/wireless/mediatek/mt7932/awdl.c b/drivers/net/wireless/mediatek/mt7932/awdl.c new file mode 100644 index 0000000000000..74fa6967b841f --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/awdl.c @@ -0,0 +1,1026 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Clean-room MT7932 AWDL support. Firmware ABI values in this file come + * from the two CLEAN-FACTS handoffs; Ethernet framing follows the licensed + * brcmfmac AWDL helper in this tree. Unlisted command bytes stay zero. + */ +#include "mt7932.h" + +#include + +#define MT7932_AWDL_STA_SLOTS 15 +#define MT7932_AWDL_WTBL_SLOTS 14 + +static bool awdl_sync_enable; +module_param_named(awdl_sync_enable, awdl_sync_enable, bool, 0444); +MODULE_PARM_DESC(awdl_sync_enable, + "enable AWDL synchronization after complete templates/schedule are supplied (default false)"); + +/* TODO(t-5787 live payload capture): replace these unknown policy defaults. */ +static uint awdl_psf_period; +module_param_named(awdl_psf_period, awdl_psf_period, uint, 0444); +MODULE_PARM_DESC(awdl_psf_period, "tag 0x03 PSF period (default 0; capture missing)"); +static uint awdl_min_af_rate; +module_param_named(awdl_min_af_rate, awdl_min_af_rate, uint, 0444); +MODULE_PARM_DESC(awdl_min_af_rate, "tag 0x0d AF rate selector (default 0; capture missing)"); +static int awdl_edge_rssi; +module_param_named(awdl_edge_rssi, awdl_edge_rssi, int, 0444); +MODULE_PARM_DESC(awdl_edge_rssi, "tag 0x07 edge RSSI (default 0; capture missing)"); +static uint awdl_af_tx_mode; +module_param_named(awdl_af_tx_mode, awdl_af_tx_mode, uint, 0444); +MODULE_PARM_DESC(awdl_af_tx_mode, "tag 0x0c AF TX mode (default 0; capture missing)"); +static uint awdl_election_metric; +module_param_named(awdl_election_metric, awdl_election_metric, uint, 0444); +MODULE_PARM_DESC(awdl_election_metric, "tag 0x0b election metric (default 0; capture missing)"); + +/* These words are copied, not decoded: their bit allocation is not known. */ +static uint awdl_primary_master_channel; +module_param_named(awdl_primary_master_channel, awdl_primary_master_channel, uint, 0444); +MODULE_PARM_DESC(awdl_primary_master_channel, "verbatim tag 0x0e channel word (default 0)"); +static uint awdl_secondary_master_channel; +module_param_named(awdl_secondary_master_channel, awdl_secondary_master_channel, uint, 0444); +MODULE_PARM_DESC(awdl_secondary_master_channel, "verbatim tag 0x0f channel word (default 0)"); + +/* TODO(t-5787 live payload capture): exact normal count/order is unknown. */ +static uint awdl_channels[MT7932_AWDL_MAX_SCHEDULE]; +static unsigned int awdl_channel_count; +module_param_array_named(awdl_channels, awdl_channels, uint, &awdl_channel_count, 0444); +MODULE_PARM_DESC(awdl_channels, "tag 0x02 channel list (default empty; capture missing)"); +static uint awdl_operating_classes[MT7932_AWDL_MAX_SCHEDULE]; +static unsigned int awdl_class_count; +module_param_array_named(awdl_operating_classes, awdl_operating_classes, uint, + &awdl_class_count, 0444); +MODULE_PARM_DESC(awdl_operating_classes, + "tag 0x02 operating-class list (default empty; capture missing; examples 149/128 and 6/81)"); + +static char *awdl_bssid = "00:25:00:ff:94:73"; +module_param_named(awdl_bssid, awdl_bssid, charp, 0444); +MODULE_PARM_DESC(awdl_bssid, "AWDL BSSID (default public common AWDL BSSID)"); +static char *awdl_sync_master = "00:00:00:00:00:00"; +module_param_named(awdl_sync_master, awdl_sync_master, charp, 0444); +MODULE_PARM_DESC(awdl_sync_master, "host-selected tag 0x09 sync-master MAC"); +static char *awdl_top_master = "0000000000000000"; +module_param_named(awdl_top_master, awdl_top_master, charp, 0444); +MODULE_PARM_DESC(awdl_top_master, "host-selected tag 0x0a 8-byte top-master identity as hex"); +static uint awdl_election_id; +module_param_named(awdl_election_id, awdl_election_id, uint, 0444); +static uint awdl_distance; +module_param_named(awdl_distance, awdl_distance, uint, 0444); +static uint awdl_top_master_metric; +module_param_named(awdl_top_master_metric, awdl_top_master_metric, uint, 0444); +static uint awdl_self_metric; +module_param_named(awdl_self_metric, awdl_self_metric, uint, 0444); + +static char *awdl_broadcast_template = ""; +module_param_named(awdl_broadcast_template, awdl_broadcast_template, charp, 0444); +MODULE_PARM_DESC(awdl_broadcast_template, "broadcast AWDL payload/TLV bytes as hex (default empty)"); +static char *awdl_psf_template = ""; +module_param_named(awdl_psf_template, awdl_psf_template, charp, 0444); +MODULE_PARM_DESC(awdl_psf_template, "PSF AWDL payload/TLV bytes as hex (default empty)"); + +static const u8 mt7932_awdl_snap[10] = { + 0xaa, 0xaa, 0x03, 0x00, 0x17, 0xf2, 0x08, 0x00, 0x03, 0x04, +}; + +int mt7932_awdl_build_queue(u8 out[MT7932_AWDL_QUEUE_RECORD_LEN], + u8 station, bool peer, bool enable) +{ + static const u8 tids[8] = { 1, 0, 0, 1, 2, 2, 3, 3 }; + + memset(out, 0, MT7932_AWDL_QUEUE_RECORD_LEN); + if (!peer) { + put_unaligned_le16(enable ? 2 : 0, out); + out[2] = 1; + out[3] = MT7932_AWDL_BSS_INDEX; + if (enable) + out[14] = 1; + return MT7932_AWDL_QUEUE_RECORD_LEN; + } + if (station >= MT7932_AWDL_STA_SLOTS) + return -EINVAL; + put_unaligned_le16(15, out); + out[2] = 5; + out[14] = 1; + out[15] = station; + put_unaligned_le16(enable ? 15 : 2, out + 16); + memcpy(out + 20, tids, sizeof(tids)); + return MT7932_AWDL_QUEUE_RECORD_LEN; +} + +int mt7932_awdl_build_bss_active(u8 out[12], const u8 *mac, u8 bmc_wtbl, + bool active) +{ + if (!mac || !is_valid_ether_addr(mac) || bmc_wtbl >= MT7932_AWDL_WTBL_SLOTS) + return -EINVAL; + memset(out, 0, 12); + out[0] = MT7932_AWDL_BSS_INDEX; + out[1] = active; + out[2] = MT7932_AWDL_NETWORK_TYPE; + out[3] = 0; /* own-MAC index, deliberately not the BSS index */ + if (active) + ether_addr_copy(out + 4, mac); + out[10] = bmc_wtbl; + out[11] = 1; + return 12; +} + +int mt7932_awdl_build_bss(u8 out[MT7932_AWDL_BSS_RECORD_LEN], u8 bmc_wtbl, + u8 wmm, u8 phy, u8 nss) +{ + if (bmc_wtbl >= MT7932_AWDL_WTBL_SLOTS || !nss) + return -EINVAL; + memset(out, 0, MT7932_AWDL_BSS_RECORD_LEN); + out[0] = 1; + out[1] = 1; + out[2] = 7; + out[42] = 1; + put_unaligned_le16(0x3fc0, out + 44); + put_unaligned_le16(0x0540, out + 46); + put_unaligned_le16(0x004b, out + 50); + out[52] = 2; + out[55] = phy; + out[58] = bmc_wtbl; + out[68] = 1; + out[69] = 2; + out[70] = 44; + out[81] = 1; + out[82] = 1; + out[83] = 2; + out[84] = 155; + put_unaligned_le16(0x0540, out + 86); + out[88] = nss; + out[89] = nss; + out[90] = 4; + out[91] = wmm; + out[93] = nss; + put_unaligned_le16(0x3ff0, out + 96); + return MT7932_AWDL_BSS_RECORD_LEN; +} + +int mt7932_awdl_build_sta(u8 out[MT7932_AWDL_STA_RECORD_LEN], u8 station, + u8 wtbl, const u8 *mac, + const struct mt7932_awdl_peer_caps *caps, + bool live, bool need_response) +{ + if (station >= MT7932_AWDL_STA_SLOTS || wtbl >= MT7932_AWDL_WTBL_SLOTS || + !mac || !is_valid_ether_addr(mac) || !caps) + return -EINVAL; + memset(out, 0, MT7932_AWDL_STA_RECORD_LEN); + out[0] = station; + out[1] = 0x10; + ether_addr_copy(out + 2, mac); + out[12] = 1; + out[13] = caps->phy; + put_unaligned_le16(caps->rates, out + 14); + put_unaligned_le16(0x0540, out + 16); + out[18] = 1; + out[20] = live ? 2 : 0; + out[21] = caps->ht_mcs; + out[22] = caps->mcs32; + out[23] = 1; + memcpy(out + 24, caps->rx_mcs, sizeof(caps->rx_mcs)); + put_unaligned_le16(caps->rx_highest, out + 34); + put_unaligned_le16(caps->ht_cap, out + 40); + put_unaligned_le16(caps->ht_ext_cap, out + 42); + put_unaligned_le32(caps->tx_bf_cap, out + 44); + out[48] = caps->ampdu; + out[49] = caps->asel; + out[50] = caps->rcpi; + out[51] = need_response; + out[54] = wtbl; + out[55] = 0xff; + put_unaligned_le32(caps->vht_cap, out + 56); + put_unaligned_le16(caps->vht_rx_mcs, out + 60); + put_unaligned_le16(caps->vht_rx_highest, out + 62); + put_unaligned_le16(caps->vht_tx_mcs, out + 64); + put_unaligned_le16(caps->vht_tx_highest, out + 66); + out[68] = caps->nss > 1 ? 2 : 1; + out[125] = 1; + memcpy(out + 136, caps->he, sizeof(caps->he)); + return MT7932_AWDL_STA_RECORD_LEN; +} + +int mt7932_awdl_build_tlv(u8 *out, size_t capacity, u32 tag, + const void *body, size_t body_len) +{ + if (!out || body_len > U32_MAX || body_len + 12 > capacity || + (body_len && !body)) + return -EINVAL; + memset(out, 0, body_len + 12); + put_unaligned_le16(1, out); + put_unaligned_le32(tag, out + 4); + put_unaligned_le32(body_len, out + 8); + if (body_len) + memcpy(out + 12, body, body_len); + return body_len + 12; +} + +int mt7932_awdl_build_channel_sequence(u8 out[134], const u8 *channels, + const u8 *classes, size_t count) +{ + if (count > MT7932_AWDL_MAX_SCHEDULE || (count && (!channels || !classes))) + return -EINVAL; + memset(out, 0, 134); + put_unaligned_le32(MT7932_AWDL_BSS_INDEX, out); + out[4] = 4; + out[5] = count; + if (count) { + memcpy(out + 6, channels, count); + memcpy(out + 70, classes, count); + } + return 134; +} + +int mt7932_awdl_event_parse(const struct mt7932_event *event, + struct mt7932_awdl_event_info *info) +{ + if (!event || !info || event->eid != 0xf7) + return -ENOMSG; + if (event->length < 44) + return -EMSGSIZE; + info->subtype = get_unaligned_le32(event->packet + 40); + info->data = event->packet + 44; + info->length = event->length - 44; + return 0; +} + +int mt7932_awdl_encap(u8 *out, size_t capacity, const u8 *ethernet, + size_t length, u16 sequence) +{ + if (!out || !ethernet || length < ETH_HLEN || + length + MT7932_AWDL_ENCAP_LEN > capacity || + length - ETH_HLEN + MT7932_AWDL_ENCAP_LEN > ETH_DATA_LEN) + return -EMSGSIZE; + if (get_unaligned_be16(ethernet + 12) < ETH_P_802_3_MIN) { + memcpy(out, ethernet, length); + return length; + } + memcpy(out, ethernet, 2 * ETH_ALEN); + put_unaligned_be16(length - ETH_HLEN + MT7932_AWDL_ENCAP_LEN, out + 12); + memcpy(out + ETH_HLEN, mt7932_awdl_snap, sizeof(mt7932_awdl_snap)); + put_unaligned_le16(sequence, out + ETH_HLEN + sizeof(mt7932_awdl_snap)); + out[ETH_HLEN + sizeof(mt7932_awdl_snap) + 2] = 0; + out[ETH_HLEN + sizeof(mt7932_awdl_snap) + 3] = 0; + /* The new 16-byte area replaces the old ethertype position; retain the + * original ethertype immediately after that area. + */ + memcpy(out + 2 * ETH_ALEN + MT7932_AWDL_ENCAP_LEN, + ethernet + 2 * ETH_ALEN, length - 2 * ETH_ALEN); + return length + MT7932_AWDL_ENCAP_LEN; +} + +int mt7932_awdl_decap(u8 *frame, size_t *length) +{ + if (!frame || !length || *length < ETH_HLEN + MT7932_AWDL_ENCAP_LEN) + return -EMSGSIZE; + if (get_unaligned_be16(frame + 12) > ETH_DATA_LEN || + memcmp(frame + ETH_HLEN, mt7932_awdl_snap, sizeof(mt7932_awdl_snap))) + return -EPROTO; + memmove(frame + MT7932_AWDL_ENCAP_LEN, frame, 2 * ETH_ALEN); + memmove(frame, frame + MT7932_AWDL_ENCAP_LEN, *length - MT7932_AWDL_ENCAP_LEN); + *length -= MT7932_AWDL_ENCAP_LEN; + return 0; +} + +static int mt_awdl_send_tlv(struct mt7932 *m, u32 tag, const void *body, + size_t body_len) +{ + u8 *payload; + int length, ret; + + payload = kmalloc(body_len + 12, GFP_KERNEL); + if (!payload) + return -ENOMEM; + length = mt7932_awdl_build_tlv(payload, body_len + 12, tag, body, body_len); + ret = length < 0 ? length : mt_request(m, 0xf7, true, true, false, + payload, length); + kfree(payload); + return ret; +} + +static int mt_awdl_fixed_u32(struct mt7932 *m, u32 tag, u32 value) +{ + u8 body[8] = {}; + + put_unaligned_le32(1, body); + put_unaligned_le32(value, body + 4); + return mt_awdl_send_tlv(m, tag, body, sizeof(body)); +} + +static int mt_awdl_capability_prefix(struct mt7932 *m, u8 out[40]) +{ + const struct ieee80211_sta_ht_cap *ht = &m->band5.ht_cap; + const struct ieee80211_sta_vht_cap *vht = &m->band5.vht_cap; + + if (!ht->ht_supported || !vht->vht_supported) + return -EOPNOTSUPP; + memset(out, 0, 40); + out[0] = 0x07; + put_unaligned_le16(20, out + 1); + put_unaligned_le16(ht->cap, out + 5); + out[7] = ht->ampdu_factor | (ht->ampdu_density << 2); + memcpy(out + 8, ht->mcs.rx_mask, 4); + out[23] = 0x11; + put_unaligned_le16(14, out + 24); + out[26] = WLAN_EID_VHT_CAPABILITY; + out[27] = 12; + put_unaligned_le32(vht->cap, out + 28); + memcpy(out + 32, &vht->vht_mcs, sizeof(vht->vht_mcs)); + return 40; +} + +static int mt_awdl_template(struct mt7932 *m, u32 tag, const char *hex) +{ + u8 prefix[40], *body; + size_t chars = strlen(hex), bytes; + int prefix_len, ret; + + if (chars & 1) + return -EINVAL; + bytes = chars / 2; + if (bytes > 1024) + return -E2BIG; + prefix_len = mt_awdl_capability_prefix(m, prefix); + if (prefix_len < 0) + return prefix_len; + body = kzalloc(8 + prefix_len + bytes, GFP_KERNEL); + if (!body) + return -ENOMEM; + put_unaligned_le32(1, body); + put_unaligned_le32(prefix_len + bytes, body + 4); + memcpy(body + 8, prefix, prefix_len); + if (bytes && hex2bin(body + 8 + prefix_len, hex, bytes)) { + ret = -EINVAL; + goto out; + } + ret = mt_awdl_send_tlv(m, tag, body, 8 + prefix_len + bytes); +out: + kfree(body); + return ret; +} + +static int mt_awdl_configure_sync(struct mt7932 *m) +{ + u8 body[134], mac[ETH_ALEN], top[8], channels[64], classes[64]; + u8 election[21] = {}, enable[6] = {}; + unsigned int i; + int ret; + + if (!awdl_sync_enable) + return 0; + if (!awdl_channel_count || awdl_channel_count != awdl_class_count || + !*awdl_broadcast_template || !*awdl_psf_template) + return -EINVAL; + for (i = 0; i < awdl_channel_count; i++) { + if (awdl_channels[i] > U8_MAX || awdl_operating_classes[i] > U8_MAX) + return -ERANGE; + channels[i] = awdl_channels[i]; + classes[i] = awdl_operating_classes[i]; + } + if (!mac_pton(awdl_sync_master, mac) || strlen(awdl_top_master) != 16 || + hex2bin(top, awdl_top_master, sizeof(top))) + return -EINVAL; + ret = mt_awdl_fixed_u32(m, 0x0e, awdl_primary_master_channel); + ret = ret ?: mt_awdl_fixed_u32(m, 0x0f, awdl_secondary_master_channel); + memset(body, 0, sizeof(body)); + put_unaligned_le32(1, body); + body[4] = awdl_min_af_rate; + ret = ret ?: mt_awdl_send_tlv(m, 0x0d, body, 5); + body[4] = clamp_t(int, awdl_edge_rssi, S8_MIN, S8_MAX); + ret = ret ?: mt_awdl_send_tlv(m, 0x07, body, 5); + if (ret) + return ret; + mt7932_awdl_build_channel_sequence(body, channels, classes, awdl_channel_count); + ret = mt_awdl_send_tlv(m, 0x02, body, 134); + memset(body, 0, 30); + put_unaligned_le32(1, body); + ether_addr_copy(body + 4, mac); + ret = ret ?: mt_awdl_send_tlv(m, 0x09, body, 30); + put_unaligned_le32(1, election); + put_unaligned_le16(awdl_election_id, election + 4); + election[6] = awdl_distance; + memcpy(election + 7, top, sizeof(top)); + put_unaligned_le32(awdl_top_master_metric, election + 15); + put_unaligned_le16(awdl_self_metric, election + 19); + ret = ret ?: mt_awdl_send_tlv(m, 0x0a, election, sizeof(election)); + ret = ret ?: mt_awdl_fixed_u32(m, 0x0b, awdl_election_metric); + ret = ret ?: mt_awdl_fixed_u32(m, 0x0c, awdl_af_tx_mode); + ret = ret ?: mt_awdl_fixed_u32(m, 0x03, awdl_psf_period); + ret = ret ?: mt_awdl_template(m, 0x04, awdl_broadcast_template); + ret = ret ?: mt_awdl_template(m, 0x05, awdl_psf_template); + put_unaligned_le32(1, enable); + put_unaligned_le16(1, enable + 4); + ret = ret ?: mt_awdl_send_tlv(m, 0x08, enable, sizeof(enable)); + if (!ret) + m->awdl.sync_enabled = true; + return ret; +} + +static int mt_awdl_alloc_id(unsigned long *map, unsigned int limit) +{ + unsigned int id = find_first_zero_bit(map, limit); + + if (id >= limit) + return -ENOSPC; + __set_bit(id, map); + return id; +} + +static void mt_awdl_release_peer(struct mt7932 *m, struct mt7932_awdl_peer *peer) +{ + __clear_bit(peer->station, &m->station_alloc); + __clear_bit(peer->wtbl, &m->wtbl_alloc); + memset(peer, 0, sizeof(*peer)); +} + +static struct mt7932_awdl_peer *mt_awdl_find_peer(struct mt7932 *m, + const u8 *mac) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(m->awdl.peers); i++) + if (m->awdl.peers[i].live && + ether_addr_equal(m->awdl.peers[i].addr, mac)) + return &m->awdl.peers[i]; + return NULL; +} + +static void mt_awdl_caps(struct mt7932_awdl_peer_caps *caps, + const struct station_parameters *params) +{ + const struct link_station_parameters *link = ¶ms->link_sta_params; + static const u8 rates[8] = { 12, 18, 24, 36, 48, 72, 96, 108 }; + unsigned int i, j; + + memset(caps, 0, sizeof(*caps)); + caps->phy = 0x7a; + caps->nss = 1; + for (i = 0; i < link->supported_rates_len; i++) + for (j = 0; j < ARRAY_SIZE(rates); j++) + if ((link->supported_rates[i] & 0x7f) == rates[j]) + caps->rates |= BIT(6 + j); + if (!caps->rates) + caps->rates = 0x3fc0; + if (link->ht_capa) { + caps->ht_cap = le16_to_cpu(link->ht_capa->cap_info); + caps->ampdu = link->ht_capa->ampdu_params_info; + memcpy(caps->rx_mcs, link->ht_capa->mcs.rx_mask, sizeof(caps->rx_mcs)); + caps->ht_mcs = caps->rx_mcs[0]; + caps->mcs32 = !!(caps->rx_mcs[4] & 1); + caps->rx_highest = le16_to_cpu(link->ht_capa->mcs.rx_highest); + caps->nss = caps->rx_mcs[1] ? 2 : 1; + } + if (link->vht_capa) { + caps->vht_cap = le32_to_cpu(link->vht_capa->vht_cap_info); + caps->vht_rx_mcs = le16_to_cpu(link->vht_capa->supp_mcs.rx_mcs_map); + caps->vht_rx_highest = le16_to_cpu(link->vht_capa->supp_mcs.rx_highest); + caps->vht_tx_mcs = le16_to_cpu(link->vht_capa->supp_mcs.tx_mcs_map); + caps->vht_tx_highest = le16_to_cpu(link->vht_capa->supp_mcs.tx_highest); + } +} + +int mt_awdl_add_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_parameters *params) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + struct mt7932_awdl_peer *peer = NULL; + u8 queue[32], sta[200]; + unsigned long flags; + int station, wtbl, ret = 0; + unsigned int i; + + if (wdev != &m->awdl.wdev || !is_valid_ether_addr(mac)) + return -ENODEV; + /* The clean tables do not identify the byte mapping of HE capabilities. */ + if (params->link_sta_params.he_capa) + return -EOPNOTSUPP; + mutex_lock(&m->command_mutex); + if (!m->awdl.active) { + ret = -ENETDOWN; + goto out; + } + if (mt_awdl_find_peer(m, mac)) { + ret = -EEXIST; + goto out; + } + for (i = 0; i < ARRAY_SIZE(m->awdl.peers); i++) + if (!is_valid_ether_addr(m->awdl.peers[i].addr)) { + peer = &m->awdl.peers[i]; + break; + } + if (!peer) { + ret = -ENOSPC; + goto out; + } + station = mt_awdl_alloc_id(&m->station_alloc, MT7932_AWDL_STA_SLOTS); + if (station < 0) { + ret = station; + goto out; + } + wtbl = mt_awdl_alloc_id(&m->wtbl_alloc, MT7932_AWDL_WTBL_SLOTS); + if (wtbl < 0) { + __clear_bit(station, &m->station_alloc); + ret = wtbl; + goto out; + } + ether_addr_copy(peer->addr, mac); + peer->station = station; + peer->wtbl = wtbl; + mt_awdl_caps(&peer->caps, params); + mt7932_awdl_build_queue(queue, station, true, true); + ret = mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + if (!ret) { + mt7932_awdl_build_sta(sta, station, wtbl, mac, &peer->caps, true, true); + ret = mt_request(m, 0x13, true, true, true, sta, sizeof(sta)); + } + if (ret) { + mt7932_awdl_build_queue(queue, station, true, false); + mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + mt_awdl_release_peer(m, peer); + } else { + spin_lock_irqsave(&m->response_lock, flags); + peer->live = true; + spin_unlock_irqrestore(&m->response_lock, flags); + } +out: + mutex_unlock(&m->command_mutex); + return ret; +} + +static int mt_awdl_remove_peer(struct mt7932 *m, struct mt7932_awdl_peer *peer) +{ + u8 queue[32], sta[200]; + unsigned long flags; + int ret; + + spin_lock_irqsave(&m->response_lock, flags); + peer->live = false; + spin_unlock_irqrestore(&m->response_lock, flags); + mt7932_awdl_build_sta(sta, peer->station, peer->wtbl, peer->addr, + &peer->caps, false, true); + ret = mt_request(m, 0x13, true, true, true, sta, sizeof(sta)); + mt7932_awdl_build_queue(queue, peer->station, true, false); + ret = ret ?: mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + if (!ret) + mt_awdl_release_peer(m, peer); + return ret; +} + +int mt_awdl_del_station(struct wiphy *wiphy, struct wireless_dev *wdev, + struct station_del_parameters *params) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned int i; + int ret = 0; + + if (wdev != &m->awdl.wdev) + return -ENODEV; + mutex_lock(&m->command_mutex); + for (i = 0; i < ARRAY_SIZE(m->awdl.peers); i++) { + struct mt7932_awdl_peer *peer = &m->awdl.peers[i]; + + if (!is_valid_ether_addr(peer->addr) || + (params->mac && !ether_addr_equal(params->mac, peer->addr))) + continue; + ret = mt_awdl_remove_peer(m, peer); + if (ret || params->mac) + break; + } + mutex_unlock(&m->command_mutex); + return ret; +} + +int mt_awdl_change_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_parameters *params) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + struct mt7932_awdl_peer_caps caps; + struct mt7932_awdl_peer *peer; + u8 sta[MT7932_AWDL_STA_RECORD_LEN]; + unsigned long flags; + int ret = 0; + + if (wdev != &m->awdl.wdev || !is_valid_ether_addr(mac)) + return -ENODEV; + if (params->link_sta_params.he_capa) + return -EOPNOTSUPP; + mt_awdl_caps(&caps, params); + mutex_lock(&m->command_mutex); + peer = mt_awdl_find_peer(m, mac); + if (!peer) { + ret = -ENOENT; + goto out; + } + if (!memcmp(&caps, &peer->caps, sizeof(caps))) + goto out; + mt7932_awdl_build_sta(sta, peer->station, peer->wtbl, peer->addr, + &caps, true, true); + ret = mt_request(m, 0x13, true, true, true, sta, sizeof(sta)); + if (!ret) { + spin_lock_irqsave(&m->response_lock, flags); + peer->caps = caps; + spin_unlock_irqrestore(&m->response_lock, flags); + } +out: + mutex_unlock(&m->command_mutex); + return ret; +} + +static int mt_awdl_clear_template(struct mt7932 *m, u32 tag) +{ + u8 body[8] = {}; + + put_unaligned_le32(1, body); + return mt_awdl_send_tlv(m, tag, body, sizeof(body)); +} + +int mt_awdl_activate(struct mt7932 *m) +{ + u8 queue[32], bss[116], active[12], tag_bssid[10]; + int bmc, wmm, ret; + + if (m->awdl.active || !m->awdl.interface_up) + return 0; + bmc = mt_awdl_alloc_id(&m->wtbl_alloc, MT7932_AWDL_WTBL_SLOTS); + if (bmc < 0) + return bmc; + m->awdl.bmc_wtbl = bmc; + wmm = mt_awdl_alloc_id(&m->wmm_alloc, 2); + if (wmm < 0) { + __clear_bit(bmc, &m->wtbl_alloc); + return wmm; + } + m->awdl.wmm = wmm; + mt7932_awdl_build_queue(queue, 0, false, true); + ret = mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + if (!ret) { + mt7932_awdl_build_bss(bss, bmc, wmm, 0x7a, + clamp_t(u8, m->phy_cap[4], 1, 2)); + ret = mt_request(m, 0x12, true, true, true, bss, sizeof(bss)); + } + if (!ret) { + mt7932_awdl_build_bss_active(active, m->awdl.netdev->dev_addr, bmc, true); + ret = mt_request(m, 0x11, true, true, true, active, sizeof(active)); + } + ret = ret ?: mt_awdl_clear_template(m, 0x04); + ret = ret ?: mt_awdl_clear_template(m, 0x05); + if (ret) + goto rollback; + m->awdl.active = true; + put_unaligned_le32(1, tag_bssid); + ether_addr_copy(tag_bssid + 4, m->awdl.bssid); + ret = mt_awdl_send_tlv(m, 0x00, tag_bssid, sizeof(tag_bssid)); + ret = ret ?: mt_awdl_configure_sync(m); + if (!ret) + return 0; + /* A rejected optional sync policy must leave a safely created, disabled BSS. */ + if (!m->awdl.sync_enabled && ret == -EINVAL) { + dev_warn(&m->pdev->dev, + "AWDL sync remains disabled: complete schedule/templates required\n"); + return 0; + } +rollback: + mt7932_awdl_build_bss_active(active, m->awdl.netdev->dev_addr, bmc, false); + mt_request(m, 0x11, true, true, true, active, sizeof(active)); + mt7932_awdl_build_queue(queue, 0, false, false); + mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + __clear_bit(bmc, &m->wtbl_alloc); + __clear_bit(wmm, &m->wmm_alloc); + m->awdl.active = false; + return ret; +} + +static int mt_awdl_deactivate(struct mt7932 *m) +{ + u8 body[116], active[12], queue[32], enable[6] = {}; + unsigned int i; + int ret = 0; + + if (!m->awdl.active) + return 0; + netif_carrier_off(m->awdl.netdev); + netif_stop_queue(m->awdl.netdev); + for (i = 0; i < ARRAY_SIZE(m->awdl.peers); i++) + if (is_valid_ether_addr(m->awdl.peers[i].addr)) { + ret = mt_awdl_remove_peer(m, &m->awdl.peers[i]); + if (ret) + return ret; + } + put_unaligned_le32(1, enable); + ret = mt_awdl_send_tlv(m, 0x08, enable, sizeof(enable)); + mt7932_awdl_build_bss_active(active, m->awdl.netdev->dev_addr, + m->awdl.bmc_wtbl, false); + ret = ret ?: mt_request(m, 0x11, true, true, true, active, sizeof(active)); + mt7932_awdl_build_bss(body, m->awdl.bmc_wtbl, m->awdl.wmm, 0x7a, + clamp_t(u8, m->phy_cap[4], 1, 2)); + body[1] = 0; + ret = ret ?: mt_request(m, 0x12, true, true, true, body, sizeof(body)); + mt7932_awdl_build_queue(queue, 0, false, false); + ret = ret ?: mt_request(m, 0x6b, true, true, true, queue, sizeof(queue)); + if (!ret) { + __clear_bit(m->awdl.bmc_wtbl, &m->wtbl_alloc); + __clear_bit(m->awdl.wmm, &m->wmm_alloc); + m->awdl.active = false; + m->awdl.sync_enabled = false; + m->awdl.slot_open = false; + } + return ret; +} + +static int mt_awdl_net_open(struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + int ret = 0; + + mutex_lock(&m->command_mutex); + if (m->stopping) { + ret = -ESHUTDOWN; + } else if (m->link_failed || m->cal_state.error) { + ret = m->cal_state.error ?: -EIO; + } else { + m->awdl.interface_up = true; + if (m->rf_ready) + ret = mt_awdl_activate(m); + if (ret) + m->awdl.interface_up = false; + } + mutex_unlock(&m->command_mutex); + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return ret; +} + +static int mt_awdl_net_stop(struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + + m->awdl.interface_up = false; + if (!m->stopping) { + mutex_lock(&m->command_mutex); + mt_awdl_deactivate(m); + mutex_unlock(&m->command_mutex); + } + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; +} + +static netdev_tx_t mt_awdl_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + struct mt7932_ring *q = &m->tx[0]; + struct mt7932_desc *desc; + unsigned long flags; + u8 *payload, *header; + dma_addr_t dma; + unsigned int slot, wtbl; + int length, ret; + + if (!m->awdl.active || !m->awdl.sync_enabled || !m->awdl.slot_open || + !m->data_ready || skb->len < ETH_HLEN || skb->len > netdev->mtu + ETH_HLEN) + goto drop; + spin_lock_irqsave(&m->response_lock, flags); + if (is_multicast_ether_addr(skb->data)) { + wtbl = m->awdl.bmc_wtbl; + } else { + struct mt7932_awdl_peer *peer = mt_awdl_find_peer(m, skb->data); + + if (!peer) { + spin_unlock_irqrestore(&m->response_lock, flags); + goto drop; + } + wtbl = peer->wtbl; + } + spin_unlock_irqrestore(&m->response_lock, flags); + spin_lock_irqsave(&m->data_lock, flags); + mt_data_clean_locked(m); + slot = q->head; + if (!m->awdl.slot_open || m->data_live[slot] || q->queued >= q->count - 1) { + netif_stop_queue(netdev); + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + return NETDEV_TX_BUSY; + } + payload = m->data_payloads + 4096 * slot; + header = m->data_headers + 64 * slot; + length = mt7932_awdl_encap(payload, 4096, skb->data, skb->len, + m->awdl.tx_sequence++); + ret = length < 0 ? length : + mt7932_data_header(header, payload, length, + m->data_payloads_dma + 4096 * slot, slot, + wtbl, 0, 0, false); + if (ret) { + spin_unlock_irqrestore(&m->data_lock, flags); + goto drop; + } + dma = m->data_headers_dma + 64 * slot; + desc = &q->desc[slot]; + desc->address = cpu_to_le32(lower_32_bits(dma)); + desc->second = 0; + desc->info = cpu_to_le32(upper_32_bits(dma)); + m->data_live[slot] = m->data_token[slot] = true; + m->data_retired[slot] = false; + dma_wmb(); + desc->control = cpu_to_le32((64 << 16) | BIT(30)); + q->head = (slot + 1) % q->count; + q->queued++; + dma_wmb(); + m->data_unkicked++; + if (!netdev_xmit_more() || q->queued >= q->count - 1) + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + netdev->stats.tx_packets++; + netdev->stats.tx_bytes += skb->len; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +drop: + if (!netdev_xmit_more() && READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + } + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +static const struct net_device_ops mt_awdl_net_ops = { + .ndo_open = mt_awdl_net_open, + .ndo_stop = mt_awdl_net_stop, + .ndo_start_xmit = mt_awdl_xmit, + .ndo_set_mac_address = eth_mac_addr, +}; + +void mt_awdl_wiphy_setup(struct mt7932 *m) +{ + static const struct ieee80211_iface_limit limits[] = { + { .max = 1, .types = BIT(NL80211_IFTYPE_STATION) }, + { .max = 1, .types = BIT(NL80211_IFTYPE_NAN) }, + }; + static const struct ieee80211_iface_combination combinations[] = { + { + .limits = limits, + .n_limits = ARRAY_SIZE(limits), + .max_interfaces = 2, + .num_different_channels = 2, + }, + }; + + m->wiphy->interface_modes |= BIT(NL80211_IFTYPE_NAN); + m->wiphy->iface_combinations = combinations; + m->wiphy->n_iface_combinations = ARRAY_SIZE(combinations); +} + +int mt_awdl_register(struct mt7932 *m) +{ + struct net_device *netdev; + u8 mac[ETH_ALEN]; + int ret; + + if (!mac_pton(awdl_bssid, m->awdl.bssid)) + return -EINVAL; + netdev = alloc_etherdev(sizeof(m)); + if (!netdev) + return -ENOMEM; + *(struct mt7932 **)netdev_priv(netdev) = m; + m->awdl.wdev.wiphy = m->wiphy; + m->awdl.wdev.iftype = NL80211_IFTYPE_NAN; + m->awdl.wdev.netdev = netdev; + netdev->ieee80211_ptr = &m->awdl.wdev; + netdev->netdev_ops = &mt_awdl_net_ops; + netdev->min_mtu = ETH_MIN_MTU; + netdev->max_mtu = ETH_DATA_LEN - MT7932_AWDL_ENCAP_LEN; + netdev->needed_headroom = MT7932_AWDL_ENCAP_LEN; + SET_NETDEV_DEV(netdev, &m->pdev->dev); + ether_addr_copy(mac, m->mac); + mac[0] |= 0x02; + mac[5] ^= 0x01; + eth_hw_addr_set(netdev, mac); + ether_addr_copy(netdev->perm_addr, mac); + strscpy(netdev->name, "awdl0", IFNAMSIZ); + netif_carrier_off(netdev); + m->awdl.netdev = netdev; + ret = register_netdev(netdev); + if (ret) { + m->awdl.netdev = NULL; + free_netdev(netdev); + return ret; + } + m->awdl.registered = true; + return 0; +} + +void mt_awdl_unregister(struct mt7932 *m) +{ + if (!m->awdl.netdev) + return; + if (m->awdl.registered) + unregister_netdev(m->awdl.netdev); + free_netdev(m->awdl.netdev); + m->awdl.netdev = NULL; + m->awdl.registered = false; +} + +void mt_awdl_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) +{ + struct sk_buff *skb; + unsigned long flags; + unsigned int i; + bool owned = false; + size_t length; + + if (!m->awdl.active || frame->bss != 1) + return; + if (!frame->translated) { + if (frame->length < 28 || (frame->frame_control & 0xfc) != IEEE80211_STYPE_ACTION || + frame->data[24] != 0x7f || memcmp(frame->data + 25, "\x00\x17\xf2", 3) || + !ether_addr_equal(frame->data + 16, m->awdl.bssid)) + return; + m->awdl.rx_actions++; + cfg80211_rx_mgmt(&m->awdl.wdev, + ieee80211_channel_to_frequency(frame->channel, frame->band), + mt7932_rx_signal(frame, m->antenna_mask), frame->data, + frame->length, 0); + return; + } + spin_lock_irqsave(&m->response_lock, flags); + if (!frame->unicast) { + owned = frame->wtbl == m->awdl.bmc_wtbl; + } else { + for (i = 0; i < ARRAY_SIZE(m->awdl.peers); i++) + if (m->awdl.peers[i].live && + m->awdl.peers[i].wtbl == frame->wtbl) { + owned = true; + break; + } + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (!owned) { + m->awdl.netdev->stats.rx_dropped++; + return; + } + skb = netdev_alloc_skb_ip_align(m->awdl.netdev, frame->length); + if (!skb) { + m->awdl.netdev->stats.rx_dropped++; + return; + } + skb_put_data(skb, frame->data, frame->length); + length = skb->len; + if (mt7932_awdl_decap(skb->data, &length)) { + m->awdl.netdev->stats.rx_dropped++; + dev_kfree_skb_any(skb); + return; + } + skb_trim(skb, length); + skb->protocol = eth_type_trans(skb, m->awdl.netdev); + skb->ip_summed = CHECKSUM_NONE; + m->awdl.netdev->stats.rx_packets++; + m->awdl.netdev->stats.rx_bytes += length; + m->awdl.rx_data++; + netif_rx(skb); +} + +void mt_awdl_event(struct mt7932 *m, const struct mt7932_event *event) +{ + struct mt7932_awdl_event_info info; + + if (mt7932_awdl_event_parse(event, &info)) + return; + if (info.subtype < ARRAY_SIZE(m->awdl.events)) + m->awdl.events[info.subtype]++; + if (!m->awdl.netdev || !m->awdl.active || !m->awdl.sync_enabled) + return; + switch (info.subtype) { + case 0: /* availability slot start */ + m->awdl.slot_open = true; + netif_carrier_on(m->awdl.netdev); + netif_wake_queue(m->awdl.netdev); + break; + case 1: /* availability slot end */ + m->awdl.slot_open = false; + netif_stop_queue(m->awdl.netdev); + break; + case 2: + /* Intentionally not decoded: its static layout conflicts with the + * proven normal-RX action report path. + */ + break; + case 3: + case 4: + break; + case 5: /* action-frame TX status */ + case 6: /* sync changed; this host ignores the payload */ + case 7: /* channel switch */ + case 8: /* channel switch complete */ + case 10: /* diagnostic trigger; accounted only */ + break; + case 9: /* conservative suspend/resume handling without payload layout */ + m->awdl.slot_open = false; + netif_stop_queue(m->awdl.netdev); + netif_carrier_off(m->awdl.netdev); + break; + default: + break; + } +} + diff --git a/drivers/net/wireless/mediatek/mt7932/awdl.h b/drivers/net/wireless/mediatek/mt7932/awdl.h new file mode 100644 index 0000000000000..295a7a253df6d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/awdl.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_AWDL_H +#define MT7932_AWDL_H + +#include +#include +#include + +#define MT7932_AWDL_BSS_INDEX 1 +#define MT7932_AWDL_NETWORK_TYPE 5 +#define MT7932_AWDL_BSS_RECORD_LEN 116 +#define MT7932_AWDL_STA_RECORD_LEN 200 +#define MT7932_AWDL_QUEUE_RECORD_LEN 32 +#define MT7932_AWDL_ENCAP_LEN 16 +#define MT7932_AWDL_MAX_PEERS 15 +#define MT7932_AWDL_MAX_SCHEDULE 64 + +struct mt7932; +struct mt7932_event; +struct mt7932_rx_frame; + +struct mt7932_awdl_peer_caps { + u8 phy; + u16 rates; + u8 ht_mcs; + u8 mcs32; + u8 rx_mcs[10]; + u16 rx_highest; + u16 ht_cap; + u16 ht_ext_cap; + u32 tx_bf_cap; + u8 ampdu; + u8 asel; + u32 vht_cap; + u16 vht_rx_mcs; + u16 vht_rx_highest; + u16 vht_tx_mcs; + u16 vht_tx_highest; + u8 he[34]; + u8 nss; + u8 rcpi; +}; + +struct mt7932_awdl_peer { + u8 addr[ETH_ALEN]; + u8 station; + u8 wtbl; + bool live; + struct mt7932_awdl_peer_caps caps; +}; + +struct mt7932_awdl_event_info { + u32 subtype; + const u8 *data; + size_t length; +}; + +struct mt7932_awdl { + struct net_device *netdev; + struct wireless_dev wdev; + struct mt7932_awdl_peer peers[MT7932_AWDL_MAX_PEERS]; + u8 bssid[ETH_ALEN]; + u8 bmc_wtbl; + u8 wmm; + u16 tx_sequence; + bool registered; + bool interface_up; + bool active; + bool sync_enabled; + bool slot_open; + u64 events[11]; + u64 rx_actions; + u64 rx_data; +}; + +int mt7932_awdl_build_queue(u8 out[MT7932_AWDL_QUEUE_RECORD_LEN], + u8 station, bool peer, bool enable); +int mt7932_awdl_build_bss_active(u8 out[12], const u8 *mac, u8 bmc_wtbl, + bool active); +int mt7932_awdl_build_bss(u8 out[MT7932_AWDL_BSS_RECORD_LEN], u8 bmc_wtbl, + u8 wmm, u8 phy, u8 nss); +int mt7932_awdl_build_sta(u8 out[MT7932_AWDL_STA_RECORD_LEN], u8 station, + u8 wtbl, const u8 *mac, + const struct mt7932_awdl_peer_caps *caps, + bool live, bool need_response); +int mt7932_awdl_build_tlv(u8 *out, size_t capacity, u32 tag, + const void *body, size_t body_len); +int mt7932_awdl_build_channel_sequence(u8 out[134], const u8 *channels, + const u8 *classes, size_t count); +int mt7932_awdl_event_parse(const struct mt7932_event *event, + struct mt7932_awdl_event_info *info); +int mt7932_awdl_encap(u8 *out, size_t capacity, const u8 *ethernet, + size_t length, u16 sequence); +int mt7932_awdl_decap(u8 *frame, size_t *length); + +#if IS_ENABLED(CONFIG_MT7932_AWDL) +void mt_awdl_wiphy_setup(struct mt7932 *m); +int mt_awdl_register(struct mt7932 *m); +void mt_awdl_unregister(struct mt7932 *m); +int mt_awdl_activate(struct mt7932 *m); +void mt_awdl_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame); +void mt_awdl_event(struct mt7932 *m, const struct mt7932_event *event); +int mt_awdl_add_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_parameters *params); +int mt_awdl_del_station(struct wiphy *wiphy, struct wireless_dev *wdev, + struct station_del_parameters *params); +int mt_awdl_change_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_parameters *params); +#else +static inline void mt_awdl_wiphy_setup(struct mt7932 *m) {} +static inline int mt_awdl_register(struct mt7932 *m) { return 0; } +static inline void mt_awdl_unregister(struct mt7932 *m) {} +static inline int mt_awdl_activate(struct mt7932 *m) { return 0; } +static inline void mt_awdl_receive(struct mt7932 *m, + const struct mt7932_rx_frame *frame) {} +static inline void mt_awdl_event(struct mt7932 *m, + const struct mt7932_event *event) {} +#endif + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index b749e18a93380..00e5430b50352 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -33,6 +33,12 @@ void mt_data_clean_locked(struct mt7932 *m) if (READ_ONCE(m->connected) && mt_rf_allowed(m) && mt_bss_tx_allowed(m) && !m->data_live[q->head] && q->queued < q->count - 1) netif_wake_queue(m->netdev); +#if IS_ENABLED(CONFIG_MT7932_AWDL) + if (READ_ONCE(m->awdl.active) && READ_ONCE(m->awdl.sync_enabled) && + READ_ONCE(m->awdl.slot_open) && !m->data_live[q->head] && + q->queued < q->count - 1) + netif_wake_queue(m->awdl.netdev); +#endif } /* diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index cfbdcd9f494d2..600094e8dacf3 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -617,7 +617,9 @@ int mt_register_interface(struct mt7932 *m) init_completion(&m->disconnect_done); init_completion(&m->discovery_done); init_completion(&m->scan_done); - m->peer_alloc = BIT(0) | BIT(19); + m->station_alloc = BIT(0) | BIT(19); + m->wtbl_alloc = BIT(0) | BIT(19); + m->wmm_alloc = BIT(0); INIT_DELAYED_WORK(&m->scan_timeout_work, mt_scan_timeout_work); m->netdev = netdev; ret = register_netdev(netdev); @@ -655,6 +657,11 @@ int mt_enable_scan(struct mt7932 *m) if (ret) return ret; } + if (IS_ENABLED(CONFIG_MT7932_AWDL) && READ_ONCE(m->awdl.interface_up)) { + ret = mt_awdl_activate(m); + if (ret) + return ret; + } /* An RX calibration failure must win over late startup publication. */ spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->cal_state.error) { @@ -696,7 +703,13 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) ieee80211_channel_to_frequency(frame.channel, frame.band)); if (!channel || (channel->flags & IEEE80211_CHAN_DISABLED)) return; + if (IS_ENABLED(CONFIG_MT7932_AWDL) && frame.bss == MT7932_AWDL_BSS_INDEX) { + mt_awdl_receive(m, &frame); + return; + } if (frame.translated) { + if (frame.bss) + return; mt_data_receive(m, &frame); return; } diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index 9a05219172bde..d20f43ccfdece 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -33,6 +33,12 @@ void mt_stop_host(struct mt7932 *m) netif_carrier_off(m->netdev); netif_tx_disable(m->netdev); } +#if IS_ENABLED(CONFIG_MT7932_AWDL) + if (m->awdl.netdev) { + netif_carrier_off(m->awdl.netdev); + netif_tx_disable(m->awdl.netdev); + } +#endif WRITE_ONCE(m->running, false); mt_write(m, W + 0x204, 0); if (m->irq_requested) { diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 538a1f15f0a3f..4f84cb1084a55 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -65,6 +65,12 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) spin_lock(&m->data_lock); netif_carrier_off(m->netdev); netif_stop_queue(m->netdev); +#if IS_ENABLED(CONFIG_MT7932_AWDL) + if (m->awdl.netdev) { + netif_carrier_off(m->awdl.netdev); + netif_stop_queue(m->awdl.netdev); + } +#endif if (data_ready) spin_unlock(&m->data_lock); if (m->connecting) { @@ -95,18 +101,25 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) /* One owned station and firmware handshake; no host key fabrication. */ static int mt_peer_reserve(struct mt7932 *m) { - u8 *ids = (u8 *)&m->peer_ids; - unsigned long allocation = m->peer_alloc; + unsigned long station = m->station_alloc, wtbl = m->wtbl_alloc; unsigned int i, id; - for (i = 0; i < 6; i++) { - id = find_first_zero_bit(&allocation, i < 2 ? 15 : 14); - if (id >= (i < 2 ? 15 : 14)) + for (i = 0; i < ARRAY_SIZE(m->peer_ids.station); i++) { + id = find_first_zero_bit(&station, 15); + if (id >= 15) + return -ENOSPC; + __set_bit(id, &station); + m->peer_ids.station[i] = id; + } + for (i = 0; i < ARRAY_SIZE(m->peer_ids.wtbl); i++) { + id = find_first_zero_bit(&wtbl, 14); + if (id >= 14) return -ENOSPC; - __set_bit(id, &allocation); - ids[i] = id; + __set_bit(id, &wtbl); + m->peer_ids.wtbl[i] = id; } - m->peer_alloc = allocation; + m->station_alloc = station; + m->wtbl_alloc = wtbl; return 0; } @@ -282,11 +295,12 @@ static int mt_association_calibration(struct mt7932 *m) /* command_mutex and response_lock held; all old DMA/readers already retired. */ static void mt_peer_release(struct mt7932 *m) { - const u8 *ids = (const u8 *)&m->peer_ids; unsigned int i; - for (i = 0; i < sizeof(m->peer_ids); i++) - __clear_bit(ids[i], &m->peer_alloc); + for (i = 0; i < ARRAY_SIZE(m->peer_ids.station); i++) + __clear_bit(m->peer_ids.station[i], &m->station_alloc); + for (i = 0; i < ARRAY_SIZE(m->peer_ids.wtbl); i++) + __clear_bit(m->peer_ids.wtbl[i], &m->wtbl_alloc); memset(&m->peer_ids, 0, sizeof(m->peer_ids)); memset(m->duplicates, 0, sizeof(m->duplicates)); memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index acf23bb0d31da..6e55131ce7a77 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -165,6 +165,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) mt_bss_presence(m, &event); goto unlock; } + if (IS_ENABLED(CONFIG_MT7932_AWDL) && event.eid == 0xf7) { + mt_awdl_event(m, &event); + goto unlock; + } if (event.eid == 0xd6) { ret = mt7932_cal_null(&m->cal_state, packet + 36, event.length >= 36 ? event.length - 36 : 0, @@ -1131,6 +1135,7 @@ static void mt_remove(struct pci_dev *pdev) free_netdev(m->netdev); m->netdev = NULL; } + mt_awdl_unregister(m); if (m->wiphy_registered) wiphy_unregister(m->wiphy); if (m->wiphy) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 172369bb3f159..b05ab3bedadda 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -25,6 +25,7 @@ #include "firmware.h" #include "calibration.h" #include "packet.h" +#include "awdl.h" #include "scan.h" #include "association.h" #include "regulatory.h" @@ -142,8 +143,10 @@ struct mt7932 { u8 scan_seq, retired_scan_seq, scan_home_channel; bool scan_finished, scan_aborted, bss_active; u64 beacons; - unsigned long peer_alloc; + /* Station and security-table identities are independent host allocators. */ + unsigned long station_alloc, wtbl_alloc, wmm_alloc; struct mt7932_peer_ids peer_ids; + struct mt7932_awdl awdl; struct cfg80211_bss *connect_bss; struct work_struct connect_work, disconnect_work; struct work_struct power_work; diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index a622861816d49..31d8ac8a1d32f 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -12,7 +12,7 @@ struct mt7932_rx_frame { size_t length; u32 timestamp, rcpi, rxv; u16 frame_control, sequence_control, qos_control, wtbl; - u8 channel, amsdu, sec_mode; + u8 channel, amsdu, sec_mode, bss; enum nl80211_band band; bool translated, group4, signal_valid, rate_valid, unicast; bool cipher_mismatch; @@ -63,6 +63,7 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r return -EPROTO; memset(out, 0, sizeof(*out)); out->translated = !!(w2 & (1U << 13)); + out->bss = w2 & 0x3f; out->channel = (w3 >> 8) & 255; out->wtbl = w1 & 0x3ff; out->unicast = ((w3 >> 16) & 3) == 1; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 7962a5fd95a54..ee93e2106ef0f 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -650,6 +650,11 @@ static const struct cfg80211_ops mt_cfg_ops = { .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, .set_cqm_rssi_config = mt_set_cqm_rssi_config, +#if IS_ENABLED(CONFIG_MT7932_AWDL) + .add_station = mt_awdl_add_station, + .del_station = mt_awdl_del_station, + .change_station = mt_awdl_change_station, +#endif }; int mt_register_regulatory_gate(struct mt7932 *m) @@ -671,6 +676,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) set_wiphy_dev(m->wiphy, &m->pdev->dev); ether_addr_copy(m->wiphy->perm_addr, m->mac); m->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + mt_awdl_wiphy_setup(m); m->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; m->wiphy->cipher_suites = ciphers; m->wiphy->n_cipher_suites = ARRAY_SIZE(ciphers); @@ -784,6 +790,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy_registered = true; mt_policy_debugfs_init(m); ret = mt_register_interface(m); + if (ret) + return ret; + ret = mt_awdl_register(m); if (ret) return ret; WRITE_ONCE(m->interface_registered, true); From 904c734881080a529a6a791bd678e57102d95213 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 09:47:13 -0500 Subject: [PATCH 290/392] tools/asahi: Model the late crashlog work in the OSLog test apple_rtkit_reinit() now cancels the late crashlog read (cancel_delayed_work_sync(&rtk->crashlog_late_work)). The OSLog test extracts that function, so give its mock struct apple_rtkit the field and add a no-op cancel_delayed_work_sync(). Older sources don't use either, so the --ref runs still build. Signed-off-by: Diljit Singh --- tools/asahi/rtkit-oslog-tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 160c163758d54..df50c90350b81 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -41,6 +41,7 @@ def function(source, name): #define spin_unlock_irqrestore(l,f) ((void)(l), (void)(f)) struct apple_rtkit_shmem { size_t size; }; struct mailbox { int rx_lock; void (*rx)(void); }; +struct delayed_work { int pending; }; struct apple_rtkit { void *dev, *wq; struct mailbox *mbox; @@ -53,6 +54,7 @@ def function(source, name): char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; + struct delayed_work crashlog_late_work; }; static unsigned sent, warnings, errors, requests, detachments, flushes, freed, checks; static int send_error, setup_error, start_error; @@ -70,6 +72,7 @@ def function(source, name): static void apple_rtkit_rx(void) {} static void apple_rtkit_detach_rx(struct apple_rtkit *r) { ++detachments; r->shutting_down=true; } static void flush_workqueue(void *w) { (void)w; ++flushes; } +#define cancel_delayed_work_sync(w) ((void)(w), false) static void apple_rtkit_free_buffer(struct apple_rtkit *r, struct apple_rtkit_shmem *b) { (void)r; ++freed; b->size=0; } #define kfree(p) ((void)(p)) From b16f72bce3d9a51bff7176d908835b0faa48c86a Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:23:17 -0800 Subject: [PATCH 291/392] dt-bindings: iommu: apple,sart: Add the T6050 SART The T6050 (M5 Ultra) SART keeps SARTv3's entries but moves the address and size arrays, so it is not compatible with apple,t6000-sart and has no fallback. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/iommu/apple,sart.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index 493e48de06a92..19ffe1bf167f7 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -26,6 +26,9 @@ description: SART3, first seen in M1 Pro/Max, extends both the address space and filter entry size to 42 bit. + SART4, seen on M5 Ultra (T6050), keeps SART3's entries but moves the + address and size arrays, so it has no SART3 fallback. + properties: compatible: oneOf: From f44022cdf8ae8602912e3d8345d38edf6a0ae4a9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:21:36 -0800 Subject: [PATCH 292/392] soc: apple: sart: Add SARTv4 for T6050 The SART that filters the T6050 (M5 Ultra) ANS's DMA keeps SARTv3's entry format (config, page address and size, 4 KiB units) but moves the address and size arrays from 0x40/0x80 to 0x60/0xc0. Read on J775D before Linux started: the config words at 0x00+4i and seven live entries whose addresses at 0x60+4i and sizes at 0xc0+4i name ANS's MMIO and two DRAM ranges. Let the v3 ops carry the array offsets, so v3 and v4 share one implementation, and match apple,t6050-sart with the v4 offsets. There is no v3 fallback compatible: a v3 driver would read the wrong registers. Log the number of entries the boot loader left (protected) at debug level. v2 and v3 SARTs keep their offsets. Testing: on J775D, every boot since protects the seven entries the boot loader left, and ANS's NVMe runs behind it (Linux boots from and writes to its NVMe partition). Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/soc/apple/sart.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 6cedcb867c61b..8270a5a731f2c 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -54,15 +55,19 @@ /* SARTv3 registers */ #define APPLE_SART3_CONFIG(idx) (0x00 + 4 * (idx)) -#define APPLE_SART3_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART3_PADDR 0x40 #define APPLE_SART3_PADDR_SHIFT 12 -#define APPLE_SART3_SIZE(idx) (0x80 + 4 * (idx)) +#define APPLE_SART3_SIZE 0x80 #define APPLE_SART3_SIZE_SHIFT 12 #define APPLE_SART3_SIZE_MAX GENMASK(29, 0) #define APPLE_SART3_FLAGS_ALLOW 0xff +/* SARTv4 (M5): SARTv3's entries, with the address and size arrays moved up */ +#define APPLE_SART4_PADDR 0x60 +#define APPLE_SART4_SIZE 0xc0 + struct apple_sart_ops { void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size); @@ -70,6 +75,9 @@ struct apple_sart_ops { phys_addr_t paddr_shifted, size_t size_shifted); /* This is probably a bitfield but the exact meaning of each bit is unknown. */ unsigned int flags_allow; + /* SARTv3/v4: where the address and size arrays start */ + unsigned int paddr_reg; + unsigned int size_reg; unsigned int size_shift; unsigned int paddr_shift; size_t size_max; @@ -179,8 +187,8 @@ static struct apple_sart_ops sart_ops_v2 = { static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size) { - phys_addr_t paddr_ = readl(sart->regs + APPLE_SART3_PADDR(index)); - size_t size_ = readl(sart->regs + APPLE_SART3_SIZE(index)); + phys_addr_t paddr_ = readl(sart->regs + sart->ops->paddr_reg + 4 * index); + size_t size_ = readl(sart->regs + sart->ops->size_reg + 4 * index); *flags = readl(sart->regs + APPLE_SART3_CONFIG(index)); *size = size_ << APPLE_SART3_SIZE_SHIFT; @@ -190,8 +198,8 @@ static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, static void sart3_set_entry(struct apple_sart *sart, int index, u8 flags, phys_addr_t paddr_shifted, size_t size_shifted) { - writel(paddr_shifted, sart->regs + APPLE_SART3_PADDR(index)); - writel(size_shifted, sart->regs + APPLE_SART3_SIZE(index)); + writel(paddr_shifted, sart->regs + sart->ops->paddr_reg + 4 * index); + writel(size_shifted, sart->regs + sart->ops->size_reg + 4 * index); writel(flags, sart->regs + APPLE_SART3_CONFIG(index)); } @@ -199,6 +207,19 @@ static struct apple_sart_ops sart_ops_v3 = { .get_entry = sart3_get_entry, .set_entry = sart3_set_entry, .flags_allow = APPLE_SART3_FLAGS_ALLOW, + .paddr_reg = APPLE_SART3_PADDR, + .size_reg = APPLE_SART3_SIZE, + .size_shift = APPLE_SART3_SIZE_SHIFT, + .paddr_shift = APPLE_SART3_PADDR_SHIFT, + .size_max = APPLE_SART3_SIZE_MAX, +}; + +static struct apple_sart_ops sart_ops_v4 = { + .get_entry = sart3_get_entry, + .set_entry = sart3_set_entry, + .flags_allow = APPLE_SART3_FLAGS_ALLOW, + .paddr_reg = APPLE_SART4_PADDR, + .size_reg = APPLE_SART4_SIZE, .size_shift = APPLE_SART3_SIZE_SHIFT, .paddr_shift = APPLE_SART3_PADDR_SHIFT, .size_max = APPLE_SART3_SIZE_MAX, @@ -236,6 +257,7 @@ static int apple_sart_probe(struct platform_device *pdev) i, flags, &paddr, size); set_bit(i, &sart->protected_entries); } + dev_dbg(dev, "%lu bootloader entries protected\n", hweight_long(sart->protected_entries)); platform_set_drvdata(pdev, sart); return 0; @@ -409,6 +431,10 @@ static void apple_sart_shutdown(struct platform_device *pdev) } static const struct of_device_id apple_sart_of_match[] = { + { + .compatible = "apple,t6050-sart", + .data = &sart_ops_v4, + }, { .compatible = "apple,t6000-sart", .data = &sart_ops_v3, From 1bcd370f7577791426029de73555c243e170f920 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:23:05 -0800 Subject: [PATCH 293/392] nvme: apple: Add the T6050 ANS on bare metal Add apple,t6050-nvme-ans2 for the M5 Ultra's ANS when Linux runs on the hardware. Its layout is T6040's, as the ADT and the registers read before Linux started show: a 64 KiB "nvme" block, a separate NVMMU and linear submission queues. The controller reports 254 queue entries (CAP.MQES 0xfd); the profile uses T6040's 64. FUA writes are not shown durable here, so the profile flushes instead (NVME_QUIRK_BROKEN_FUA). The boot loader leaves ANS's firmware running and the controller shut down (CC.EN 0, CSTS.RDY 0). The driver adopts that session, and, like every post-M4 ANS without cold_reset, never resets it through PMGR, so the profile sets reset_optional, a hardware-profile flag that takes the controller reset as optional, and the binding requires no reset for it. Every other profile still requires its reset. Testing: on J775D, every boot since 2026-10-30 adopts ANS's running firmware and enables the controller with this profile; Linux runs from its NVMe root partition (ext4) through daily builds and reboots without an NVMe error. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/nvme/host/apple.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 732d7b63be3d4..2164b13f4e787 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -210,6 +210,8 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; + /* The ANS is never reset through PMGR here, so no reset is named. */ + bool reset_optional; u32 max_queue_depth; unsigned long quirks; }; @@ -2407,7 +2409,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto out_detach_genpd; } - anv->reset = devm_reset_control_array_get_exclusive(anv->dev); + anv->reset = anv->hw->reset_optional ? + devm_reset_control_array_get_optional_exclusive(anv->dev) : + devm_reset_control_array_get_exclusive(anv->dev); if (IS_ERR(anv->reset)) { ret = dev_err_probe(dev, PTR_ERR(anv->reset), "Failed to get reset control"); @@ -2776,8 +2780,26 @@ static const struct apple_nvme_hw apple_nvme_t6040_hw = { .quirks = NVME_QUIRK_NO_APST, }; +/* + * T6050 (M5 Ultra) ANS on bare metal: T6040's split layout (a 64 KiB "nvme" + * block, a separate NVMMU, linear submission queues), 64 queue entries. Like + * every post-M4 ANS without cold_reset it is never reset through PMGR, so no + * reset is named. + */ +static const struct apple_nvme_hw apple_nvme_t6050_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .lsq_in_nvmmu = true, + .reset_optional = true, + .max_queue_depth = 64, + /* FUA is not shown durable on this ANS: flush instead. */ + .quirks = NVME_QUIRK_NO_APST | NVME_QUIRK_BROKEN_FUA, +}; + static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t6041-nvme-ans2", .data = &apple_nvme_t6040_hw }, + { .compatible = "apple,t6050-nvme-ans2", .data = &apple_nvme_t6050_hw }, { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, From e6e2873d43060788fe591a3e75b92e80c2bb3b79 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:23:19 -0800 Subject: [PATCH 294/392] dt-bindings: nvme: apple: Allow the boot loader's RTKit handoff on T6050 The boot loader leaves the T6050 ANS's firmware running with the controller disabled and, as on T8132 and T8140, adds apple,rtkit-handoff to the node so the kernel adopts that RTKit session instead of starting the co-processor again. The T6050 profile relies on that adoption. Allow the property on apple,t6050-nvme-ans2. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 3d08734f0ee74..7f289a02d4b21 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -117,6 +117,7 @@ allOf: not: contains: enum: + - apple,t6050-nvme-ans2 - apple,t8132-nvme-ans2 - apple,t8140-nvme-ans2 then: From 4954d7062a3e96b36013c04bae6501a91d28340b Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:18 -0400 Subject: [PATCH 295/392] mfd: macsmc: Support T6050 power and pcIO consumers Add the T6050 SMC consumer profile, described pcIO GPIO outputs, power-supply identity sizing, and charging policy. Preserve the newer shared SRAM accessor and metadata serialization. Source snapshot: t6050-gpu ee68abb105200e99636a5edfc647a5cfc1d89954 Integration base: 9cbdd0734d9e186c5f5f5949a21b909d10c2571b Signed-off-by: Dj Signed-off-by: Erik Bourget --- .../devicetree/bindings/mfd/apple,smc.yaml | 2 +- drivers/gpio/gpio-macsmc.c | 77 ++++++++++++++++++- drivers/mfd/macsmc.c | 35 +++++++-- drivers/power/supply/macsmc-power.c | 41 ++++++++-- 4 files changed, 141 insertions(+), 14 deletions(-) diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index 050ac9c6388ca..c35e7d0e10faa 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) AND GPL-2.0-only %YAML 1.2 --- $id: http://devicetree.org/schemas/mfd/apple,smc.yaml# diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 5f43b79b6013c..6a26f900bba28 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple SMC GPIO driver * Copyright The Asahi Linux Contributors @@ -16,6 +16,7 @@ #include #include #include +#include #define MAX_GPIO 64 @@ -103,6 +104,7 @@ struct macsmc_gpio { struct apple_smc *smc; struct gpio_chip gc; + u64 *output_commands; int first_index; smc_key base_key; @@ -383,6 +385,76 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, return 0; } +/* Each DT template identifies one output; its low 16 bits carry the value. */ +static int macsmc_pcio_set(struct gpio_chip *gc, unsigned int offset, int value) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + u8 command[8]; + int ret; + + put_unaligned_le64(smcgp->output_commands[offset] | !!value, command); + ret = apple_smc_write(smcgp->smc, SMC_KEY(pcIO), command, sizeof(command)); + /* SMC writes return status, not a byte count. */ + return ret < 0 ? ret : 0; +} + +static int macsmc_pcio_get(struct gpio_chip *gc, unsigned int offset) +{ + /* The native output command has no established input/readback operation. */ + return -EOPNOTSUPP; +} + +static int macsmc_pcio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + return GPIO_LINE_DIRECTION_OUT; +} + +static int macsmc_pcio_probe(struct macsmc_gpio *smcgp) +{ + struct device *dev = smcgp->dev; + struct apple_smc_key_info info; + int count, ret; + + /* A board that uses none of the outputs leaves the list out. */ + if (!of_property_present(dev->of_node, "apple,output-commands")) + return -ENODEV; + + ret = apple_smc_get_key_info(smcgp->smc, SMC_KEY(pcIO), &info); + if (ret < 0) + return ret; + if (info.size != 8 || !(info.flags & APPLE_SMC_WRITABLE)) + return dev_err_probe(dev, -EINVAL, "pcIO is not an eight-byte output key\n"); + count = of_property_count_u64_elems(dev->of_node, "apple,output-commands"); + if (count <= 0 || count > MAX_GPIO) + return -EINVAL; + smcgp->output_commands = devm_kcalloc(dev, count, sizeof(u64), GFP_KERNEL); + if (!smcgp->output_commands) + return -ENOMEM; + ret = of_property_read_u64_array(dev->of_node, "apple,output-commands", + smcgp->output_commands, count); + if (ret) + return ret; + for (int i = 0; i < count; i++) { + if (smcgp->output_commands[i] & 0xffff) + return -EINVAL; + for (int j = 0; j < i; j++) + if (smcgp->output_commands[i] == smcgp->output_commands[j]) + return -EINVAL; + } + + smcgp->gc.label = "macsmc-pcio-output"; + smcgp->gc.owner = THIS_MODULE; + smcgp->gc.get = macsmc_pcio_get; + smcgp->gc.set = macsmc_pcio_set; + smcgp->gc.direction_output = macsmc_pcio_set; + smcgp->gc.get_direction = macsmc_pcio_get_direction; + smcgp->gc.can_sleep = true; + smcgp->gc.ngpio = count; + smcgp->gc.base = -1; + smcgp->gc.parent = dev; + return devm_gpiochip_add_data(dev, &smcgp->gc, smcgp); +} + struct macsmc_gpio_of_match_data { smc_key base_key; }; @@ -449,6 +521,8 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; + if (device_is_compatible(&pdev->dev, "apple,smc-pcio-gpio")) + return macsmc_pcio_probe(smcgp); smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); smcgp->j700_pcio = of_machine_is_compatible("apple,j700") && device_is_compatible(&pdev->dev, "apple,smc-gpio"); @@ -498,6 +572,7 @@ static const struct macsmc_gpio_of_match_data macsmc_gpio_low_data = { }; static const struct of_device_id macsmc_gpio_of_table[] = { + { .compatible = "apple,smc-pcio-gpio" }, { .compatible = "apple,smc-gpio", .data = &macsmc_gpio_up_data, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 546e5cdbb72ad..80f965194bc3a 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple SMC (System Management Controller) MFD driver * @@ -59,6 +59,16 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; +/* This profile exposes notifications and the explicitly described outputs. */ +static const struct mfd_cell apple_smc_t6050_devs[] = { + MFD_CELL_NAME("macsmc-input"), + MFD_CELL_NAME("macsmc-power"), + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), +}; + +static const struct mfd_cell apple_smc_pcio_dev = + MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"); + /* The second GPIO bank ('gp' keys) only exists on some SMCs. */ static const struct mfd_cell apple_smc_low_gpio_dev = MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); @@ -250,8 +260,8 @@ int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_ u8 key_info[6]; int ret; + /* Keep the shared reply buffer stable until its metadata is copied. */ guard(mutex)(&smc->mutex); - ret = apple_smc_cmd_locked(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); if (ret >= 0 && info) { apple_smc_read_sram(smc, key_info, sizeof(key_info)); @@ -559,12 +569,26 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - apple_smc_devs, ARRAY_SIZE(apple_smc_devs), - NULL, 0, NULL); + if (of_device_is_compatible(dev->of_node, "apple,t6050-smc")) + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_t6050_devs, + ARRAY_SIZE(apple_smc_t6050_devs), NULL, 0, NULL); + else + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_devs, ARRAY_SIZE(apple_smc_devs), + NULL, 0, NULL); if (ret) return dev_err_probe(smc->dev, ret, "Failed to register sub-devices"); + np = of_get_compatible_child(dev->of_node, "apple,smc-pcio-gpio"); + if (np) { + of_node_put(np); + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + &apple_smc_pcio_dev, 1, NULL, 0, NULL); + if (ret) + return dev_err_probe(dev, ret, "Failed to register pcIO outputs\n"); + } + /* * Register the second GPIO bank only where the device tree describes * it: the MFD core would otherwise create an unbound device and warn @@ -585,6 +609,7 @@ static int apple_smc_probe(struct platform_device *pdev) static const struct of_device_id apple_smc_of_match[] = { { .compatible = "apple,t8152-smc" }, + { .compatible = "apple,t6050-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 1bc23703654a4..1e306c7bb3f6c 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -960,10 +960,34 @@ static int macsmc_power_event(struct notifier_block *nb, unsigned long event, vo return NOTIFY_DONE; } +static int macsmc_read_identity(struct apple_smc *smc, smc_key key, + char *out, size_t capacity) +{ + struct apple_smc_key_info info; + int ret; + + if (!capacity) + return -EINVAL; + out[0] = '\0'; + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!info.size || info.size >= capacity || !(info.flags & APPLE_SMC_READABLE)) + return -EINVAL; + /* SMC read requests must describe this key, not the destination buffer. */ + ret = apple_smc_read(smc, key, out, info.size); + if (ret != info.size) { + out[0] = '\0'; + return ret < 0 ? ret : -EIO; + } + out[info.size] = '\0'; + return 0; +} + static int macsmc_power_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct apple_smc *smc; struct power_supply_config psy_cfg = {}; struct apple_smc_key_info info; struct macsmc_power *power; @@ -976,6 +1000,9 @@ static int macsmc_power_probe(struct platform_device *pdev) enum power_supply_property *props; size_t nprops; + if (!dev->parent) + return dev_err_probe(dev, -ENODEV, "Missing SMC parent\n"); + smc = dev_get_drvdata(dev->parent); if (!smc) return -ENODEV; @@ -1121,12 +1148,12 @@ static int macsmc_power_probe(struct platform_device *pdev) power->batt_desc.num_properties = nprops; /* Fetch identity strings */ - apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, - sizeof(power->model_name) - 1); - apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, - sizeof(power->serial_number) - 1); - apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, - sizeof(power->mfg_date) - 1); + macsmc_read_identity(smc, SMC_KEY(BMDN), power->model_name, + sizeof(power->model_name)); + macsmc_read_identity(smc, SMC_KEY(BMSN), power->serial_number, + sizeof(power->serial_number)); + macsmc_read_identity(smc, SMC_KEY(BMDT), power->mfg_date, + sizeof(power->mfg_date)); apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells); power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells; From 853ccc6231b615d39ea101dd436f65c148bdf195 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 296/392] mfd: macsmc: Register T6050's reboot and RTC cells where the DT has them The T6050 profile (Aurora's t6050/smc) registers input, power and hwmon only, so a T6050 kernel that owns the machine (J775D on bare metal) had no SMC restart or power-off and no RTC: poweroff halted Linux and left the machine on. Add macsmc-reboot and macsmc-rtc to the T6050 profile as cells that are registered only where the device tree has their child node, the rule the pcIO outputs and the second GPIO bank already follow; those two blocks become one table and one helper. A T6050 device tree without reboot or rtc children registers exactly what it did, and other SMCs are unchanged (same cells in the same order; the registration failure messages now name the cell). Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/mfd/macsmc.c | 80 +++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 80f965194bc3a..927ddaa00e0df 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -66,12 +66,47 @@ static const struct mfd_cell apple_smc_t6050_devs[] = { MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), }; -static const struct mfd_cell apple_smc_pcio_dev = - MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"); +/* + * Cells registered only where the device tree describes them: the MFD core would otherwise + * create an unbound device and warn that it found no node on every SMC without one. The pcIO + * outputs and the second GPIO bank ('gp' keys) only exist on some SMCs. + */ +static const struct mfd_cell apple_smc_described_devs[] = { + MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-pcio-gpio"), + MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"), +}; -/* The second GPIO bank ('gp' keys) only exists on some SMCs. */ -static const struct mfd_cell apple_smc_low_gpio_dev = - MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +/* + * T6050's reboot/power-off and RTC, for a kernel that owns the machine (J775D, booted bare + * metal). Registered only where the device tree describes them, so a T6050 device tree without + * them registers what it did before. + */ +static const struct mfd_cell apple_smc_t6050_described_devs[] = { + MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), + MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), +}; + +static int apple_smc_add_described_devices(struct apple_smc *smc, const struct mfd_cell *cells, + size_t count) +{ + struct device_node *np; + size_t i; + int ret; + + for (i = 0; i < count; i++) { + np = of_get_compatible_child(smc->dev->of_node, cells[i].of_compatible); + if (!np) + continue; + of_node_put(np); + ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, &cells[i], 1, + NULL, 0, NULL); + if (ret) + return dev_err_probe(smc->dev, ret, "Failed to register %s\n", + cells[i].name); + } + + return 0; +} /* T8152 currently exposes read-only telemetry; firmware owns fan control. */ static const struct mfd_cell apple_smc_telemetry_devs[] = { @@ -493,9 +528,9 @@ static void apple_smc_disable_notifications(void *data) static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np; struct apple_smc *smc; __be32 count = 0; + bool t6050; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -569,7 +604,8 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; - if (of_device_is_compatible(dev->of_node, "apple,t6050-smc")) + t6050 = of_device_is_compatible(dev->of_node, "apple,t6050-smc"); + if (t6050) ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, apple_smc_t6050_devs, ARRAY_SIZE(apple_smc_t6050_devs), NULL, 0, NULL); @@ -580,31 +616,13 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return dev_err_probe(smc->dev, ret, "Failed to register sub-devices"); - np = of_get_compatible_child(dev->of_node, "apple,smc-pcio-gpio"); - if (np) { - of_node_put(np); - ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - &apple_smc_pcio_dev, 1, NULL, 0, NULL); - if (ret) - return dev_err_probe(dev, ret, "Failed to register pcIO outputs\n"); - } - - /* - * Register the second GPIO bank only where the device tree describes - * it: the MFD core would otherwise create an unbound device and warn - * that it found no node on every SMC without one. - */ - np = of_get_compatible_child(dev->of_node, "apple,smc-low-gpio"); - if (np) { - of_node_put(np); - ret = devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, - &apple_smc_low_gpio_dev, 1, NULL, 0, NULL); - if (ret) - return dev_err_probe(smc->dev, ret, - "Failed to register the second GPIO bank"); - } + ret = apple_smc_add_described_devices(smc, apple_smc_described_devs, + ARRAY_SIZE(apple_smc_described_devs)); + if (ret || !t6050) + return ret; - return 0; + return apple_smc_add_described_devices(smc, apple_smc_t6050_described_devs, + ARRAY_SIZE(apple_smc_t6050_described_devs)); } static const struct of_device_id apple_smc_of_match[] = { From ad0f93637543787d6156515baf9a38422add5c29 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 297/392] hwmon: macsmc: Let discovered temperatures join the SoC die zone apple,soc-die-keys adds a "SoC die " channel per listed key for the macsmc_soc_die zone, and apple,discover-temperatures refuses to run once any temperature channel exists, so a machine that discovers its temperatures (T8142, J775D) could not have the zone: the probe failed with -EINVAL. With discovery, a listed key now adds no channel; the channel discovery creates for it (labelled with the key) is marked for the zone instead. Machines that set only one of the two properties are unchanged. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- .../bindings/hwmon/apple,smc-hwmon.yaml | 4 +++- Documentation/hwmon/macsmc-hwmon.rst | 2 ++ drivers/hwmon/macsmc-hwmon.c | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index a4e4dfd3c0886..5ba2a01b323e5 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -62,7 +62,9 @@ properties: follow the temperature-* children in the listed order and are labelled "SoC die ". Keys absent from the running firmware or with an unsupported format are skipped. These channels do not change the indices - used by thermal-sensors references to temperature-* children. + used by thermal-sensors references to temperature-* children. With + apple,discover-temperatures, a listed key adds no channel; its + discovered channel (labelled with the key) supplies the zone. apple,thermal-pressure-key: $ref: /schemas/types.yaml#/definitions/string diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index b0c78c3412088..23baece30154a 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -43,6 +43,8 @@ Only readable four-byte floating-point keys are used. When thermal support is enabled, their hottest reading supplies the ``macsmc_soc_die`` thermal zone. A failed or implausible reading fails the zone read rather than reporting a partial result. Keys absent from the firmware are skipped. +With ``apple,discover-temperatures`` the listed keys add no channels: the +discovered channel of each listed key, labelled with the key, joins the zone. The optional ``apple,thermal-pressure-key`` on the same node describes a readable signed 32-bit firmware thermal pressure level. Neither feature needs diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index d0206489fa372..ddf287a190efa 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -901,6 +901,9 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) if (!smc->key_count || smc->key_count > U16_MAX) return -EINVAL; + ret = of_property_count_strings(hwmon->dev->of_node, "apple,soc-die-keys"); + hwmon->die_listed = ret > 0 ? ret : 0; + hwmon->temp.sensors = devm_kcalloc(hwmon->dev, smc->key_count, sizeof(*sensor), GFP_KERNEL); if (!hwmon->temp.sensors) @@ -934,6 +937,12 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) /* Key identity only: do not invent locations or calibration. */ snprintf(sensor->label, sizeof(sensor->label), "%c%c%c%c", key >> 24, (key >> 16) & 0xff, (key >> 8) & 0xff, key & 0xff); + /* A listed SoC die key keeps its discovered channel and joins the die zone. */ + if (of_property_match_string(hwmon->dev->of_node, "apple,soc-die-keys", + sensor->label) >= 0) { + sensor->die = true; + hwmon->die_count++; + } } return 0; @@ -1117,7 +1126,11 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { n_temperature++; } - n_temperature += n_die_keys; + /* With discovery, listed die keys mark discovered channels instead (see there). */ + if (!of_property_read_bool(hwmon_node, "apple,discover-temperatures")) + n_temperature += n_die_keys; + else + n_die_keys = 0; if (n_temperature) { hwmon->temp.sensors = devm_kcalloc(hwmon->dev, n_temperature, From 4d11bfdc0ef5e960fd4712854bdc34fbdb2ee955 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 01:35:36 -0800 Subject: [PATCH 298/392] power: reset: macsmc-reboot: Add ac_power_mode, with power-on at connect Expose the PMU's power-on policy byte as the ac_power_mode attribute Asahi's downstream driver had, with a third mode. The byte has two policy bits: bit 0 keeps the machine off when power returns after a loss, bit 5 keeps it off when power is connected. Asahi's driver wrote the whole byte (0x02 restore, 0x03 off), which clears bit 5: on machines whose PMU powers on at connect that write means "always", the opposite of "off". Change only the two bits and leave the rest to the PMU: off bit 0 and bit 5 set restore bit 5 set, bit 0 clear always bit 5 clear (bit 0 set) The pm_setting cell is optional; the attribute appears only with it. The binding takes it as a fifth cell, as the t600x/t602x device trees already list it. Based on Asahi's ac_power_mode (marcan). Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- .../ABI/testing/sysfs-driver-macsmc-reboot | 16 +++ drivers/power/reset/macsmc-reboot.c | 118 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-driver-macsmc-reboot diff --git a/Documentation/ABI/testing/sysfs-driver-macsmc-reboot b/Documentation/ABI/testing/sysfs-driver-macsmc-reboot new file mode 100644 index 0000000000000..8a485592fedba --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-macsmc-reboot @@ -0,0 +1,16 @@ +What: /sys/bus/platform/drivers/macsmc-reboot//ac_power_mode +Date: November 2026 +KernelVersion: 7.4 +Description: + What the machine does when external power arrives, as kept in + the PMU (present only when the device tree gives the reboot + node a pm_setting cell). Reading lists the modes with the + current one in brackets; writing a mode name sets it: + + off: + stay off + restore: + power on after a power loss if the machine was on + always: + power on whenever power is connected (the 2024 and later + desktops; other machines may ignore it) diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index 7688cfd4ed60d..a2e1422e05bd8 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -8,10 +8,12 @@ #include #include #include +#include #include #include #include #include +#include struct macsmc_reboot_nvmem { struct nvmem_cell *shutdown_flag; @@ -33,10 +35,39 @@ enum boot_stage { BOOT_STAGE_KERNEL_STARTED = 0x30, /* Normal OS booting */ }; +/* + * The PMU's power-on policy byte: bit 0 set keeps the machine off when power returns after a + * loss, bit 5 set keeps it off when power is connected (bit 5 clear powers it on at connect + * whatever bit 0 says). Changes are read-modify-write: the other bits are the PMU's. + */ +#define PM_SETTING_NO_RESTORE BIT(0) +#define PM_SETTING_NO_POWER_ON_CONNECT BIT(5) +#define PM_SETTING_AC_POWER_MASK (PM_SETTING_NO_RESTORE | PM_SETTING_NO_POWER_ON_CONNECT) + +enum ac_power_mode { + AC_POWER_OFF, /* stay off */ + AC_POWER_RESTORE, /* power on after a power loss if it was on */ + AC_POWER_ALWAYS, /* power on whenever power is connected */ +}; + +static const char * const ac_power_modes[] = { + [AC_POWER_OFF] = "off", + [AC_POWER_RESTORE] = "restore", + [AC_POWER_ALWAYS] = "always", +}; + +static const u8 ac_power_mode_bits[] = { + [AC_POWER_OFF] = PM_SETTING_NO_RESTORE | PM_SETTING_NO_POWER_ON_CONNECT, + [AC_POWER_RESTORE] = PM_SETTING_NO_POWER_ON_CONNECT, + [AC_POWER_ALWAYS] = PM_SETTING_NO_RESTORE, +}; + struct macsmc_reboot { struct device *dev; struct apple_smc *smc; struct notifier_block reboot_notify; + struct nvmem_cell *pm_setting; + struct mutex pm_setting_lock; union { struct macsmc_reboot_nvmem nvm; @@ -198,6 +229,80 @@ static void macsmc_power_init_error_counts(struct macsmc_reboot *reboot) dev_err(reboot->dev, "Failed to reset boot_error_count\n"); } +static enum ac_power_mode macsmc_ac_power_mode(u8 val) +{ + if (!(val & PM_SETTING_NO_POWER_ON_CONNECT)) + return AC_POWER_ALWAYS; + if (val & PM_SETTING_NO_RESTORE) + return AC_POWER_OFF; + return AC_POWER_RESTORE; +} + +static ssize_t ac_power_mode_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + enum ac_power_mode mode; + int val, i, len = 0; + + val = nvmem_cell_get_u8(reboot->pm_setting); + if (val < 0) + return val; + mode = macsmc_ac_power_mode(val); + + for (i = 0; i < ARRAY_SIZE(ac_power_modes); i++) + len += sysfs_emit_at(buf, len, i == mode ? "[%s] " : "%s ", ac_power_modes[i]); + buf[len - 1] = '\n'; + + return len; +} + +static ssize_t ac_power_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t n) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + int mode, val, ret; + u8 new; + + mode = sysfs_match_string(ac_power_modes, buf); + if (mode < 0) + return mode; + + mutex_lock(&reboot->pm_setting_lock); + val = nvmem_cell_get_u8(reboot->pm_setting); + if (val < 0) { + ret = val; + goto out; + } + new = (val & ~PM_SETTING_AC_POWER_MASK) | ac_power_mode_bits[mode]; + ret = new == val ? 0 : nvmem_cell_set_u8(reboot->pm_setting, new); + if (!ret && new != val) + dev_info(dev, "PM setting %02x => %02x (ac_power_mode %s)\n", val, new, + ac_power_modes[mode]); +out: + mutex_unlock(&reboot->pm_setting_lock); + + return ret < 0 ? ret : n; +} +static DEVICE_ATTR_RW(ac_power_mode); + +static struct attribute *macsmc_reboot_attrs[] = { + &dev_attr_ac_power_mode.attr, + NULL +}; + +static umode_t macsmc_reboot_attr_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(kobj_to_dev(kobj)); + + return reboot && reboot->pm_setting ? attr->mode : 0; +} + +static const struct attribute_group macsmc_reboot_group = { + .attrs = macsmc_reboot_attrs, + .is_visible = macsmc_reboot_attr_visible, +}; +__ATTRIBUTE_GROUPS(macsmc_reboot); + static int macsmc_reboot_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -237,6 +342,18 @@ static int macsmc_reboot_probe(struct platform_device *pdev) reboot->nvm_cells[i] = cell; } + /* Optional: the power-on policy byte, for ac_power_mode */ + mutex_init(&reboot->pm_setting_lock); + reboot->pm_setting = devm_nvmem_cell_get(&pdev->dev, "pm_setting"); + if (IS_ERR(reboot->pm_setting)) { + ret = PTR_ERR(reboot->pm_setting); + if (ret == -EPROBE_DEFER) + return ret; + if (ret != -ENOENT) + dev_warn(&pdev->dev, "Missing NVMEM cell pm_setting (%d)\n", ret); + reboot->pm_setting = NULL; + } + /* Set the boot_stage to indicate we're running the OS kernel */ if (reboot->nvm.boot_stage && nvmem_cell_set_u8(reboot->nvm.boot_stage, BOOT_STAGE_KERNEL_STARTED) < 0) @@ -287,6 +404,7 @@ static struct platform_driver macsmc_reboot_driver = { .driver = { .name = "macsmc-reboot", .of_match_table = macsmc_reboot_of_table, + .dev_groups = macsmc_reboot_groups, }, .probe = macsmc_reboot_probe, }; From 0f2bdecdbfd1a9f6562ff367cb2d728e535fcd9e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 09:50:46 -0500 Subject: [PATCH 299/392] dt-bindings: phy: Drop the duplicate T6050 combo PHY schema integration/aurora already describes apple,t6050-dp-phy and apple,t6050-combo-phy in apple,combo-phy.yaml, with matrix-typed tunables, the CIO tunables and the optional "usb" domain. The copy in apple,t6050-dp-phy.yaml claims the same compatibles, so dt-validate reports a duplicate schema, picks up its uint32-array types and its required tunables, and every T6050 combo PHY node that the boot loader fills in (J714s, J716s, J775c/d, J873s, K114/K116) fails validation. Remove the copy, as erikwb's restacked #1268 does, so each compatible has one schema. Signed-off-by: Diljit Singh --- .../bindings/phy/apple,t6050-dp-phy.yaml | 257 ------------------ 1 file changed, 257 deletions(-) delete mode 100644 Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml diff --git a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml b/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml deleted file mode 100644 index 3b22c3f19ee9b..0000000000000 --- a/Documentation/devicetree/bindings/phy/apple,t6050-dp-phy.yaml +++ /dev/null @@ -1,257 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/phy/apple,t6050-dp-phy.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Apple T6050 DisplayPort and Type-C combo PHY - -maintainers: - - Dj - -description: | - Fixed four-lane DisplayPort PHY for HDMI bridges, or one shared owner of - Type-C USB2, USB3 and DisplayPort alternate mode. DisplayPort supports RBR - through HBR3 without spread-spectrum clocking. The combo provider serializes - controller shutdown, PIPE handoff, orientation and alternate-mode changes. - Register resources, power domains and board tunables come from the producer. - -properties: - compatible: - enum: - - apple,t6050-dp-phy - - apple,t6050-combo-phy - - reg: - minItems: 6 - maxItems: 11 - - reg-names: - minItems: 6 - maxItems: 11 - - '#phy-cells': - enum: [0, 1] - - '#reset-cells': - const: 1 - - apple,port-index: - $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 3, 4, 5] - description: Type-C port; the debug port (0) is excluded - - apple,guest-port-reserved: - type: boolean - description: The boot producer has excluded this port from host ownership - - mode-switch: true - orientation-switch: true - - mboxes: - maxItems: 1 - - port: - $ref: /schemas/graph.yaml#/properties/port - - apple,tunable-aus40cmn-shm: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples for the common analog bank - - apple,tunable-cio4pll: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples for the CIO PLL bank - - apple,tunable-usb2-default: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples for the USB2 PHY bank - - apple,tunable-usb2-host: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples for the USB2 host bank - - apple,tunable-pipe-default: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples for the PIPE handler - - power-domains: - description: - A combo PHY may name the USB controller's domain fourth ("usb"). Its - USB2 PHY and digital banks are in that domain, and the PHY then holds it - while it runs, also on a port with no USB controller bound. - minItems: 3 - maxItems: 4 - - power-domain-names: - minItems: 3 - items: - - const: usb-aon - - const: common - - const: wrapper - - const: usb - - apple,tunable-axi2af: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-fabric: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-common-cfg: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-lioa: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-lane0-rx-clock: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-lane1-rx-clock: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-lane0-rx-eq: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-lane1-rx-eq: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-auspll: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: Offset, mask and value tuples within the named bank - - apple,tunable-uc-regs-cio: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: - Offset, mask and value tuples for the PHY firmware's registers in CIO - (USB4/Thunderbolt) mode. Optional; an absent CIO tunable is skipped. - - apple,tunable-lane0-rx-top-cio: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: As apple,tunable-uc-regs-cio, for lane 0's receiver top bank - - apple,tunable-lane1-rx-top-cio: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: As apple,tunable-uc-regs-cio, for lane 1's receiver top bank - - apple,tunable-lane0-rx-eq-cio: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: As apple,tunable-uc-regs-cio, for lane 0's receiver equalizer - - apple,tunable-lane1-rx-eq-cio: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: As apple,tunable-uc-regs-cio, for lane 1's receiver equalizer - -patternProperties: - '^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$': - $ref: /schemas/types.yaml#/definitions/uint32-array - minItems: 16 - maxItems: 16 - description: Drive presets indexed by voltage swing times four plus pre-emphasis - -required: - - compatible - - reg - - reg-names - - '#phy-cells' - - power-domains - - power-domain-names - - apple,tunable-axi2af - - apple,tunable-fabric - - apple,tunable-common-cfg - - apple,tunable-lioa - - apple,tunable-lane0-rx-clock - - apple,tunable-lane1-rx-clock - - apple,tunable-lane0-rx-eq - - apple,tunable-lane1-rx-eq - - apple,tunable-auspll - -allOf: - - if: - properties: - compatible: - const: apple,t6050-combo-phy - then: - properties: - '#phy-cells': - const: 1 - reg: - minItems: 11 - maxItems: 11 - reg-names: - items: - - const: core - - const: usb2phy - - const: usb2-digital - - const: reset - - const: fabric - - const: axi2af - - const: lioa - - const: common-cfg - - const: usb-control - - const: firmware-data - - const: pipehandler - required: - - '#reset-cells' - - apple,port-index - - apple,guest-port-reserved - - mode-switch - - orientation-switch - - mboxes - - apple,tunable-aus40cmn-shm - - apple,tunable-cio4pll - - apple,tunable-usb2-default - - apple,tunable-usb2-host - - apple,tunable-pipe-default - else: - properties: - '#phy-cells': - const: 0 - power-domains: - maxItems: 3 - power-domain-names: - maxItems: 3 - if: - required: - - mode-switch - then: - properties: - reg: - minItems: 8 - maxItems: 8 - reg-names: - items: - - const: core - - const: reset - - const: fabric - - const: axi2af - - const: lioa - - const: common-cfg - - const: usb-control - - const: firmware-data - required: - - orientation-switch - - mboxes - - apple,tunable-aus40cmn-shm - - apple,tunable-cio4pll - else: - properties: - reg: - minItems: 6 - maxItems: 6 - reg-names: - items: - - const: core - - const: reset - - const: fabric - - const: axi2af - - const: lioa - - const: common-cfg - -additionalProperties: false From b092bd463ed0f21e7b755b19fda3e611ae790755 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 09:51:43 -0500 Subject: [PATCH 300/392] dt-bindings: soc: apple: pmp: Describe the T6050 PMP with its mailbox The T6050 PMP node now has the form its driver uses on the other SoCs: the firmware SRAM and ASC views named "pmp" and "asc", and the ASC mailbox, with the mailbox interrupts moving to that node. Describe it that way, keeping the second power domain (PMS SRAM). The node is no longer required to stay disabled. Signed-off-by: Diljit Singh --- .../bindings/soc/apple/apple,pmp.yaml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index bcbd0ecf4568a..4ae4380167632 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -60,7 +60,7 @@ properties: interrupts: maxItems: 4 - description: Coprocessor mailbox and status interrupts, on T6050 in ADT order. + description: Coprocessor mailbox and status interrupts. iommus: maxItems: 1 @@ -148,23 +148,19 @@ allOf: const: apple,t6050-pmp-v2 then: description: - The T6050 PMP is described from the ADT and stays disabled until its - firmware handoff is supported; its mailbox is not described yet. + The T6050 PMP's firmware SRAM and ASC views and its ASC mailbox. It + also holds the PMS SRAM's power domain. properties: reg: - minItems: 4 - maxItems: 4 - interrupts: - minItems: 4 - maxItems: 4 + maxItems: 2 power-domains: minItems: 2 - status: - const: disabled + interrupts: false required: - - interrupts + - reg-names + - mboxes + - mbox-names - power-domains - - status else: properties: reg: From 249746eda843e09db2460a79eebcfb926a7e203e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:04:55 +1100 Subject: [PATCH 301/392] phy: apple: t6040: hard-recover a wedged coprocessor Treat the Power Down reply as asynchronous and leave RTKit's mailbox transport stopped after detach. If shutdown was not acknowledged or a tune fails, force common_dp and phymxwrap off for a configurable dwell, restart the transport, and retry one complete PHY bring-up. Add a one-shot debugfs acknowledgement drop and KUnit coverage for missed replies, failed tunes, and retry exhaustion. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/phy/apple/Kconfig | 17 +- drivers/phy/apple/t6040-recovery-test.c | 50 ++++ drivers/phy/apple/t6040-usb2.c | 341 +++++++++++++++++++++--- 3 files changed, 375 insertions(+), 33 deletions(-) create mode 100644 drivers/phy/apple/t6040-recovery-test.c diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 5348987cf1dc8..5065d98295286 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -44,15 +44,26 @@ config PHY_APPLE_T6040_USB2 USB3 host attach decoded from the paired AppleT6040TypeCPhy driver, including the start of the PHY coprocessor, and the DisplayPort mode (all four lanes, RBR to HBR3) a Type-C controller selects through the - PHY's mode switch or that an HDMI converter's PHY runs permanently. - It provides no device mode, DisplayPort pin assignment D, USB4, - Thunderbolt or retimer support. + PHY's mode switch or that an HDMI converter's PHY runs permanently. The + same mode switch configures both lanes for USB4 or Thunderbolt. It + provides no device mode or retimer support. This driver performs no ATC-register writes at probe time. Enabling a matching device-tree node causes MMIO writes when a USB2, USB3 or DisplayPort consumer powers on the PHY and therefore requires platform-specific safety review. +config PHY_APPLE_T6040_KUNIT_TEST + bool "KUnit tests for Apple T6040 PHY recovery" if !KUNIT_ALL_TESTS + depends on PHY_APPLE_T6040_USB2 && KUNIT + depends on PHY_APPLE_T6040_USB2=m || KUNIT=y + default KUNIT_ALL_TESTS + help + Test the bounded hard-recovery state machine for a missed Power Down + acknowledgment, a failed firmware tune and retry exhaustion. The tests + access no hardware and run when the PHY driver is loaded. They are only + useful for kernel developers. + config PHY_APPLE_DPTX tristate "Apple DPTX PHY" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/phy/apple/t6040-recovery-test.c b/drivers/phy/apple/t6040-recovery-test.c new file mode 100644 index 0000000000000..552afba86f117 --- /dev/null +++ b/drivers/phy/apple/t6040-recovery-test.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* Included by t6040-usb2.c to test its recovery state machine. */ + +#include + +static void t6040_recovery_missed_ack_test(struct kunit *test) +{ + struct t6040_recovery_state state = {}; + + t6040_recovery_power_down(&state, false, true); + KUNIT_EXPECT_TRUE(test, state.cycle_pending); + KUNIT_EXPECT_TRUE(test, t6040_recovery_start(&state)); + KUNIT_EXPECT_EQ(test, state.retries, (u8)0); +} + +static void t6040_recovery_failed_tune_test(struct kunit *test) +{ + struct t6040_recovery_state state = {}; + + KUNIT_EXPECT_FALSE(test, t6040_recovery_start(&state)); + KUNIT_EXPECT_TRUE(test, t6040_recovery_retry(&state)); + KUNIT_EXPECT_TRUE(test, state.cycle_pending); + KUNIT_EXPECT_EQ(test, state.retries, (u8)1); + t6040_recovery_complete(&state); + KUNIT_EXPECT_FALSE(test, state.cycle_pending); +} + +static void t6040_recovery_retry_exhaustion_test(struct kunit *test) +{ + struct t6040_recovery_state state = {}; + + KUNIT_ASSERT_TRUE(test, t6040_recovery_retry(&state)); + KUNIT_EXPECT_FALSE(test, t6040_recovery_retry(&state)); + KUNIT_EXPECT_TRUE(test, state.cycle_pending); + KUNIT_EXPECT_EQ(test, state.retries, (u8)T6040_RECOVERY_RETRIES); +} + +static struct kunit_case t6040_recovery_test_cases[] = { + KUNIT_CASE(t6040_recovery_missed_ack_test), + KUNIT_CASE(t6040_recovery_failed_tune_test), + KUNIT_CASE(t6040_recovery_retry_exhaustion_test), + {} +}; + +static struct kunit_suite t6040_recovery_test_suite = { + .name = "phy-apple-t6040-recovery", + .test_cases = t6040_recovery_test_cases, +}; + +kunit_test_suite(t6040_recovery_test_suite); diff --git a/drivers/phy/apple/t6040-usb2.c b/drivers/phy/apple/t6040-usb2.c index 73b687686294d..032fcc72a3e53 100644 --- a/drivers/phy/apple/t6040-usb2.c +++ b/drivers/phy/apple/t6040-usb2.c @@ -34,11 +34,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -172,6 +174,46 @@ #define T6040_CPU_EP_APP 32 #define T6040_CPU_MSG_POWER_DOWN 0x0100000000000000ULL +#define T6040_RECOVERY_RETRIES 1 + +static unsigned int recovery_dwell_ms = 50; +module_param(recovery_dwell_ms, uint, 0644); +MODULE_PARM_DESC(recovery_dwell_ms, + "PHY coprocessor hard-recovery off dwell in milliseconds (minimum 1, default 50)"); + +struct t6040_recovery_state { + bool cycle_pending; + u8 retries; +}; + +static void t6040_recovery_power_down(struct t6040_recovery_state *state, + bool acknowledged, bool clean_stop) +{ + if (!acknowledged || !clean_stop) + state->cycle_pending = true; +} + +static bool t6040_recovery_start(struct t6040_recovery_state *state) +{ + state->retries = 0; + return state->cycle_pending; +} + +static bool t6040_recovery_retry(struct t6040_recovery_state *state) +{ + state->cycle_pending = true; + if (state->retries >= T6040_RECOVERY_RETRIES) + return false; + state->retries++; + return true; +} + +static void t6040_recovery_complete(struct t6040_recovery_state *state) +{ + state->cycle_pending = false; + state->retries = 0; +} + enum t6040_usb3_region { T6040_USB3_AXI2AF, T6040_USB3_AXI2AF_LIOA, @@ -239,7 +281,11 @@ struct t6040_usb2_phy { /* ADT tunable_USB2PHY_HOST, applied after the signal setup. */ struct apple_tunable *tunable_usb2_host; struct apple_rtkit *rtk; + struct device *mbox_dev; struct completion app_reply; + struct t6040_recovery_state recovery; + atomic_t drop_next_power_down_ack; + struct dentry *debugfs; enum typec_orientation orientation; bool usb3_up; /* tuned by the coprocessor this power cycle */ bool usb3_on; /* dwc3 holds the USB3 phy powered on */ @@ -577,7 +623,33 @@ static int t6040_usb3_wait_tuned(struct t6040_usb2_phy *tphy) u32 uc; return readl_poll_timeout(t6040_core(tphy) + T6040_CORE_UC_CTRL, uc, - uc & T6040_CORE_UC_CTRL_TNBL_DONE, 1000, USEC_PER_SEC); + uc & T6040_CORE_UC_CTRL_TNBL_DONE, 1, USEC_PER_SEC); +} + +static const struct apple_rtkit_ops t6040_cpu_rtkit_ops; + +static int t6040_rtkit_start_transport(struct t6040_usb2_phy *tphy) +{ + if (tphy->rtk) + return 0; + + tphy->rtk = apple_rtkit_init(tphy->dev, tphy, "phy-cpu", 0, + &t6040_cpu_rtkit_ops); + if (IS_ERR(tphy->rtk)) { + int ret = PTR_ERR(tphy->rtk); + + tphy->rtk = NULL; + return ret; + } + return 0; +} + +static void t6040_rtkit_stop_transport(struct t6040_usb2_phy *tphy) +{ + if (!tphy->rtk) + return; + apple_rtkit_free(tphy->rtk); + tphy->rtk = NULL; } /* @@ -589,6 +661,11 @@ static int t6040_usb3_wait_tuned(struct t6040_usb2_phy *tphy) static int t6040_usb3_cpu_start(struct t6040_usb2_phy *tphy) { void __iomem *cpu = t6040_cpu(tphy); + int ret; + + ret = t6040_rtkit_start_transport(tphy); + if (ret) + return ret; if (readl(cpu + T6040_CPU_RUN) & T6040_CPU_RUN_START) { dev_err(tphy->dev, "USB3: the PHY coprocessor runs without an RTKit session\n"); @@ -618,7 +695,7 @@ static int t6040_usb3_run_firmware(struct t6040_usb2_phy *tphy, bool usb, bool s fsleep(10); uc_before = readl(t6040_core(tphy) + T6040_CORE_UC_CTRL); - if (!apple_rtkit_is_running(tphy->rtk)) { + if (!tphy->rtk || !apple_rtkit_is_running(tphy->rtk)) { ret = t6040_usb3_cpu_start(tphy); if (ret) { dev_err(tphy->dev, "USB3: the PHY coprocessor was not started: %d (run %#x, idle %#x, status %#x)\n", @@ -644,43 +721,57 @@ static int t6040_usb3_run_firmware(struct t6040_usb2_phy *tphy, bool usb, bool s /* * Detach, RTBuddy::setPowerState(0): Power Down, RTKit AP then IOP to idle, - * wait up to 10 ms for the CPU to idle, STOP. The RTKit state is reset so the - * next attach boots the firmware as the first one did. The mailbox outbox - * enable that macOS clears here is left as iBoot set it: Linux never clears it. + * wait up to 10 ms for the CPU to idle, STOP. The RTKit transport stays stopped + * so the next attach can either restart it normally or first power-cycle the + * domains. The mailbox outbox enable that macOS clears here is left as iBoot + * set it: Linux never clears it. */ static void t6040_usb3_firmware_power_down(struct t6040_usb2_phy *tphy) { void __iomem *cpu = t6040_cpu(tphy); + bool acknowledged = false; + bool clean_stop = true; u32 status; int ret; - if (apple_rtkit_is_running(tphy->rtk)) { - reinit_completion(&tphy->app_reply); + reinit_completion(&tphy->app_reply); + if (tphy->rtk && apple_rtkit_is_running(tphy->rtk)) { ret = apple_rtkit_start_ep(tphy->rtk, T6040_CPU_EP_APP); if (!ret) ret = apple_rtkit_send_message(tphy->rtk, T6040_CPU_EP_APP, T6040_CPU_MSG_POWER_DOWN, NULL, false); - if (!ret && !wait_for_completion_timeout(&tphy->app_reply, msecs_to_jiffies(500))) - ret = -ETIMEDOUT; if (ret) - dev_warn(tphy->dev, "USB3: no Power Down acknowledge: %d\n", ret); + dev_warn(tphy->dev, "USB3: Power Down submission failed: %d\n", ret); ret = apple_rtkit_idle(tphy->rtk); - if (ret) + if (ret) { dev_warn(tphy->dev, "USB3: PHY coprocessor idle failed: %d\n", ret); + clean_stop = false; + } } - if (!(readl(cpu + T6040_CPU_RUN) & T6040_CPU_RUN_START)) - return; - if (readl_poll_timeout(cpu + T6040_CPU_STATUS, status, status & T6040_CPU_STATUS_IDLE, - 1000, 10000)) - dev_warn(tphy->dev, "USB3: the PHY coprocessor did not idle (status %#x)\n", - status); - writel(~T6040_CPU_RUN_START, cpu + T6040_CPU_RUN); + if (readl(cpu + T6040_CPU_RUN) & T6040_CPU_RUN_START) { + if (readl_poll_timeout(cpu + T6040_CPU_STATUS, status, + status & T6040_CPU_STATUS_IDLE, 1000, 10000)) { + dev_warn(tphy->dev, + "USB3: the PHY coprocessor did not idle (status %#x)\n", + status); + clean_stop = false; + } + writel(~T6040_CPU_RUN_START, cpu + T6040_CPU_RUN); + if (readl(cpu + T6040_CPU_RUN) & T6040_CPU_RUN_START) { + dev_warn(tphy->dev, "USB3: the PHY coprocessor STOP did not latch\n"); + clean_stop = false; + } + } - ret = apple_rtkit_reinit(tphy->rtk); - if (ret) - dev_warn(tphy->dev, "USB3: RTKit reset failed: %d\n", ret); + /* Leave the transport stopped until the next firmware start. Free drains RX. */ + t6040_rtkit_stop_transport(tphy); + /* Power Down is asynchronous; never wait solely for its application reply. */ + acknowledged = completion_done(&tphy->app_reply); + if (!acknowledged) + dev_warn(tphy->dev, "USB3: no asynchronous Power Down acknowledge\n"); + t6040_recovery_power_down(&tphy->recovery, acknowledged, clean_stop); } /* @@ -711,6 +802,76 @@ static void t6040_usb3_sleep(struct t6040_usb2_phy *tphy) udelay(1); } +static int t6040_common_dp_force_resume(struct t6040_usb2_phy *tphy) +{ + int ret; + + ret = pm_runtime_force_resume(tphy->dev); + if (ret) + return ret; + /* force_resume may preserve an idle device in suspend; this caller needs MMIO now. */ + ret = pm_runtime_resume_and_get(tphy->dev); + if (ret < 0) + return ret; + pm_runtime_put_noidle(tphy->dev); + return 0; +} + +/* + * Drop both domains with RTKit fully detached. phymxwrap is the mailbox + * provider's runtime domain; common_dp is this device's runtime domain. Force + * common_dp through a balanced suspend/resume because the USB2 PHY and DWC3 + * reset can hold independent runtime references while the SuperSpeed lanes are + * quiesced. + */ +static int t6040_usb3_hard_recover(struct t6040_usb2_phy *tphy) +{ + unsigned int dwell = max(recovery_dwell_ms, 1U); + int ret, resume_ret; + + if (WARN_ON(tphy->rtk)) + t6040_rtkit_stop_transport(tphy); + + ret = pm_runtime_force_suspend(tphy->dev); + if (ret < 0) + return ret; + + pm_runtime_barrier(tphy->mbox_dev); + ret = pm_runtime_suspend(tphy->mbox_dev); + if (ret < 0 && ret != -EACCES) { + dev_err(tphy->dev, "USB3: phymxwrap suspend failed: %d\n", ret); + goto resume_phy; + } + if (!pm_runtime_status_suspended(tphy->mbox_dev)) { + ret = -EBUSY; + dev_err(tphy->dev, "USB3: phymxwrap did not power off\n"); + goto resume_phy; + } + + msleep(dwell); + ret = t6040_rtkit_start_transport(tphy); + if (ret) { + dev_err(tphy->dev, "USB3: RTKit transport restart failed: %d\n", ret); + goto resume_phy; + } + + resume_ret = t6040_common_dp_force_resume(tphy); + if (resume_ret < 0) { + dev_err(tphy->dev, "USB3: common_dp resume failed: %d\n", resume_ret); + t6040_rtkit_stop_transport(tphy); + return resume_ret; + } + + dev_warn(tphy->dev, "USB3: completed hard recovery with %u ms off dwell\n", dwell); + return 0; + +resume_phy: + resume_ret = t6040_common_dp_force_resume(tphy); + if (resume_ret < 0) + return resume_ret; + return ret; +} + /* Sets PHY_STATUS(_2) handshake bits in the BIST block and waits as macOS does, without failing. */ static void t6040_usb3_bist_pulse(struct t6040_usb2_phy *tphy, u32 reg, u32 bit) { @@ -783,7 +944,6 @@ static void t6040_phy_tx_cmn_down(struct t6040_usb2_phy *tphy) udelay(1); } - static void t6040_usb3_shutdown_lanes(struct t6040_usb2_phy *tphy) { void __iomem *bist = t6040_bist(tphy); @@ -904,6 +1064,7 @@ static void t6040_usb3_pipe_park(struct t6040_usb2_phy *tphy) */ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) { + bool cycle; bool swap; int ret; @@ -931,6 +1092,14 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) /* PIPE_HANDLER_DEFAULT: the dummy PIPE clock the controller keeps using at USB2 */ apple_tunable_apply(tphy->pipehandler, tphy->tunable_pipehandler); + cycle = t6040_recovery_start(&tphy->recovery); +retry: + if (cycle) { + ret = t6040_usb3_hard_recover(tphy); + if (ret) + goto failed; + } + ret = t6040_usb3_wake(tphy); if (ret) goto sleep; /* the coprocessor block may be unclocked: never touch it */ @@ -938,17 +1107,27 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) t6040_usb3_lanes(tphy, swap); ret = t6040_usb3_run_firmware(tphy, true, swap); if (ret) - goto usb2_only; + goto retry_or_fail; + t6040_recovery_complete(&tphy->recovery); tphy->usb3_up = true; dev_dbg(tphy->dev, "USB3 PHY up, lane %d\n", swap); return; - -usb2_only: - /* power_off skips a PHY that never came up: undo here, so the next attach starts clean */ +retry_or_fail: + /* A retune is a full detach sequence followed by one complete fresh start. */ + t6040_usb3_shutdown_lanes(tphy); t6040_usb3_firmware_power_down(tphy); + t6040_usb3_sleep(tphy); + if (t6040_recovery_retry(&tphy->recovery)) { + cycle = true; + goto retry; + } + goto failed; + sleep: t6040_usb3_sleep(tphy); + +failed: dev_warn(tphy->dev, "USB3 PHY bring-up failed (%d), USB2 only\n", ret); } @@ -1524,6 +1703,7 @@ static bool t6040_cio_state(const struct typec_mux_state *state) static int t6040_cio_up_locked(struct t6040_usb2_phy *tphy) { + bool cycle; bool flip; int ret; @@ -1548,6 +1728,15 @@ static int t6040_cio_up_locked(struct t6040_usb2_phy *tphy) ret = pm_runtime_resume_and_get(tphy->dev); if (ret) return ret; + + cycle = t6040_recovery_start(&tphy->recovery); +retry: + if (cycle) { + ret = t6040_usb3_hard_recover(tphy); + if (ret) + goto put; + } + ret = t6040_usb3_wake(tphy); if (ret) goto sleep; /* the coprocessor block may be unclocked: never touch it */ @@ -1556,15 +1745,25 @@ static int t6040_cio_up_locked(struct t6040_usb2_phy *tphy) t6040_cio_lanes_on(tphy); ret = t6040_usb3_run_firmware(tphy, false, flip); if (ret) { + t6040_phy_tx_cmn_down(tphy); + t6040_cio_rx_down(tphy); + fsleep(200); t6040_usb3_firmware_power_down(tphy); - goto sleep; + t6040_usb3_sleep(tphy); + if (t6040_recovery_retry(&tphy->recovery)) { + cycle = true; + goto retry; + } + goto put; } + t6040_recovery_complete(&tphy->recovery); tphy->cio_up = true; dev_info(tphy->dev, "CIO PHY up, %s orientation\n", flip ? "reverse" : "normal"); return 0; sleep: t6040_usb3_sleep(tphy); +put: pm_runtime_put(tphy->dev); dev_warn(tphy->dev, "CIO PHY bring-up failed: %d\n", ret); return ret; @@ -1892,10 +2091,16 @@ static void t6040_cpu_recv_message(void *cookie, u8 ep, u64 msg) { struct t6040_usb2_phy *tphy = cookie; - if (ep == T6040_CPU_EP_APP) + if (ep == T6040_CPU_EP_APP) { + if (atomic_xchg(&tphy->drop_next_power_down_ack, 0)) { + dev_warn(tphy->dev, "USB3: dropped Power Down acknowledge (fault injection)\n"); + return; + } + dev_dbg(tphy->dev, "USB3: Power Down acknowledge %#llx\n", msg); complete(&tphy->app_reply); - else + } else { dev_dbg(tphy->dev, "USB3: coprocessor message on endpoint %u: %#llx\n", ep, msg); + } } static const struct apple_rtkit_ops t6040_cpu_rtkit_ops = { @@ -1906,6 +2111,65 @@ static const struct apple_rtkit_ops t6040_cpu_rtkit_ops = { .shmem_destroy = t6040_cpu_shmem_destroy, }; +static void t6040_rtkit_cleanup(void *data) +{ + t6040_rtkit_stop_transport(data); +} + +static void t6040_mbox_dev_put(void *data) +{ + put_device(data); +} + +static int t6040_get_mbox_dev(struct t6040_usb2_phy *tphy) +{ + struct of_phandle_args args; + struct platform_device *pdev; + int ret; + + ret = of_parse_phandle_with_args(tphy->dev->of_node, "mboxes", "#mbox-cells", 0, + &args); + if (ret) + return ret; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!pdev) + return -EPROBE_DEFER; + + tphy->mbox_dev = &pdev->dev; + return devm_add_action_or_reset(tphy->dev, t6040_mbox_dev_put, tphy->mbox_dev); +} + +static void t6040_debugfs_remove(void *data) +{ + debugfs_remove_recursive(data); +} + +static int t6040_debugfs_init(struct t6040_usb2_phy *tphy) +{ + const char *name; + int ret; + + name = devm_kasprintf(tphy->dev, GFP_KERNEL, "t6040-atcphy-%s", + dev_name(tphy->dev)); + if (!name) + return -ENOMEM; + tphy->debugfs = debugfs_create_dir(name, NULL); + if (IS_ERR(tphy->debugfs)) { + if (PTR_ERR(tphy->debugfs) == -ENODEV) { + tphy->debugfs = NULL; + return 0; + } + return PTR_ERR(tphy->debugfs); + } + debugfs_create_atomic_t("drop_next_power_down_ack", 0600, tphy->debugfs, + &tphy->drop_next_power_down_ack); + ret = devm_add_action_or_reset(tphy->dev, t6040_debugfs_remove, tphy->debugfs); + if (ret) + tphy->debugfs = NULL; + return ret; +} + /* * Map the PHY register windows shared by the USB3 and DisplayPort paths. Each * is optional: a USB-C PHY has the full set, the fixed HDMI DP PHY (atcphy3) @@ -2003,14 +2267,26 @@ static int t6040_usb3_probe(struct t6040_usb2_phy *tphy, struct platform_device return 0; } - tphy->rtk = devm_apple_rtkit_init(dev, tphy, "phy-cpu", 0, &t6040_cpu_rtkit_ops); + ret = t6040_get_mbox_dev(tphy); + if (ret) + return dev_err_probe(dev, ret, "failed to find PHY coprocessor mailbox\n"); + + tphy->rtk = apple_rtkit_init(dev, tphy, "phy-cpu", 0, &t6040_cpu_rtkit_ops); if (IS_ERR(tphy->rtk)) { ret = PTR_ERR(tphy->rtk); + tphy->rtk = NULL; if (ret == -EPROBE_DEFER) return ret; dev_warn(dev, "USB3: no PHY coprocessor mailbox (%d), USB2 only\n", ret); return 0; } + ret = devm_add_action_or_reset(dev, t6040_rtkit_cleanup, tphy); + if (ret) + return ret; + + ret = t6040_debugfs_init(tphy); + if (ret) + return ret; tphy->usb3_ok = true; return 0; @@ -2135,6 +2411,7 @@ static int t6040_usb2_probe(struct platform_device *pdev) */ tphy->mode = PHY_MODE_USB_HOST; mutex_init(&tphy->lock); + atomic_set(&tphy->drop_next_power_down_ack, 0); platform_set_drvdata(pdev, tphy); tphy->usb2 = devm_platform_ioremap_resource_byname(pdev, "bank0"); @@ -2223,3 +2500,7 @@ module_platform_driver(t6040_usb2_driver); MODULE_AUTHOR("The Asahi Linux Contributors"); MODULE_DESCRIPTION("Apple T6040 Type-C PHY, USB2/USB3 host and DisplayPort"); MODULE_LICENSE("Dual BSD/GPL"); + +#if IS_ENABLED(CONFIG_PHY_APPLE_T6040_KUNIT_TEST) +#include "t6040-recovery-test.c" +#endif From 1dba98af42d13b4d041348ff858587ecc680bdd9 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:05:05 +1100 Subject: [PATCH 302/392] pmdomain: apple: add T6041 CIO reconfiguration Implement the T6041 PMGR CIO reset contract as a dedicated reset provider: serialize the shared register, wait up to 192 ms before and after the request, and force a read-modify-write of the per-port request bit. Describe the exact compatible and use it for the DT-provided 0x2024 resource instead of the shorter generic fallback. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- .../bindings/reset/apple,t8103-cio-reset.yaml | 7 +- arch/arm64/boot/dts/apple/t6040-usb4.dtsi | 2 +- drivers/pmdomain/apple/Kconfig | 11 ++ drivers/pmdomain/apple/Makefile | 1 + drivers/pmdomain/apple/t6041-cio-reconfig.c | 112 ++++++++++++++++++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 drivers/pmdomain/apple/t6041-cio-reconfig.c diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index a7bc6bf4885c9..2fdc19629f3d7 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -22,6 +22,11 @@ description: | On t8142, a standalone register page contains a shared request register with alternating request and busy bits for four ports. + On t6041, a PMGR child register contains one request bit per CIO port and + the corresponding busy bit at 16 plus the port index. The controller waits + up to 192 ms for that busy bit to clear both before and after its + read-modify-write request. + In all cases the reset id is the CIO port index on the die. properties: @@ -32,6 +37,7 @@ properties: - apple,t8142-cio-reset - apple,t6050-cio-reset - apple,t6000-cio-reset + - apple,t6041-cio-reset - items: - const: apple,t8112-cio-reset - const: apple,t8103-cio-reset @@ -40,7 +46,6 @@ properties: - apple,t6020-cio-reset - apple,t6030-cio-reset - apple,t6031-cio-reset - - apple,t6041-cio-reset - apple,t8122-cio-reset - apple,t8132-cio-reset - const: apple,t6000-cio-reset diff --git a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi index cb15a0516393c..1c239f2459b9d 100644 --- a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi @@ -537,7 +537,7 @@ status = "disabled"; cio_reset: reset-controller@2024 { - compatible = "apple,t6041-cio-reset", "apple,t6000-cio-reset"; + compatible = "apple,t6041-cio-reset"; reg = <0x2024 0x4>; #reset-cells = <1>; }; diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 52fe2669e6fe8..f0507d8c15ae6 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -26,6 +26,17 @@ config APPLE_PMGR_PWRSTATE_KUNIT_TEST and need no Apple hardware. Also check the T6000 GFX missing-parent wait, its timeout, and the separate actual-state acknowledgment. +config APPLE_T6041_CIO_RECONFIG + bool "Apple T6041 CIO cluster reconfiguration" + depends on OF + select MFD_SYSCON + select REGMAP + select RESET_CONTROLLER + default ARCH_APPLE + help + Provide the PMGR reconfiguration request used after the T6041 Type-C + PHY is tuned and before an ACIO USB4/Thunderbolt router starts. + config APPLE_PMP_REPORT bool "Apple PMP report control" depends on OF diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index b03554cd70051..7e8eaf9df95a6 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += gfx-tvm.o +obj-$(CONFIG_APPLE_T6041_CIO_RECONFIG) += t6041-cio-reconfig.o obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o obj-$(CONFIG_APPLE_PMP_REPORT_KUNIT_TEST) += pmp-report-test.o obj-$(CONFIG_APPLE_PMP_DRAM_KUNIT_TEST) += pmp-dram-test.o diff --git a/drivers/pmdomain/apple/t6041-cio-reconfig.c b/drivers/pmdomain/apple/t6041-cio-reconfig.c new file mode 100644 index 0000000000000..a6c13b41a801d --- /dev/null +++ b/drivers/pmdomain/apple/t6041-cio-reconfig.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T6041 CIO cluster reconfiguration controller + * + * The three CIO ports share one PMGR register. Each port has a request bit + * and a busy bit sixteen positions above it. Firmware waits before and after + * the read/modify/write request, with a 192 ms budget for each wait. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define T6041_CIO_RECONFIG_PORTS 3 +#define T6041_CIO_RECONFIG_POLL_US 100 +#define T6041_CIO_RECONFIG_TIMEOUT_US 192000 + +struct t6041_cio_reconfig { + struct reset_controller_dev rcdev; + struct regmap *regmap; + struct mutex lock; /* serializes the shared request register */ + u32 offset; +}; + +static int t6041_cio_reconfig_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct t6041_cio_reconfig *reconfig = + container_of(rcdev, struct t6041_cio_reconfig, rcdev); + u32 busy = BIT(16 + id); + u32 request = BIT(id); + u32 value; + int ret; + + guard(mutex)(&reconfig->lock); + + ret = regmap_read_poll_timeout(reconfig->regmap, reconfig->offset, value, + !(value & busy), T6041_CIO_RECONFIG_POLL_US, + T6041_CIO_RECONFIG_TIMEOUT_US); + if (ret) + return ret; + + /* Force the RMW write even if a stale request bit still reads as one. */ + ret = regmap_write_bits(reconfig->regmap, reconfig->offset, request, request); + if (ret) + return ret; + + return regmap_read_poll_timeout(reconfig->regmap, reconfig->offset, value, + !(value & busy), T6041_CIO_RECONFIG_POLL_US, + T6041_CIO_RECONFIG_TIMEOUT_US); +} + +static const struct reset_control_ops t6041_cio_reconfig_ops = { + .deassert = t6041_cio_reconfig_deassert, +}; + +static int t6041_cio_reconfig_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t6041_cio_reconfig *reconfig; + int ret; + + reconfig = devm_kzalloc(dev, sizeof(*reconfig), GFP_KERNEL); + if (!reconfig) + return -ENOMEM; + + ret = devm_mutex_init(dev, &reconfig->lock); + if (ret) + return ret; + + reconfig->regmap = syscon_node_to_regmap(dev->of_node->parent); + if (IS_ERR(reconfig->regmap)) + return dev_err_probe(dev, PTR_ERR(reconfig->regmap), + "failed to get parent PMGR regmap\n"); + + ret = of_property_read_u32(dev->of_node, "reg", &reconfig->offset); + if (ret) + return dev_err_probe(dev, ret, "failed to read reconfiguration offset\n"); + + reconfig->rcdev.owner = THIS_MODULE; + reconfig->rcdev.ops = &t6041_cio_reconfig_ops; + reconfig->rcdev.of_node = dev->of_node; + reconfig->rcdev.nr_resets = T6041_CIO_RECONFIG_PORTS; + + return devm_reset_controller_register(dev, &reconfig->rcdev); +} + +static const struct of_device_id t6041_cio_reconfig_of_match[] = { + { .compatible = "apple,t6041-cio-reset" }, + {} +}; +MODULE_DEVICE_TABLE(of, t6041_cio_reconfig_of_match); + +static struct platform_driver t6041_cio_reconfig_driver = { + .probe = t6041_cio_reconfig_probe, + .driver = { + .name = "apple-t6041-cio-reconfig", + .of_match_table = t6041_cio_reconfig_of_match, + }, +}; +module_platform_driver(t6041_cio_reconfig_driver); + +MODULE_AUTHOR("The Asahi Linux Contributors"); +MODULE_DESCRIPTION("Apple T6041 CIO cluster reconfiguration controller"); +MODULE_LICENSE("Dual MIT/GPL"); From 0387d4c03a869a9a21e07fac23b228c105b12c72 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:06:03 +1100 Subject: [PATCH 303/392] dt-bindings: spmi: apple,spmi: an interrupt controller may have no AP line The T8112 AOP SPMI controllers demultiplex follower interrupts but have no interrupt line to the AP. Require #interrupt-cells with interrupt-controller, but not interrupts. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- Documentation/devicetree/bindings/spmi/apple,spmi.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index 5c9e77075b303..48706095da208 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -53,7 +53,12 @@ properties: maxItems: 1 description: Optional, operates in polled mode if not present - interrupt-controller: true + interrupt-controller: + description: + The controller demultiplexes follower interrupts. Some instances (the + T8112 AOP controllers) have no interrupt line to the AP. Follower + interrupts are then described but cannot be delivered, and drivers + fall back to polling or stay idle. "#interrupt-cells": const: 2 @@ -68,7 +73,6 @@ required: dependentRequired: interrupt-controller: - "#interrupt-cells" - - interrupts unevaluatedProperties: false From c6773e9de5a9780081e5c40eb5812c4f5341f9b5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 10:10:53 -0500 Subject: [PATCH 304/392] pmdomain: apple: pmp-report: Add T6050 The T6050 PMP nub's ptd-range lists the PTD entries T6041 has, at the same places: PMP-STATUS 1, SOC-DEV-PS-REQ 480, SOC-DEV-PS-ACK 488 and FAST-DIE-CE 600 with three lanes. Use T6041's offsets. Tested on J714s (M5 Pro) with the report at 0x284240000 and always-on requests for the three CPU clusters, the internal display and ANS: the driver reads "PMP running (status 0x1)" and writes requests 0x1000047000, the PMP acknowledges every bit, and its PMC vote drops from the static DCS 5 / fabric 1 iBoot leaves to 0 and stays there at idle and under CPU and GPU load. Signed-off-by: Diljit Singh --- .../devicetree/bindings/power/apple,t8140-pmp-report.yaml | 2 ++ drivers/pmdomain/apple/pmp-report.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 5b36d27734605..ec06771e3a534 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -50,6 +50,7 @@ properties: - apple,t6020-pmp-v2-report - apple,t6030-pmp-v2-report - apple,t6041-pmp-v2-report + - apple,t6050-pmp-v2-report - apple,t8112-pmp-v2-report - apple,t8122-pmp-v2-report - apple,t8140-pmp-v2-report @@ -113,6 +114,7 @@ patternProperties: - apple,t6030-pmp-v2-report-entry - apple,t6031-pmp-v2-report-entry - apple,t6041-pmp-v2-report-entry + - apple,t6050-pmp-v2-report-entry - apple,t8112-pmp-v2-report-entry - apple,t8122-pmp-v2-report-entry - const: apple,t6000-pmp-v2-report-entry diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 789cab1c29397..11178eef8fe7c 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1537,6 +1537,8 @@ static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,t6030-pmp-v2-report", .data = &apple_pmp_offsets_t6030 }, { .compatible = "apple,t6031-pmp-v2-report", .data = &apple_pmp_offsets_t6031 }, { .compatible = "apple,t6041-pmp-v2-report", .data = &apple_pmp_offsets_t6041 }, + /* T6050: the ADT's ptd-range lists T6041's entries at the same places. */ + { .compatible = "apple,t6050-pmp-v2-report", .data = &apple_pmp_offsets_t6041 }, { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, { .compatible = "apple,t8122-pmp-v2-report", .data = &apple_pmp_offsets_t8122 }, { .compatible = "apple,t8132-pmp-v2-report", .data = &apple_pmp_offsets_t8132 }, From aaa8797c3ce3e2b459ec04da691be594208d9b37 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 10:10:53 -0500 Subject: [PATCH 305/392] arm64: dts: apple: t6050: Describe the PMP report The host tells the PMP which soc-devices are on through the PTD: the read aperture at 0x284240000, and the update page 0x10000 above it, which is pmp0's ptd-update register (ADT reg[3]); together they are the PMGR's ADT reg[7]. Describe it with always-on requests for the three CPU clusters, the internal display and storage, the entries it was tested with. It stays disabled with the PMP. Signed-off-by: Diljit Singh --- .../boot/dts/apple/t6050-peripherals.dtsi | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index c4be3fcf47ca7..72c24739a5a51 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2203,6 +2203,69 @@ status = "disabled"; }; + /* + * PMP report (PTD): reads 16 bytes per entry from the base, writes 8 bytes per entry from + * base + 0x10000, which is pmp0's ptd-update register (ADT pmp0 reg[3]); together they are + * /arm-io/pmgr reg[7]. Entries are PMP soc-devices; an entry's reg is its soc-device ID - 1 + * (ADT iop-pmp0-nub soc-device). Without requests the running PMP keeps the vote iBoot left. + */ + pmp0_report: pmp-report@284240000 { + compatible = "apple,t6050-pmp-v2-report"; + reg = <0x2 0x84240000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + /* The CPU clusters: MACC0, MACC1, PACC */ + report@c { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-macc0"; + #power-domain-cells = <0>; + apple,always-on; + }; + + report@d { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-macc1"; + #power-domain-cells = <0>; + apple,always-on; + }; + + report@e { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-pacc"; + #power-domain-cells = <0>; + apple,always-on; + }; + + /* The internal display (DISPINT) */ + report@12 { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-dispint"; + #power-domain-cells = <0>; + apple,always-on; + }; + + /* Storage (ANS) */ + report@24 { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x24>; + label = "pmp-ans"; + #power-domain-cells = <0>; + apple,always-on; + }; + }; + pmp0_dart: iommu@2841a0000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x2 0x841a0000 0x0 0xc000>, From 8e704cbd0700b3b9d99d58633bd3361367fff126 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:00:24 +1100 Subject: [PATCH 306/392] arm64: dts: apple: expose the boot NOR system regions read-only The boot NOR below the nvram partition holds the iBoot firmware images and the syscfg factory data on the M1 and M2 family layout, and the syscfg data alone on J700. Describe that range as a read-only MTD partition so it can be inspected without risking a write. The new partition follows nvram in the node list so the nvram MTD index does not change. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> (cherry picked from commit 91d007d7faeab646ead12b5be273a38b5ac8eefa) --- arch/arm64/boot/dts/apple/spi1-nvram.dtsi | 10 ++++++++++ arch/arm64/boot/dts/apple/t8140-j700.dts | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi index 268d9ffb94044..aa7799a26d6d7 100644 --- a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi +++ b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi @@ -43,6 +43,16 @@ reg = <0x0 0x0>; status = "disabled"; }; + + /* + * iBoot firmware and the syscfg factory data. Listed + * after nvram so the nvram MTD index stays the same. + */ + partition@0 { + label = "firmware"; + reg = <0x0 0x700000>; + read-only; + }; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index a9acaf70b90d5..54bc359a0e9de 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -540,6 +540,16 @@ label = "nvram"; reg = <0x24000 0x1d0000>; }; + + /* + * syscfg factory data. Listed after nvram so the nvram + * MTD index stays the same. + */ + partition@0 { + label = "syscfg"; + reg = <0x0 0x24000>; + read-only; + }; }; }; }; From a58009c15c038759cd2d789f8209e4fa51c32d10 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:15:21 +1100 Subject: [PATCH 307/392] arm64: dts: apple: t8112-j413: enable the read-only aux PMIC view Enable the auxiliary PMIC SPMI bus. The Vale PMIC on it is exported as a read-only apple-spmi-nvmem device. The bus has no interrupt, so the controller polls, makes no register writes at probe, and no SPMI transaction happens until the NVMEM device is read. Tested on the M2: 4 KiB read in 256-byte pages with no SPMI errors, page 0 stable on re-read. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8112-j413.dts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 9c9e7532ae048..15629b518e467 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -306,11 +306,12 @@ }; &nub_spmi1 { + status = "okay"; + pmic@e { compatible = "apple,vale-pmic", "apple,spmi-nvmem"; reg = <0xe SPMI_USID>; read-only; - status = "disabled"; }; }; From a125bb5557b4907ecb433acccb93007654b567bb Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:08:20 +1100 Subject: [PATCH 308/392] nvmem: apple-spmi-nvmem: make the sysfs file root-only The nvmem file of an SPMI follower reads PMIC registers on the bus. Restrict it to root, like the other register-backed nvmem providers. In-kernel cell consumers are unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> (cherry picked from commit 320c84189a2c58ce0f17a7af6df27313c2a9aaa4) --- drivers/nvmem/apple-spmi-nvmem.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index 755c429730dce..b31b8212d3fcc 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -46,6 +46,8 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) .word_size = 1, .stride = 1, .size = 0xffff, + /* PMIC registers: only root may read them through sysfs. */ + .root_only = true, .reg_read = apple_spmi_nvmem_read, .reg_write = apple_spmi_nvmem_write, }; From 8fbdc1d40e3b35dd0fba9b16e842cc9a34221a10 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:27:32 +1100 Subject: [PATCH 309/392] arm64: dts: apple: t8112-j413: add the GPU temperature sensors The M2 MacBook Air exposed no GPU temperature channel. Its SMC has six GPU keys: Tg0e, Tg0m and Tg0q, and Tg0f, Tg0n and Tg0r, which read the first three plus a fixed 4.6 C. Measured on a J413: 37.9-44.3 C during a glmark2 run. As on the M4 Pro, the SMC publishes them only while the GPU is powered. With the GPU off at idle they read 0 C and 4.6 C. Mark readings below 10 C as no data. They are read-only hwmon channels with no thermal zone or trip, added after the channels the thermal zones index. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-j413.dts | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 9c9e7532ae048..720468c0cd3d3 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -272,6 +272,50 @@ }; #include "hwmon-laptop.dtsi" + +/* + * GPU temperatures, after the channels the thermal zones index. The SMC + * publishes them only while the GPU is powered: with the GPU off, Tg0e, + * Tg0m and Tg0q read 0 C and Tg0f, Tg0n and Tg0r 4.6 C (they track the + * first three plus a fixed 4.6 C). Those placeholders are no data. + */ +&smc_hwmon { + temperature-Tg0e { + apple,key-id = "Tg0e"; + label = "GPU Tg0e"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0f { + apple,key-id = "Tg0f"; + label = "GPU Tg0f"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0m { + apple,key-id = "Tg0m"; + label = "GPU Tg0m"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0n { + apple,key-id = "Tg0n"; + label = "GPU Tg0n"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0q { + apple,key-id = "Tg0q"; + label = "GPU Tg0q"; + apple,no-data-below-millicelsius = <10000>; + }; + + temperature-Tg0r { + apple,key-id = "Tg0r"; + label = "GPU Tg0r"; + apple,no-data-below-millicelsius = <10000>; + }; +}; /* * Mesa fingerprint sensor on spi2. The J413 platform device tree * (/arm-io/spi2/mesa) describes it as J415's does: power on AP GPIO 178, From 923e129a86a861bd16b2c8a58793d635799719be Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Mon, 2 Nov 2026 02:26:16 -0800 Subject: [PATCH 310/392] serial: apple-dockchannel: Never wait for a reader with apple,tx-never-wait The DockChannel console drains only while a debug USB host reads the channel. With none attached, as on an M5 Ultra Mac Studio (J775D) that boots unattended, the TX FIFO stays full: the write loops spun up to 10000 reads of TX_FREE per character and every close waited for output that could not drain, so boot and every console write stalled. With apple,tx-never-wait, console writes (the driver's and the early console's) send what TX_FREE says fits and drop the rest, tty output is sent from the drain timer and dropped after two ticks without room, a close does not wait, and tx_empty reports empty. Without the property, every other DockChannel Mac's driver and early console behave as before. Testing: J775D boots unattended to its login with nothing reading the channel (the property set), and with a reader attached the console carries the whole boot log. Signed-off-by: Jeff Queisser Co-Authored-By: Claude Opus 5.5 Signed-off-by: Erik Bourget --- drivers/tty/serial/apple_dockchannel.h | 73 +++++++++ .../tty/serial/apple_dockchannel_early_uart.c | 31 +++- drivers/tty/serial/apple_dockchannel_uart.c | 155 +++++++++++++++--- 3 files changed, 226 insertions(+), 33 deletions(-) create mode 100644 drivers/tty/serial/apple_dockchannel.h diff --git a/drivers/tty/serial/apple_dockchannel.h b/drivers/tty/serial/apple_dockchannel.h new file mode 100644 index 0000000000000..97fcccc6a5a57 --- /dev/null +++ b/drivers/tty/serial/apple_dockchannel.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple DockChannel UART: console output shared by the earlycon and the driver. + */ +#ifndef _APPLE_DOCKCHANNEL_H +#define _APPLE_DOCKCHANNEL_H + +#include +#include +#include + +#define DOCKCHANNEL_TX8 0x04 +#define DOCKCHANNEL_TX_FREE 0x14 +#define DOCKCHANNEL_TX_RETRIES 10000 + +/* + * Whether console and tty output never wait for a reader: the DT property + * apple,tx-never-wait on the UART node, which only J775D (t6050-j775d.dts) + * sets, where unattended boots run with nothing reading the channel. Every + * other DockChannel Mac keeps waiting for room as before. Before widening + * it (or making it the default), check on one other DockChannel Mac with a + * slow attached reader (m1n1 proxy, USB debug) that long printk lines and tty + * output still arrive whole. + */ +#define DOCKCHANNEL_NEVER_WAIT "apple,tx-never-wait" + +/* Waiting for room, at most DOCKCHANNEL_TX_RETRIES reads per character. */ +static inline void dockchannel_wait_putc(void __iomem *data, unsigned char c) +{ + unsigned int retries = DOCKCHANNEL_TX_RETRIES; + + while (readl(data + DOCKCHANNEL_TX_FREE) == 0 && --retries) + cpu_relax(); + if (!retries) + return; + + writel(c, data + DOCKCHANNEL_TX8); +} + +/* + * With DOCKCHANNEL_NEVER_WAIT: the channel drains only while a debug USB host + * reads it; with none, the TX FIFO stays full for good. So console output + * never waits for room: each + * write sends what TX_FREE says fits and drops the rest. One write reads + * TX_FREE again when the room it read runs out, and stops reading once it + * reads 0. Callers serialize writes, so the room read stays true. + */ +struct dockchannel_tx { + u32 room; + bool full; +}; + +static inline void dockchannel_tx_begin(struct dockchannel_tx *tx) +{ + tx->room = 0; + tx->full = false; +} + +static inline void dockchannel_console_putc(void __iomem *data, + struct dockchannel_tx *tx, + unsigned char c) +{ + if (!tx->room && !tx->full) { + tx->room = readl(data + DOCKCHANNEL_TX_FREE); + tx->full = !tx->room; + } + if (!tx->room) + return; + tx->room--; + writel(c, data + DOCKCHANNEL_TX8); +} + +#endif diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c index 1048dde41d794..6c427189d34be 100644 --- a/drivers/tty/serial/apple_dockchannel_early_uart.c +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -8,22 +8,33 @@ #include #include +#include +#include #include -#define DOCKCHANNEL_TX8 0x04 -#define DOCKCHANNEL_TX_FREE 0x14 -#define DOCKCHANNEL_TX_RETRIES 10000 +#include "apple_dockchannel.h" + +static struct dockchannel_tx dockchannel_early_tx; +static bool dockchannel_early_never_wait; static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = DOCKCHANNEL_TX_RETRIES; + if (dockchannel_early_never_wait) + dockchannel_console_putc(port->membase, &dockchannel_early_tx, c); + else + dockchannel_wait_putc(port->membase, c); +} - while (readl(port->membase + DOCKCHANNEL_TX_FREE) == 0 && --retries) - cpu_relax(); - if (!retries) - return; +/* The DT's DockChannel UART node asks for DOCKCHANNEL_NEVER_WAIT (flat DT). */ +static bool __init dockchannel_uart_early_never_wait(void) +{ + const void *fdt = initial_boot_params; + int node; - writel(c, port->membase + DOCKCHANNEL_TX8); + if (!fdt) + return false; + node = fdt_node_offset_by_compatible(fdt, -1, "apple,dockchannel-uart"); + return node >= 0 && fdt_getprop(fdt, node, DOCKCHANNEL_NEVER_WAIT, NULL); } static void dockchannel_uart_early_write(struct console *con, const char *s, @@ -31,6 +42,7 @@ static void dockchannel_uart_early_write(struct console *con, const char *s, { struct earlycon_device *dev = con->data; + dockchannel_tx_begin(&dockchannel_early_tx); uart_console_write(&dev->port, s, n, dockchannel_uart_putc); } @@ -42,6 +54,7 @@ static int __init dockchannel_uart_early_setup(struct earlycon_device *device, if (!port->mapbase) return -ENODEV; + dockchannel_early_never_wait = dockchannel_uart_early_never_wait(); device->con->write = dockchannel_uart_early_write; return 0; } diff --git a/drivers/tty/serial/apple_dockchannel_uart.c b/drivers/tty/serial/apple_dockchannel_uart.c index c015d916ac432..d4255cb949848 100644 --- a/drivers/tty/serial/apple_dockchannel_uart.c +++ b/drivers/tty/serial/apple_dockchannel_uart.c @@ -16,6 +16,8 @@ #include #include +#include "apple_dockchannel.h" + #define FIFO_SIZE 0x800 #define IRQ_MASK 0x0 @@ -44,7 +46,13 @@ * that is about 115 bytes per poll, far below the FIFO size. */ #define SW_DRAIN_MS 10 -#define TX_RETRIES 10000 +/* + * TX never waits for FIFO room: what does not fit stays in the xmit buffer + * and the timer sends it as room appears. After this many ticks with no room + * at all, nothing is reading the channel: drop what is queued, so writers do + * not block on a reader that may never come. + */ +#define TX_STUCK_TICKS 2 struct dcuart_port { struct uart_port port; @@ -57,6 +65,12 @@ struct dcuart_port { struct timer_list drain_timer; /* Serializes IRQ mask updates and RX FIFO draining. */ spinlock_t lock; + /* DOCKCHANNEL_NEVER_WAIT: TX never waits for room (J775D only). */ + bool never_wait; + /* One console write's view of the TX FIFO (dockchannel_console_putc). */ + struct dockchannel_tx console_tx; + /* Timer ticks in a row that found TX output queued and no FIFO room. */ + unsigned int tx_stuck_ticks; }; static struct uart_driver dcuart_drv; @@ -97,22 +111,38 @@ static void dcuart_irq_disable(struct dcuart_port *dcuart, u32 bits) static void dcuart_putc(struct uart_port *port, unsigned char c) { - unsigned int retries = TX_RETRIES; + struct dcuart_port *dcuart = container_of(port, struct dcuart_port, port); - while (readl(port->membase + DATA_TX_FREE) == 0 && --retries) - cpu_relax(); - if (!retries) - return; - - writel(c, port->membase + DATA_TX8); + if (dcuart->never_wait) + dockchannel_console_putc(port->membase, &dcuart->console_tx, c); + else + dockchannel_wait_putc(port->membase, c); } static void dcuart_write(struct console *co, const char *s, unsigned int n) { struct dcuart_port *dcuart = dcuart_ports[co->index]; + struct uart_port *port = &dcuart->port; + unsigned long flags; + bool locked = true; + + if (!dcuart->never_wait) { + uart_console_write(port, s, n, dcuart_putc); + return; + } + + /* The TX path writes the same FIFO, and the room read must stay true. */ + if (oops_in_progress) + locked = uart_port_trylock_irqsave(port, &flags); + else + uart_port_lock_irqsave(port, &flags); + + dockchannel_tx_begin(&dcuart->console_tx); + uart_console_write(port, s, n, dcuart_putc); - uart_console_write(&dcuart->port, s, n, dcuart_putc); + if (locked) + uart_port_unlock_irqrestore(port, flags); } static int dcuart_console_setup(struct console *co, char *options) @@ -198,39 +228,106 @@ static irqreturn_t dcuart_interrupt(int irq, void *data) return IRQ_HANDLED; } +/* + * Send what the TX FIFO has room for, without waiting. From the timer (tick), + * also count ticks without room and drop the queue after TX_STUCK_TICKS. + * Called with the port lock held. Returns the bytes left queued. + */ +static unsigned int dcuart_tx_chars(struct dcuart_port *dcuart, bool tick) +{ + struct uart_port *port = &dcuart->port; + struct tty_port *tport = &port->state->port; + unsigned int queued = kfifo_len(&tport->xmit_fifo), pending; + u8 ch; + + if (!queued && !port->x_char) + return 0; + + pending = uart_port_tx_limited(port, ch, readl(port->membase + DATA_TX_FREE), + true, writel(ch, port->membase + DATA_TX8), ({})); + if (!pending || pending < queued) { + dcuart->tx_stuck_ticks = 0; + return pending; + } + if (!tick || ++dcuart->tx_stuck_ticks < TX_STUCK_TICKS) + return pending; + + kfifo_reset_out(&tport->xmit_fifo); + dcuart->tx_stuck_ticks = 0; + uart_write_wakeup(port); + return 0; +} + static void dcuart_drain_timer(struct timer_list *t) { struct dcuart_port *dcuart = timer_container_of(dcuart, t, drain_timer); + struct uart_port *port = &dcuart->port; unsigned long flags; - bool got; + unsigned int pending; - spin_lock_irqsave(&dcuart->lock, flags); - got = dcuart_drain_rx(dcuart); - spin_unlock_irqrestore(&dcuart->lock, flags); + if (!dcuart->never_wait) { + bool got; - if (got) - tty_flip_buffer_push(&dcuart->port.state->port); - mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); + spin_lock_irqsave(&dcuart->lock, flags); + got = dcuart_drain_rx(dcuart); + spin_unlock_irqrestore(&dcuart->lock, flags); + + if (got) + tty_flip_buffer_push(&port->state->port); + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); + return; + } + + if (dcuart->sw_drain) { + bool got; + + spin_lock_irqsave(&dcuart->lock, flags); + got = dcuart_drain_rx(dcuart); + spin_unlock_irqrestore(&dcuart->lock, flags); + + if (got) + tty_flip_buffer_push(&port->state->port); + } + + uart_port_lock_irqsave(port, &flags); + pending = dcuart_tx_chars(dcuart, true); + uart_port_unlock_irqrestore(port, flags); + + if (dcuart->sw_drain || pending) + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); } +/* + * Bytes in the FIFO stay there for a host to read whenever it comes; a full + * FIFO means none is reading, not that a transmission is in progress. So + * report empty: a close or tcdrain must not wait for a reader (it waited + * twice the port timeout, 0.4 s, on every close of an unread channel). + */ static unsigned int dcuart_tx_empty(struct uart_port *port) { - return (readl(port->membase + DATA_TX_FREE) != 0) ? - TIOCSER_TEMT : - 0; + if (!port_to_dc(port)->never_wait) + return (readl(port->membase + DATA_TX_FREE) != 0) ? TIOCSER_TEMT : 0; + return TIOCSER_TEMT; } static void dcuart_start_tx(struct uart_port *port) { - char ch; - struct tty_port *tport = &port->state->port; + struct dcuart_port *dcuart = port_to_dc(port); + + if (!dcuart->never_wait) { + struct tty_port *tport = &port->state->port; + char ch; - if (!kfifo_is_empty(&tport->xmit_fifo)) { while (kfifo_get(&tport->xmit_fifo, &ch)) { dcuart_putc(port, ch); port->icount.tx++; } + return; } + + /* The rest goes from the timer, which the RX poll keeps running anyway. */ + if (dcuart_tx_chars(dcuart, false) && !timer_pending(&dcuart->drain_timer)) + mod_timer(&dcuart->drain_timer, jiffies + msecs_to_jiffies(SW_DRAIN_MS)); } static void dcuart_stop_tx(struct uart_port *port) @@ -258,6 +355,14 @@ static int dcuart_startup(struct uart_port *port) struct dcuart_port *dcuart = port_to_dc(port); unsigned long flags; + /* + * A close waits for the xmit buffer to empty, which on an unread + * channel takes until the timer drops it (systemd closes the console + * after every status line: 30 ms each). Never wait for a reader. + */ + if (dcuart->never_wait) + port->state->port.closing_wait = ASYNC_CLOSING_WAIT_NONE; + spin_lock_irqsave(&dcuart->lock, flags); dcuart_irq_disable(dcuart, dcuart->irq_tx | dcuart->irq_rx); writel_relaxed(1, dcuart->config_base + CONFIG_RX_THRESH); @@ -278,9 +383,10 @@ static void dcuart_shutdown(struct uart_port *port) struct dcuart_port *dcuart = port_to_dc(port); unsigned long flags; - if (dcuart->sw_drain) + /* With never_wait the timer also sends TX in IRQ mode. */ + if (dcuart->sw_drain || dcuart->never_wait) timer_delete_sync(&dcuart->drain_timer); - else + if (!dcuart->sw_drain) disable_irq(port->irq); spin_lock_irqsave(&dcuart->lock, flags); @@ -376,6 +482,7 @@ static int dcuart_probe(struct platform_device *pdev) dcuart->irq_tx = IRQ_TX(channel); dcuart->irq_rx = IRQ_RX(channel); dcuart->sw_drain = of_property_read_bool(pdev->dev.of_node, "apple,sw-drain"); + dcuart->never_wait = of_property_read_bool(pdev->dev.of_node, DOCKCHANNEL_NEVER_WAIT); timer_setup(&dcuart->drain_timer, dcuart_drain_timer, 0); dcuart->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); From 9bae877594f50061812ed0a4cbb369f984f51939 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:38:05 +1100 Subject: [PATCH 311/392] soc: apple: sep: Remember the logged lock state per user The enclave alternates notifications for the users it knows (the system user 501 and the Linux users), so comparing with the previous notification alone still logged a repeat of each user's state: nine lines on an M2 boot, three of them the same state of user 501. Keep the last logged state of up to four users and log a notification only when it changes that user's state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 33 +++++++++++++++++++++++++++++---- drivers/soc/apple/sep.rs | 8 ++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index b2847c7f96d80..b1f703cf805ab 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -4360,13 +4360,13 @@ impl SepData { let state = (word >> proto::MSG_TYPE_SHIFT) as u16; let user = (word >> proto::MSG_DATA_SHIFT) as u32; let word = u64::from(state) | (u64::from(user) << 16); - let previous = self.bio_lock_state.xchg(word, Relaxed); + self.bio_lock_state.store(word, Relaxed); self.bio_lock_state_seq.fetch_add(1, Relaxed); let lockout = if state & LOCK_STATE_NEEDS_PASSWORD != 0 { "set" } else { "clear" }; let n = lock_state_names(state); - // The enclave repeats the same state after each query; log a change - // once, and a repeat of the last notification only at debug level. - if previous != word { + // The enclave repeats a user's state after each query; log each + // user's state when it changes, and a repeat only at debug level. + if self.lock_state_changed(word) { dev_info!( self.dev, "sbio: lock state of user {} is {:#06x} (lockout {}:{}{}{}{}{}{}{})\n", @@ -4392,6 +4392,31 @@ impl SepData { self.sbio_wq.notify_all(); } + /// Records `word` (state and user, as in `bio_lock_state`) as the last + /// lock state logged for its user. Returns false if it is that user's + /// last logged state already. A user not in the table takes the slot of + /// one that is, in turn, so a new user is always logged. + fn lock_state_changed(&self, word: u64) -> bool { + let user = word >> 16; + let slots = &self.bio_lock_logged; + let mut free = None; + for (i, slot) in slots.iter().enumerate() { + let seen = slot.load(Relaxed); + if seen == 0 { + free.get_or_insert(i); + } else if seen >> 16 == user { + if seen == word { + return false; + } + slot.store(word, Relaxed); + return true; + } + } + let i = free.unwrap_or_else(|| (self.bio_lock_state_seq.load(Relaxed) as usize) % slots.len()); + slots[i].store(word, Relaxed); + true + } + /// Waits up to `timeout_ms` for a lock-state notification for `user` /// newer than `since` (a `bio_lock_state_seq` value). Returns its state. fn await_lock_state(&self, user: u32, since: u32, timeout_ms: u32) -> Option { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index ff21f59e65b01..a2bd2d5dfb4c7 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -94,6 +94,9 @@ const SETTLE_MS: time::Msecs = 200; /// Messages on an endpoint without a handler are logged this many times per /// boot; diag/unhandled_messages counts them all. const UNHANDLED_LOG_LIMIT: u32 = 4; +/// Users whose last logged lock state is remembered (a Mac has the system +/// user 501 and one or two Linux users). +pub(crate) const LOCK_STATE_LOG_USERS: usize = 4; const FIRST_RESPONSE_MS: time::Msecs = 3000; const RNG_SURVEY_ATTEMPTS: u32 = 5; @@ -827,6 +830,10 @@ struct SepData { // from a stale one. bio_lock_state: Atomic, bio_lock_state_seq: Atomic, + // The last state logged for each of the most recent users, in the same + // layout, so a repeated notification is not logged again. Notifications + // are handled one at a time, on the receive path. + bio_lock_logged: [Atomic; LOCK_STATE_LOG_USERS], #[pin] store: Mutex>, @@ -1092,6 +1099,7 @@ impl SepData { match_scope_misses: Atomic::new(0), bio_lock_state: Atomic::new(0), bio_lock_state_seq: Atomic::new(0), + bio_lock_logged: [Atomic::new(0), Atomic::new(0), Atomic::new(0), Atomic::new(0)], store <- new_mutex!(store), host_store <- new_mutex!(host_store), xarm <- new_mutex!(XarmState::new()), From 448114bec41c58645342d39b362dc70c7a707ac2 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:40:21 +1100 Subject: [PATCH 312/392] wifi: mt7932: complete AWDL KUnit dependencies Enable networking and the 16K arm64 page size required by the Apple PCIe dependency chain in the standalone QEMU KUnit fragment. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/wireless/mediatek/mt7932/awdl-kunit.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config index b2c173277d03a..c6fc8e0f79ca4 100644 --- a/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config +++ b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config @@ -1,7 +1,11 @@ CONFIG_KUNIT=y +CONFIG_NET=y +CONFIG_WLAN=y CONFIG_PCI=y CONFIG_OF=y CONFIG_ARCH_APPLE=y +CONFIG_ARM64_16K_PAGES=y +# CONFIG_ARM64_4K_PAGES is not set CONFIG_IOMMU_SUPPORT=y CONFIG_IOMMU_DMA=y CONFIG_CFG80211=y From 03a7731770cbe862b158aa797d7c622f13de0ebc Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:58:05 +1100 Subject: [PATCH 313/392] dt-bindings: serial: apple,dockchannel-uart: add the T6000 compatible T6000 has the same DockChannel serial channel as T8112: separate data and FIFO configuration windows, a shared interrupt window, and receive data that has to be drained in software. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../devicetree/bindings/serial/apple,dockchannel-uart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml index 5891ba9a6cf13..2583b3a4f7306 100644 --- a/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml +++ b/Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml @@ -23,6 +23,7 @@ properties: - const: apple,dockchannel-uart - items: - enum: + - apple,t6000-dockchannel-uart - apple,t8112-dockchannel-uart - apple,t6020-dockchannel-uart - apple,t6021-dockchannel-uart From 5624f61ed1cbd13be2cf26f71f7d40a45f8c5d69 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:54:42 +1100 Subject: [PATCH 314/392] soc: apple: add read-only debugfs views of debug registers Add root-only debugfs views of the apple,aic-timebase and apple,debug-dock register blocks: a "regs" file that reads exactly the 32-bit words the reader asks for, and a "snapshot" file that reads the first words of the window between two reads of the arch counter, so a counter in the window can be checked against CNTVCT. The driver writes no device register and requests no interrupt; registers are only read when a file is read. Builds with W=1 and is checkpatch --strict clean. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/soc/apple/Kconfig | 12 +++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/dbgview.c | 190 ++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 drivers/soc/apple/dbgview.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 4639f5b8ab9bf..af4afabca59e0 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,18 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DBGVIEW + tristate "Apple SoC read-only debug register views" + depends on ARCH_APPLE || COMPILE_TEST + depends on ARM64 && OF && DEBUG_FS + help + Root-only debugfs views of Apple SoC debug register blocks: the + interrupt controller timebase and the debug dock connection + status. The driver only reads registers, and only when a debugfs + file is read. + + Say m here if you debug an Apple SoC. + config APPLE_DOCKCHANNEL tristate "Apple DockChannel FIFO" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 57d06930401c4..4cb853ad128e4 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DBGVIEW) += apple-dbgview.o +apple-dbgview-y = dbgview.o + obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o apple-dockchannel-y = dockchannel.o diff --git a/drivers/soc/apple/dbgview.c b/drivers/soc/apple/dbgview.c new file mode 100644 index 0000000000000..f9824df5d18be --- /dev/null +++ b/drivers/soc/apple/dbgview.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC read-only debug register views + * + * Root-only debugfs views of register blocks that only debug tools use: + * + * - apple,aic-timebase: the timebase aperture of the interrupt controller + * - apple,debug-dock: the debug dock connection status register + * + * The driver never writes a device register and requests no interrupt. + * Registers are read only when a debugfs file is read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static unsigned int snap_bytes = 0x40; +module_param(snap_bytes, uint, 0444); +MODULE_PARM_DESC(snap_bytes, "Bytes of an MMIO view read by its snapshot file"); + +struct dbgview { + void __iomem *base; + resource_size_t size; + struct dentry *dir; +}; + +static struct dentry *dbgview_root; + +static u64 dbgview_cntvct(void) +{ + return __arch_counter_get_cntvct_stable(); +} + +static ssize_t dbgview_regs_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct dbgview *dv = file->private_data; + loff_t pos = *ppos; + size_t len, i; + u32 *buf; + + if (pos < 0 || !IS_ALIGNED(pos, 4)) + return -EINVAL; + if (pos >= dv->size) + return 0; + + len = min_t(size_t, count, dv->size - pos); + len = min_t(size_t, len, PAGE_SIZE); + len = ALIGN_DOWN(len, 4); + if (!len) + return -EINVAL; + + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (i = 0; i < len / 4; i++) + buf[i] = readl(dv->base + pos + i * 4); + + if (copy_to_user(ubuf, buf, len)) { + kfree(buf); + return -EFAULT; + } + kfree(buf); + + *ppos = pos + len; + return len; +} + +static const struct file_operations dbgview_regs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = dbgview_regs_read, + .llseek = default_llseek, +}; + +static int dbgview_snapshot_show(struct seq_file *s, void *unused) +{ + struct dbgview *dv = s->private; + size_t words = min_t(size_t, snap_bytes, dv->size) / 4; + unsigned long flags; + u64 before, after; + u32 *val; + size_t i; + + val = kzalloc_objs(*val, max_t(size_t, words, 1)); + if (!val) + return -ENOMEM; + + /* Bracket the reads by the arch counter with nothing in between. */ + local_irq_save(flags); + before = dbgview_cntvct(); + for (i = 0; i < words; i++) + val[i] = readl(dv->base + i * 4); + after = dbgview_cntvct(); + local_irq_restore(flags); + + seq_printf(s, "cntvct_before %016llx\n", before); + for (i = 0; i < words; i++) + seq_printf(s, "%04zx %08x\n", i * 4, val[i]); + seq_printf(s, "cntvct_after %016llx\n", after); + seq_printf(s, "cntfrq %u\n", arch_timer_get_cntfrq()); + + kfree(val); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dbgview_snapshot); + +static int dbgview_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct dbgview *dv; + + dv = devm_kzalloc(dev, sizeof(*dv), GFP_KERNEL); + if (!dv) + return -ENOMEM; + + dv->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(dv->base)) + return PTR_ERR(dv->base); + dv->size = resource_size(res); + + dv->dir = debugfs_create_dir(dev_name(dev), dbgview_root); + + debugfs_create_file_size("regs", 0400, dv->dir, dv, + &dbgview_regs_fops, dv->size); + debugfs_create_file("snapshot", 0400, dv->dir, dv, + &dbgview_snapshot_fops); + + platform_set_drvdata(pdev, dv); + return 0; +} + +static void dbgview_remove(struct platform_device *pdev) +{ + struct dbgview *dv = platform_get_drvdata(pdev); + + debugfs_remove_recursive(dv->dir); +} + +static const struct of_device_id dbgview_of_match[] = { + { .compatible = "apple,aic-timebase" }, + { .compatible = "apple,debug-dock" }, + { } +}; +MODULE_DEVICE_TABLE(of, dbgview_of_match); + +static struct platform_driver dbgview_driver = { + .driver = { + .name = "apple-dbgview", + .of_match_table = dbgview_of_match, + }, + .probe = dbgview_probe, + .remove = dbgview_remove, +}; + +static int __init dbgview_init(void) +{ + int ret; + + dbgview_root = debugfs_create_dir("apple-dbgview", NULL); + ret = platform_driver_register(&dbgview_driver); + if (ret) + debugfs_remove_recursive(dbgview_root); + return ret; +} +module_init(dbgview_init); + +static void __exit dbgview_exit(void) +{ + platform_driver_unregister(&dbgview_driver); + debugfs_remove_recursive(dbgview_root); +} +module_exit(dbgview_exit); + +MODULE_AUTHOR("Ryan Murray <81541646+RyanTheTide@users.noreply.github.com>"); +MODULE_DESCRIPTION("Apple SoC read-only debug register views"); +MODULE_LICENSE("Dual MIT/GPL"); From cae57aa17f8c90b8792b06cb4b8aae46c080c4f9 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:54:53 +1100 Subject: [PATCH 315/392] arm64: dts: apple: t8112: expose the DockChannel UART and AIC timebase Enable the DebugUSB DockChannel serial channel as a second, non-console serial port (ttyDC), and the AIC timebase aperture. The console stays on the S5L UART: stdout-path is unchanged, and the DockChannel port is only opened by a user. Tested on J413: ttyDC0 bound and not a console, a TX marker reaches the host, and the timebase word at 0x20 tracks CNTVCT at a 1.0 ratio. The full suite (12 sleep cycles, 3 s2idle RTC wakes) showed no change against the default kernel. dtc W=1 and CHECK_DTBS report nothing for the changed nodes. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-peripherals.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi index c370343fcb668..db97f50178b28 100644 --- a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi @@ -89,7 +89,6 @@ m2_aic_timebase: timer@23b0dc000 { compatible = "apple,t8112-aic-timebase", "apple,aic-timebase"; reg = <0x2 0x3b0dc000 0x0 0x1000>; - status = "disabled"; }; m2_dwi: dwi@23b200000 { @@ -147,7 +146,6 @@ <0x2 0x3d10c000 0x0 0x1000>; interrupt-parent = <&aic>; interrupts = ; - status = "disabled"; apple,sw-drain; }; From 4e50830ecb40905838e56051bf3bb91e7aef464b Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:54:55 +1100 Subject: [PATCH 316/392] arm64: dts: apple: t600x: describe the DockChannel UART as a serial port The DockChannel block at 0x29250c000 carries the DebugUSB serial channel; it has no FIFO bus consumer on these SoCs. Describe it as an apple,dockchannel-uart with its data, FIFO configuration and interrupt windows and software RX drain. On T6000, enable it as a non-console serial port (ttyDC) and enable the AIC timebase aperture. T6001 and T6002 keep the port disabled. Tested on J316s: ttyDC0 bound and not a console, a TX marker reaches the host, and the timebase word at 0x20 tracks CNTVCT at a 1.0 ratio. dtc W=1 and CHECK_DTBS report nothing for the changed nodes. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../boot/dts/apple/t6000-peripherals.dtsi | 21 ++++++++----------- .../boot/dts/apple/t6001-peripherals.dtsi | 17 +++++++-------- .../boot/dts/apple/t6002-peripherals.dtsi | 17 +++++++-------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi index 8dbf14d92da28..0700ad51ec3c1 100644 --- a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT -/* Disabled resource descriptions for T6000. */ +/* Additional T6000 hardware resources. Keep unqualified blocks disabled. */ &{/soc} { ane0_dma: dma-controller@285804000 { @@ -14,7 +14,6 @@ aic_timebase: timer@28e148000 { compatible = "apple,t6000-aic-timebase", "apple,aic-timebase"; reg = <0x2 0x8e148000 0x0 0x1000>; - status = "disabled"; }; dwi: bus-controller@28e208000 { @@ -26,18 +25,16 @@ status = "disabled"; }; - dockchannel_uart: fifo@29250c000 { - compatible = "apple,t6000-dockchannel", "apple,dockchannel"; - reg = <0x2 0x9250c000 0x0 0x18>; + /* The DebugUSB DockChannel serial channel; RX is drained in software. */ + dockchannel_uart: serial@29252c000 { + compatible = "apple,t6000-dockchannel-uart", "apple,dockchannel-uart"; + reg = <0x2 0x9252c000 0x0 0x4000>, + <0x2 0x92528000 0x0 0x4000>, + <0x2 0x9250c000 0x0 0x18>; + reg-names = "data", "config", "irq"; interrupt-parent = <&aic>; interrupts = ; - reg-names = "irq"; - ranges = <0 0x2 0x92528000 0x10004>; - #address-cells = <1>; - #size-cells = <1>; - interrupt-controller; - #interrupt-cells = <2>; - status = "disabled"; + apple,sw-drain; }; apr0: video-codec@311000000 { diff --git a/arch/arm64/boot/dts/apple/t6001-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6001-peripherals.dtsi index 51b94d55a5a9b..8184537da866e 100644 --- a/arch/arm64/boot/dts/apple/t6001-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6001-peripherals.dtsi @@ -26,17 +26,16 @@ status = "disabled"; }; - dockchannel_uart: fifo@29250c000 { - compatible = "apple,t6000-dockchannel", "apple,dockchannel"; - reg = <0x2 0x9250c000 0x0 0x18>; + /* The DebugUSB DockChannel serial channel; RX is drained in software. */ + dockchannel_uart: serial@29252c000 { + compatible = "apple,t6000-dockchannel-uart", "apple,dockchannel-uart"; + reg = <0x2 0x9252c000 0x0 0x4000>, + <0x2 0x92528000 0x0 0x4000>, + <0x2 0x9250c000 0x0 0x18>; + reg-names = "data", "config", "irq"; interrupt-parent = <&aic>; interrupts = ; - reg-names = "irq"; - ranges = <0 0x2 0x92528000 0x10004>; - #address-cells = <1>; - #size-cells = <1>; - interrupt-controller; - #interrupt-cells = <2>; + apple,sw-drain; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t6002-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6002-peripherals.dtsi index ba6616910477e..ad89730134583 100644 --- a/arch/arm64/boot/dts/apple/t6002-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6002-peripherals.dtsi @@ -26,17 +26,16 @@ status = "disabled"; }; - dockchannel_uart: fifo@29250c000 { - compatible = "apple,t6000-dockchannel", "apple,dockchannel"; - reg = <0x2 0x9250c000 0x0 0x18>; + /* The DebugUSB DockChannel serial channel; RX is drained in software. */ + dockchannel_uart: serial@29252c000 { + compatible = "apple,t6000-dockchannel-uart", "apple,dockchannel-uart"; + reg = <0x2 0x9252c000 0x0 0x4000>, + <0x2 0x92528000 0x0 0x4000>, + <0x2 0x9250c000 0x0 0x18>; + reg-names = "data", "config", "irq"; interrupt-parent = <&aic>; interrupts = ; - reg-names = "irq"; - ranges = <0 0x2 0x92528000 0x10004>; - #address-cells = <1>; - #size-cells = <1>; - interrupt-controller; - #interrupt-cells = <2>; + apple,sw-drain; status = "disabled"; }; From dbe881e3847a30413d68d14762ad4842aaac7171 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:31:48 +1100 Subject: [PATCH 317/392] drm/asahi: finish an exiting client's work before its mappings go Killing a Vulkan client with rendering in flight on T8112 (and once on T6000) gave a firmware timeout with a read fault on an unmapped address of the client's VM (UL1C or dVDM_CSD), then recovery. SIGTERM of gpubench-vk render.heavy fill 1080p at depth 4 reproduces it on the first try. The legacy (G13/G14) VMs carry no job lifetime. When the client's DRM file goes away, DRM first closes its GEM handles, and closing a handle unmaps the object from the client's VMs at once; File then drops the VMs, which unmaps the rest of the user range, before its queues. All of it happens while the client's jobs still run in the firmware. Give the queue trait a drain() hook. The legacy queue waits there, for at most 5 s, on the finished fence of its last submitted job; the jobs of one queue complete in order, so that fence covers all of them. The first GEM handle close of an exiting task drains every queue of the file once, and the queues now drop before the VMs and wait the same way (also when destroyed by ioctl). G15 and later VMs keep user mappings until their jobs release them, so their queues keep the no-op drain. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/gpu/drm/asahi/file.rs | 46 ++++++++++++++++++++++++++-- drivers/gpu/drm/asahi/queue/mod.rs | 48 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 244dca39e41ac..3778ee5a4aa68 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -33,6 +33,7 @@ use core::mem::MaybeUninit; use core::ops::Deref; use core::ops::Range; use core::ptr::addr_of_mut; +use core::sync::atomic::{AtomicBool, Ordering}; use kernel::bindings; use kernel::dma_fence::RawDmaFence; use kernel::drm::gem::BaseObject; @@ -269,16 +270,20 @@ pub(crate) struct File { id: u64, /// Available VM range for the user user_range: Range, - // #[pin] - vms: xarray::XArray>, + // Fields drop in order: the queues go first, so that a closing client's submitted work + // can finish before its VMs unmap the user ranges. // #[pin] queues: xarray::XArray>>>, // #[pin] + vms: xarray::XArray>, + // #[pin] objects: xarray::XArray>, /// GPU memory this client created and holds. mem: Arc, /// Completed engine time, retained by queues after handle destruction. usage: Arc, + /// The queues were drained once for an exiting task. + drained: AtomicBool, } /// Convenience type alias for our DRM `File` type. @@ -371,6 +376,7 @@ impl File { (*slot).user_range = user_range; addr_of_mut!((*slot).mem).write(mem); addr_of_mut!((*slot).usage).write(usage); + addr_of_mut!((*slot).drained).write(AtomicBool::new(false)); Ok(()) }) } @@ -1151,7 +1157,43 @@ impl File { Ok(0) } + /// Waits for the submitted work of every queue of `file`, once, when its task exits. + /// + /// An exiting or killed client's DRM file closes its GEM handles before the driver sees + /// the file go, and closing a handle unmaps the object at once from VMs that do not track + /// jobs (G13/G14). Work the client left running would then fault on the unmapped memory + /// and time out in the firmware. A live client unmaps only what its work no longer uses. + fn drain_queues_on_exit(file: &DrmFile) { + // SAFETY: `current` is valid for the running task. + let exiting = + unsafe { (*kernel::task::Task::current_raw()).flags } & bindings::PF_EXITING != 0; + if !exiting || file.inner().drained.swap(true, Ordering::AcqRel) { + return; + } + let mut index = 0; + loop { + // Clone since we can't hold the xarray spinlock while waiting. + let Some((idx, queue)) = + file.inner() + .queues() + .find(index, usize::MAX) + .map(|(idx, guard)| { + let queue: Arc>> = guard.borrow().into(); + (idx, queue) + }) + else { + break; + }; + queue.lock().drain(); + if idx == usize::MAX { + break; + } + index = idx + 1; + } + } + pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result { + Self::drain_queues_on_exit(file); // TODO: use iter() let mut index = 0; loop { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index aa5dc1fba4723..b82abf9602762 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -82,6 +82,10 @@ pub(crate) trait Queue: Send + Sync { cmdbuf_raw: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result; + + /// Waits for the work submitted so far to finish, before the mappings of a closing + /// client go. Queues whose VMs keep user mappings for accepted jobs need not wait. + fn drain(&mut self) {} } #[versions(AGX)] @@ -142,6 +146,8 @@ pub(crate) struct Queue { fence_ctx: FenceContexts, inner: QueueInner::ver, stats: Arc, + /// Finished fence of the last submitted job, waited for when the queue goes away. + last_out: Option, } #[versions(AGX)] @@ -551,6 +557,7 @@ impl Queue::ver { q_comp: None, fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, stats, + last_out: None, inner: QueueInner::ver { dev: dev.clone(), ualloc, @@ -680,8 +687,40 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result( &mut self, reg_data: &AsahiRegData<'drm>, @@ -1008,6 +1047,7 @@ impl Queue for Queue::ver { mod_dev_dbg!(self.dev, "Queue {}: Arming job {}\n", self.inner.id, job.id); let mut job = job.arm(); let out_fence = job.fences().finished(); + self.last_out = Some(out_fence.clone()); mod_dev_dbg!( self.dev, "Queue {}: Pushing job {}\n", @@ -1039,5 +1079,13 @@ impl Queue for Queue::ver { impl Drop for Queue::ver { fn drop(&mut self) { mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.inner.id); + // A client that exits or is killed drops its queues with jobs still queued or running + // on the GPU, and its VMs right after (see `file::File`). Those VMs do not track jobs: + // dropping one unmaps the user range at once, and the firmware then faults on the + // next access of the running job and times out. Let the submitted work finish first. + self.wait_submitted(); } } + +/// How long a closing queue waits for its submitted work to finish. +const QUEUE_CLOSE_WAIT_MS: u32 = 5000; From 10b32030d2b92b0ee2f137e8836a5c321518df7f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:00:51 +1100 Subject: [PATCH 318/392] soc: apple: Add read-only register views The memory cache controller has no functional driver on T8112 and T6000, but its carveout and cache status registers are useful when studying the hardware. Add a driver that exposes a fixed list of registers per block as a root-only debugfs file under /sys/kernel/debug/apple-regview//regs. It never writes to the hardware and does not request the memory regions, since other drivers may own parts of the same windows. The list for the memory cache controller on T8112 and T6000 holds the carveout start, end and enable registers of plane 0 and the cache status register of every plane. The boot loader reads these registers on every boot of these SoCs, so reading them is known to be harmless. Tested with a W=1 build, in tree and as an out-of-tree module against a 7.3 kernel. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/soc/apple/Kconfig | 12 ++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/regview.c | 237 ++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 drivers/soc/apple/regview.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 4639f5b8ab9bf..6081f8ad5c35b 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -238,6 +238,18 @@ config APPLE_T8132_PLATFORM_KUNIT_TEST tree without enabling any Apple hardware drivers. These tests only read metadata and never access the described MMIO resources. +config APPLE_REGVIEW + tristate "Apple SoC read-only register views" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF && DEBUG_FS + help + Exposes fixed lists of registers of Apple SoC blocks that have no + functional driver, such as the memory cache controller, as + read-only files under /sys/kernel/debug/apple-regview. The driver never writes to these + blocks. Only registers known to be safe to read are listed. + + This is a development aid. If unsure, say 'n'. + config APPLE_CLPC tristate "Apple host performance controller floors" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 57d06930401c4..a98ceda2fdd5f 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -56,4 +56,7 @@ apple-mcc-y := mcc.o obj-$(CONFIG_APPLE_CLPC) += clpc.o +obj-$(CONFIG_APPLE_REGVIEW) += apple-regview.o +apple-regview-y := regview.o + obj-$(CONFIG_APPLE_SART_APIF) += apple-sart-apif.o diff --git a/drivers/soc/apple/regview.c b/drivers/soc/apple/regview.c new file mode 100644 index 0000000000000..9bbd6fbf277e8 --- /dev/null +++ b/drivers/soc/apple/regview.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC read-only register views + * + * Exposes fixed, per-block lists of 32-bit registers through debugfs so that + * blocks without a functional driver can be inspected. The driver never + * writes to the hardware. Each list names only registers that are known to + * be safe to read: + * + * - memory cache controller (T8112, T6000): the carveout bounds and enables + * of plane 0 and the cache status of every plane. The boot loader reads + * exactly these registers on every boot of these SoCs. + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* One run of 32-bit registers inside one DT reg entry. */ +struct apple_regview_range { + u8 reg; /* index into the node's reg property */ + u32 offset; /* byte offset from the start of that entry */ + u32 count; /* number of 32-bit registers */ + u32 stride; /* byte distance between registers; 0 means 4 */ + const char *name; +}; + +struct apple_regview_hw { + const struct apple_regview_range *ranges; + unsigned int nr_ranges; + unsigned int nr_regs; /* reg entries the node must have */ +}; + +struct apple_regview_map { + void __iomem *base; + resource_size_t size; +}; + +static struct dentry *apple_regview_root; + +struct apple_regview { + struct device *dev; + const struct apple_regview_hw *hw; + struct apple_regview_map *maps; + struct dentry *dir; +}; + +#define MCC_PLANE_STRIDE 0x40000 +#define MCC_PLANE_CACHE_STATUS 0x1c04 +#define MCC_TZ_BASE 0x6a0 +#define MCC_TZ_STRIDE 0x10 +#define MCC_TZ_REGIONS 4 + +/* + * Plane 0 carveout start, end and enable of each region, then the cache + * status of each plane. + */ +static const struct apple_regview_range apple_regview_t8112_mcc[] = { + /* T8112: reg[1] holds the four planes. */ + { 1, MCC_TZ_BASE + 0x0, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-start" }, + { 1, MCC_TZ_BASE + 0x4, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-end" }, + { 1, MCC_TZ_BASE + 0x8, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-enable" }, + { 1, MCC_PLANE_CACHE_STATUS, 4, MCC_PLANE_STRIDE, "cache-status" }, +}; + +static const struct apple_regview_hw apple_regview_t8112_mcc_hw = { + .ranges = apple_regview_t8112_mcc, + .nr_ranges = ARRAY_SIZE(apple_regview_t8112_mcc), + .nr_regs = 2, +}; + +/* T6000: one reg entry per controller, each with four planes at offset 0. */ +static const struct apple_regview_range apple_regview_t6000_mcc[] = { + { 0, MCC_TZ_BASE + 0x0, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-start" }, + { 0, MCC_TZ_BASE + 0x4, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-end" }, + { 0, MCC_TZ_BASE + 0x8, MCC_TZ_REGIONS, MCC_TZ_STRIDE, "tz-enable" }, + { 0, MCC_PLANE_CACHE_STATUS, 4, MCC_PLANE_STRIDE, "cache-status" }, + { 1, MCC_PLANE_CACHE_STATUS, 4, MCC_PLANE_STRIDE, "cache-status" }, + { 2, MCC_PLANE_CACHE_STATUS, 4, MCC_PLANE_STRIDE, "cache-status" }, + { 3, MCC_PLANE_CACHE_STATUS, 4, MCC_PLANE_STRIDE, "cache-status" }, +}; + +static const struct apple_regview_hw apple_regview_t6000_mcc_hw = { + .ranges = apple_regview_t6000_mcc, + .nr_ranges = ARRAY_SIZE(apple_regview_t6000_mcc), + .nr_regs = 4, +}; + +static u32 apple_regview_stride(const struct apple_regview_range *r) +{ + return r->stride ? r->stride : 4; +} + +static int apple_regview_check(const struct apple_regview *rv) +{ + const struct apple_regview_hw *hw = rv->hw; + unsigned int i; + + for (i = 0; i < hw->nr_ranges; i++) { + const struct apple_regview_range *r = &hw->ranges[i]; + u64 last; + + if (r->reg >= hw->nr_regs || !r->count) + return -EINVAL; + last = r->offset + (u64)(r->count - 1) * apple_regview_stride(r) + 4; + if (last > rv->maps[r->reg].size) + return -EINVAL; + } + + return 0; +} + +static int apple_regview_show(struct seq_file *s, void *unused) +{ + struct apple_regview *rv = s->private; + const struct apple_regview_hw *hw = rv->hw; + unsigned int i, j; + + for (i = 0; i < hw->nr_ranges; i++) { + const struct apple_regview_range *r = &hw->ranges[i]; + const struct apple_regview_map *map = &rv->maps[r->reg]; + u32 stride = apple_regview_stride(r); + + for (j = 0; j < r->count; j++) { + u32 off = r->offset + j * stride; + + seq_printf(s, "%u+0x%07x %s[%u]: 0x%08x\n", + r->reg, off, r->name, j, + readl_relaxed(map->base + off)); + } + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_regview); + +static void apple_regview_remove_debugfs(void *data) +{ + debugfs_remove_recursive(data); +} + +static int apple_regview_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_regview *rv; + unsigned int i; + int ret; + + rv = devm_kzalloc(dev, sizeof(*rv), GFP_KERNEL); + if (!rv) + return -ENOMEM; + + rv->dev = dev; + rv->hw = of_device_get_match_data(dev); + if (!rv->hw) + return -ENODEV; + + rv->maps = devm_kcalloc(dev, rv->hw->nr_regs, sizeof(*rv->maps), + GFP_KERNEL); + if (!rv->maps) + return -ENOMEM; + + for (i = 0; i < rv->hw->nr_regs; i++) { + struct resource res; + + ret = of_address_to_resource(dev->of_node, i, &res); + if (ret) + return dev_err_probe(dev, ret, "missing reg %u\n", i); + if (resource_size(&res) & 3) + return dev_err_probe(dev, -EINVAL, "reg %u not word sized\n", i); + + /* + * Mapping does not touch the hardware. Other drivers may own + * parts of these windows, so they are not requested. + */ + rv->maps[i].base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!rv->maps[i].base) + return -ENOMEM; + rv->maps[i].size = resource_size(&res); + } + + ret = apple_regview_check(rv); + if (ret) + return dev_err_probe(dev, ret, "register list exceeds the apertures\n"); + + rv->dir = debugfs_create_dir(dev_name(dev), apple_regview_root); + debugfs_create_file("regs", 0400, rv->dir, rv, &apple_regview_fops); + + return devm_add_action_or_reset(dev, apple_regview_remove_debugfs, + rv->dir); +} + +static const struct of_device_id apple_regview_of_match[] = { + { .compatible = "apple,t8112-mcc", .data = &apple_regview_t8112_mcc_hw }, + { .compatible = "apple,t6000-mcc", .data = &apple_regview_t6000_mcc_hw }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_regview_of_match); + +static struct platform_driver apple_regview_driver = { + .driver = { + .name = "apple-regview", + .of_match_table = apple_regview_of_match, + }, + .probe = apple_regview_probe, +}; + +static int __init apple_regview_init(void) +{ + int ret; + + apple_regview_root = debugfs_create_dir("apple-regview", NULL); + ret = platform_driver_register(&apple_regview_driver); + if (ret) + debugfs_remove_recursive(apple_regview_root); + return ret; +} +module_init(apple_regview_init); + +static void __exit apple_regview_exit(void) +{ + platform_driver_unregister(&apple_regview_driver); + debugfs_remove_recursive(apple_regview_root); +} +module_exit(apple_regview_exit); + +MODULE_DESCRIPTION("Apple SoC read-only register views"); +MODULE_LICENSE("Dual MIT/GPL"); From 3013fe0e4259d52caaf2d4d2223db5b810fb2386 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:00:52 +1100 Subject: [PATCH 319/392] arm64: dts: apple: t8112, t6000: Enable the MCC node Let the read-only register view bind to the memory cache controller. Tested on J413 and J316s: the view binds, every listed register reads back stable across reads, the carveouts and cache status match what the boot loader set up, and the test suite (J413: full, with 12 sleep cycles; J316s: quick) shows no change against the default kernel. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t6000-peripherals.dtsi | 1 - arch/arm64/boot/dts/apple/t8112-peripherals.dtsi | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi index 8dbf14d92da28..633ecabec7d30 100644 --- a/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6000-peripherals.dtsi @@ -249,7 +249,6 @@ <0x2 0x01000000 0x0 0x1000000>, <0x2 0x02000000 0x0 0x1000000>, <0x2 0x03000000 0x0 0x1000000>; - status = "disabled"; }; soc_events: event-log@28e080000 { diff --git a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi index c370343fcb668..4126826bbaa4f 100644 --- a/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-peripherals.dtsi @@ -51,7 +51,6 @@ <0x2 0x1000000 0x0 0x2000000>, <0x2 0xf00000 0x0 0x100000>, <0x2 0x3b3c0000 0x0 0x20000>; - status = "disabled"; }; m2_aft: fabric-tracker@200110000 { From 860f2cd352219547b1bbb24222ef0fc16a040cf5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:05:38 +1100 Subject: [PATCH 320/392] mmc: sdhci-pci-gli: never use the I2C4 0x20 client for card detection The J316s board describes an unidentified I2C4 client at address 0x20 next to the SD slot. A bare read of an unknown device can change its state, so the power sequencing core documents that card detection never probes, scans or reads it. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- drivers/mmc/host/sdhci-pci-gli-pwrseq.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/mmc/host/sdhci-pci-gli-pwrseq.c b/drivers/mmc/host/sdhci-pci-gli-pwrseq.c index e446b8399ff1a..25dc0fe83eafa 100644 --- a/drivers/mmc/host/sdhci-pci-gli-pwrseq.c +++ b/drivers/mmc/host/sdhci-pci-gli-pwrseq.c @@ -2,6 +2,10 @@ /* * GL9755 board power sequencing core. No hardware adapter is enabled until an * always-on insertion wake source and config-access exclusion are available. + * + * Card detection must never use the unidentified I2C4 client at address 0x20 + * (the board "sd-card" node): no probe, scan, quick command or read of it. + * Insertion comes from the SMC helper interrupt or the GL9755 itself. */ #include #include From e2f51c7ae111f49ea1173806de5dba88b7070a27 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:05:47 +1100 Subject: [PATCH 321/392] arm64: dts: apple: t600x-j314-j316: document the I2C4 0x20 client as do-not-touch The I2C4 client at 0x20 is unidentified, and a bare read of an unknown device can change its state. Say in the DT that it stays disabled and is never probed, scanned or read; SD card detection does not use it. No change to the compiled DTB. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t600x-j314-j316-control.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316-control.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316-control.dtsi index 39de40bcac695..e4415e7d9d405 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316-control.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316-control.dtsi @@ -2,6 +2,11 @@ /* Populated board control clients. Transport ownership is not transferred. */ &i2c4 { + /* + * Unidentified client: a bare read may change its state. Keep it + * disabled, and never probe, scan or read address 0x20. SD card + * detection does not use it. + */ sd_card_control: sd-card-control@20 { compatible = "apple,t6000-sd-card-control", "apple,sd-card-control"; reg = <0x20>; From fb4a1b0408d9add77ba5272104bd01fcde2d3fba Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:05:52 +1100 Subject: [PATCH 322/392] wifi: mt7932: enable netdevices in AWDL KUnit config Select the parent networking device menu so the WLAN and MT7932 symbols are visible to the standalone ARM64 KUnit configuration. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/wireless/mediatek/mt7932/awdl-kunit.config | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config index c6fc8e0f79ca4..c38e5204e0588 100644 --- a/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config +++ b/drivers/net/wireless/mediatek/mt7932/awdl-kunit.config @@ -1,5 +1,6 @@ CONFIG_KUNIT=y CONFIG_NET=y +CONFIG_NETDEVICES=y CONFIG_WLAN=y CONFIG_PCI=y CONFIG_OF=y From 0bc4f95b94c073af04a68b2f9eb77954067220b1 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:25:19 +1100 Subject: [PATCH 323/392] arm64: dts: apple: adt-coverage: t6000-j316s: map the DockChannel UART to its serial node The DockChannel block is now described as serial@29252c000, so the coverage manifest points the ADT dockchannel-uart node there. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt b/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt index 5baa84eeaa229..d1064fbbb2a0d 100644 --- a/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt +++ b/arch/arm64/boot/dts/apple/adt-coverage/t6000-j316s.txt @@ -566,7 +566,7 @@ /arm-io/smc/iop-smc-nub/smc-charger-util - -> /soc/smc@290400000 /arm-io/uart0 0x39b200000 -> /soc/serial@39b200000 /arm-io/uart0/debug-console - -> /soc/serial@39b200000 -/arm-io/dockchannel-uart 0x29250c000 -> /soc/fifo@29250c000 +/arm-io/dockchannel-uart 0x29250c000 -> /soc/serial@29252c000 /arm-io/spi1 0x39b104000 -> /soc/spi@39b104000 /arm-io/spi1/spinor /arm-io/spi1@0x0 -> /soc/spi@39b104000/flash@0 /arm-io/spi1/spinor/firmware - -> WAIVED: Firmware flash partition; parent flash hardware is inventoried separately. From c8b611e112433b6201b97e8557fd01b20322a332 Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:19 -0400 Subject: [PATCH 324/392] net: apple: Add T6050 N1 firmware and ACIPC transport Keep this transport alongside the current J813/J873 implementation. Select it only for a T6050 host, with separate objects and private helper names so both implementations can be built and registered together. Boot the N1 firmware and expose its shared ACIPC transport behind the T6050 PCIe host. After the ROM boot, isolate and quiesce function 0, put it in D3hot with PME enabled, let the host's root-port reset take the link through L2, PERST# and retraining, then restore the root port's and the endpoint's state. Keep firmware and calibration as external inputs, and retain supplier lifetime for the separate Wi-Fi and Bluetooth consumers. Source snapshot: t6050-gpu ee68abb105200e99636a5edfc647a5cfc1d89954 Integration base: 9cbdd0734d9e186c5f5f5949a21b909d10c2571b Signed-off-by: Dj Co-developed-by: Jeff Queisser Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../bindings/net/wireless/apple,centauri.yaml | 4 +- drivers/net/wireless/apple/Kconfig | 12 + drivers/net/wireless/apple/Makefile | 2 +- drivers/net/wireless/apple/centauri/Makefile | 1 + .../wireless/apple/centauri/centauri_pci.c | 6 + .../wireless/apple/centauri/t6050/Makefile | 4 + .../apple/centauri/t6050/centauri_ipc.c | 519 ++++++++++++++++++ .../apple/centauri/t6050/centauri_pci.c | 428 +++++++++++++++ .../apple/centauri/t6050/centauri_t6050.h | 87 +++ 9 files changed, 1060 insertions(+), 3 deletions(-) create mode 100644 drivers/net/wireless/apple/centauri/t6050/Makefile create mode 100644 drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c create mode 100644 drivers/net/wireless/apple/centauri/t6050/centauri_pci.c create mode 100644 drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h diff --git a/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml index 4e46c3573edbf..f73dd9afceca1 100644 --- a/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml +++ b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml @@ -13,8 +13,8 @@ description: | PCI function zero owns the shared N1 firmware bootstrap. The boot producer supplies the personalized FTAB filename and packed firmware platform ID. The endpoint must be left in its stopped ROM personality with DMA and MSI - quiesced. A supported APIF PCI host performs the isolated port reset after - ROM bootstrap. This binding does not authorize adopting an existing live + quiesced. The host's root-port reset retrains the link after the ROM + bootstrap. This binding does not authorize adopting an existing live firmware session or releasing another owner's retained DART streams. properties: diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig index d3ead4d9259e7..344099f427c5e 100644 --- a/drivers/net/wireless/apple/Kconfig +++ b/drivers/net/wireless/apple/Kconfig @@ -22,4 +22,16 @@ config APPLE_CENTAURI This driver does not include Bluetooth HCI support. To build as a module, choose M; the module is apple_centauri. + +config APPLE_CENTAURI_T6050 + tristate "Apple N1 Centauri T6050 firmware and IPC transport" + depends on ARCH_APPLE || COMPILE_TEST + depends on PCI && CFG80211 && PCIE_APPLE + select FW_LOADER + help + Boot the N1 firmware on a T6050 host that provides root-port reset. + The transport starts automatically from its device-tree firmware, + platform and protocol properties. It binds only behind a T6050 host; + the J813/J873 transport remains available independently. + endif diff --git a/drivers/net/wireless/apple/Makefile b/drivers/net/wireless/apple/Makefile index ca72cd0b4da06..969c58e0ebe98 100644 --- a/drivers/net/wireless/apple/Makefile +++ b/drivers/net/wireless/apple/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_APPLE_CENTAURI) += centauri/ +obj-y += centauri/ diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile index 1a17e5283e227..235e1090294dc 100644 --- a/drivers/net/wireless/apple/centauri/Makefile +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o apple_centauri-y := centauri_pci.o centauri_port.o centauri_ipc.o centauri_wifi.o +obj-$(CONFIG_APPLE_CENTAURI_T6050) += t6050/ diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 86ad99c48db2e..57fefbc840c6f 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -197,6 +197,12 @@ static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) dma_addr_t dma; void *page; int ret; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + /* The T6050 transport owns every function behind an adopted host. */ + if (bridge && bridge->dev.parent && + of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie")) + return -ENODEV; c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); if (!c) diff --git a/drivers/net/wireless/apple/centauri/t6050/Makefile b/drivers/net/wireless/apple/centauri/t6050/Makefile new file mode 100644 index 0000000000000..ec75629ea1272 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI_T6050) += apple_centauri_t6050.o +apple_centauri_t6050-y := centauri_pci.o centauri_ipc.o +apple_centauri_t6050-$(CONFIG_APPLE_CENTAURI_T6050_WIFI) += centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c b/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c new file mode 100644 index 0000000000000..2ff1fbf61b85a --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_ipc.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri_t6050.h" + +struct cen_buffer { + struct list_head list; + struct device *owner; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_t6050_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + bool enabled[3], owned[3]; + bool alpha_pm_domain; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + u16 cchi_head, cchi_tail; +}; + +static struct cen_buffer *buffer_alloc(struct centauri_t6050 *c, unsigned int function, + size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + + if (!buffer) + return NULL; + buffer->owner = get_device(&c->ipc->function[function]->dev); + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(buffer->owner, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + put_device(buffer->owner); + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +static int secondary_init(struct centauri_t6050 *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, 0, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + ret = pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf90, required); + if (ret) + return pcibios_err_to_errno(ret); + + return 0; +} + +static int context_init(struct centauri_t6050 *c, struct cen_context *ctx, + unsigned int function, u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) { + pr_err("centauri_t6050: IPC init timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; + } + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); + if (ret) + pr_err("centauri_t6050: IPC run timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri_t6050 *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + return 0; +} + +static int memswap_handoff(struct centauri_t6050 *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, function, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + ret = pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf90, size); + if (ret) + return pcibios_err_to_errno(ret); + return 0; +} + +static struct dev_pm_domain centauri_t6050_alpha_pm_domain; + +static int enumerate_functions(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + int i, ret; + + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + for (i = 0; i < 3; i++) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(PCI_SLOT(c->pdev->devfn), i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (ipc->function[i]->cfg_size < 0xf94 || + !(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM) || + pci_resource_len(ipc->function[i], 0) < (i ? 0x2000 : 0x9080)) + return -ENODEV; + if (i && ipc->function[i]->dev.driver) + return -EBUSY; + ipc->owned[i] = true; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + if (i) { + ret = pci_enable_device_mem(ipc->function[i]); + if (ret) + return ret; + ipc->enabled[i] = true; + } + if (i == 1 && !ipc->function[i]->dev.driver) { + dev_pm_domain_set(&ipc->function[i]->dev, ¢auri_t6050_alpha_pm_domain); + ipc->alpha_pm_domain = true; + } + ret = dma_set_mask_and_coherent(&ipc->function[i]->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + if (i == 1) { + ret = centauri_t6050_request_irqs(&c->alpha_irqs, ipc->function[i], + 16, centauri_t6050_alpha_irq, c); + if (ret) + return ret; + } + pci_set_master(ipc->function[i]); + } + return 0; +} + +static int centauri_t6050_start_ipc_locked(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, &ipc->control, 0, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, 1, CEN_ALPHA_WINDOW_SIZE); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, &ipc->alpha, 1, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(CEN_ALPHA_WINDOW_SIZE, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + ret = centauri_t6050_wifi_register(c); + if (ret) + return ret; + + /* Publish usable shared firmware before the Beta consumer probes. */ + smp_store_release(&c->ready, true); + ret = device_attach(&ipc->function[2]->dev); + if (ret < 0) + dev_warn(&c->pdev->dev, "Beta driver attachment failed: %d\n", ret); + return 0; +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + if (ipc->bar[1]) + dev_err(&c->pdev->dev, "Alpha stage=%#x IPC=%#x window=%#x/%#x/%#x\n", + readl(ipc->bar[1] + 8), readl(ipc->bar[1] + 0xc), + readl(ipc->bar[1] + 0x18), readl(ipc->bar[1] + 0x1c), readl(ipc->bar[1] + 0x20)); + if (ipc->bar[2]) + dev_err(&c->pdev->dev, "Beta stage=%#x IPC=%#x\n", + readl(ipc->bar[2] + 0x18), readl(ipc->bar[2] + 0x1c)); + + return ret; +} + +int centauri_t6050_start_ipc(struct centauri_t6050 *c) +{ + int ret; + + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + ret = READ_ONCE(c->removing) ? -ECANCELED : centauri_t6050_start_ipc_locked(c); + pci_unlock_rescan_remove(); + return ret; +} + +void centauri_t6050_free_ipc(struct centauri_t6050 *c) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i; + + if (!ipc) + return; + if (ipc->owned[2]) + device_release_driver(&ipc->function[2]->dev); + centauri_t6050_wifi_unregister(c); + centauri_t6050_free_irqs(&c->alpha_irqs); + for (i = 0; i < 3; i++) { + if (!ipc->owned[i]) + continue; + pci_clear_master(ipc->function[i]); + } + for (i = 0; i < 3; i++) { + if (ipc->owned[i] && !centauri_t6050_stop_dma(ipc->function[i])) { + dev_warn(&c->pdev->dev, + "retaining IPC DMA after pending transaction timeout\n"); + return; + } + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + memzero_explicit(buffer->cpu, buffer->size); + dma_free_coherent(buffer->owner, buffer->size, buffer->cpu, buffer->dma); + put_device(buffer->owner); + list_del(&buffer->list); + kfree(buffer); + } + if (ipc->alpha_pm_domain) + dev_pm_domain_set(&ipc->function[1]->dev, NULL); + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + if (ipc->enabled[i]) + pci_disable_device(ipc->function[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_t6050_alpha_resources(struct centauri_t6050 *c, struct cen_alpha_resources *resources) +{ + struct centauri_t6050_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->pdev = ipc->function[1]; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c b/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c new file mode 100644 index 0000000000000..9fd9d5bd1d8e8 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_pci.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri_t6050.h" + +static struct pci_driver centauri_t6050_driver; + +bool apple_centauri_control_ready(struct pci_dev *pdev) +{ + struct centauri_t6050 *c; + + if (pdev->driver != ¢auri_t6050_driver) + return false; + c = pci_get_drvdata(pdev); + return c && !READ_ONCE(c->removing) && smp_load_acquire(&c->ready); +} +EXPORT_SYMBOL_GPL(apple_centauri_control_ready); + +/* No DMA backing is released without verified bus-master shutdown. */ +bool centauri_t6050_stop_dma(struct pci_dev *pdev) +{ + u16 command; + int ret; + + pci_clear_master(pdev); + ret = pci_read_config_word(pdev, PCI_COMMAND, &command); + if (ret || (command & PCI_COMMAND_MASTER)) + return false; + return pci_wait_for_pending_transaction(pdev); +} + +void centauri_t6050_free_irqs(struct centauri_t6050_irqs *irqs) +{ + if (!irqs->pdev) + return; + while (irqs->count) + free_irq(pci_irq_vector(irqs->pdev, --irqs->count), irqs->data); + pci_free_irq_vectors(irqs->pdev); + irqs->pdev = NULL; +} + +int centauri_t6050_request_irqs(struct centauri_t6050_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data) +{ + int ret; + + ret = pci_alloc_irq_vectors(pdev, count, count, PCI_IRQ_MSI); + if (ret < 0) + return ret; + irqs->pdev = pdev; + irqs->data = data; + for (irqs->count = 0; irqs->count < count; irqs->count++) { + ret = request_irq(pci_irq_vector(pdev, irqs->count), handler, 0, + "centauri_t6050", data); + if (ret) { + centauri_t6050_free_irqs(irqs); + return ret; + } + } + return 0; +} + +static irqreturn_t centauri_t6050_control_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} + +static void centauri_t6050_free_boot(struct centauri_t6050 *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + while (c->alpha_irqs.count) + free_irq(pci_irq_vector(c->alpha_irqs.pdev, --c->alpha_irqs.count), c); + while (c->irqs.count) + free_irq(pci_irq_vector(c->pdev, --c->irqs.count), c); + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + + centauri_t6050_free_ipc(c); + centauri_t6050_free_irqs(&c->irqs); + if (c->ipc || !centauri_t6050_stop_dma(c->pdev)) { + dev_warn(dev, "retaining boot DMA until outstanding transactions stop\n"); + return; + } + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + c->firmware = NULL; + c->descriptor = NULL; +} + +static int centauri_t6050_rom_boot(struct centauri_t6050 *c) +{ + const struct firmware *fw; + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if (c->pdev->device != 0x1900 && c->pdev->device != 0x1901) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0) + return -EBUSY; + + ret = request_firmware_direct(&fw, c->firmware_name, dev); + if (ret) { + dev_err(dev, "required firmware %s unavailable: %d\n", c->firmware_name, ret); + return ret; + } + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + ret = centauri_t6050_request_irqs(&c->irqs, c->pdev, 4, + centauri_t6050_control_irq, c); + if (ret) + goto free_buffers; + pci_set_master(c->pdev); + writel(c->platform_id, c->bar + 0x807c); + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + READ_ONCE(c->removing) || c->image_response != 0, 20000, 20000000); + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (!ret && c->image_response != 1) + ret = -EIO; + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, + READ_ONCE(c->removing) || stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + if (READ_ONCE(c->removing)) + ret = -ECANCELED; + dma_rmb(); + + if (!ret && stage != 4) + ret = -EIO; + goto release; +free_buffers: + centauri_t6050_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +/* + * After the ROM boot the N1 runs its firmware from host memory and must come back on a + * retrained link: the isolated function 0 is quiesced, put in D3hot with PME enabled + * (the host's PME_Turn_Off then takes the link to L2), the host resets and retrains the + * root port (PERST# asserted and released around it; the firmware keeps running), and the + * root port's and the endpoint's state are restored. + */ +static int centauri_t6050_port_cycle(struct pci_dev *endpoint) +{ + struct pci_host_bridge *host = pci_find_host_bridge(endpoint->bus); + struct pci_dev *bridge = endpoint->bus->self, *sibling; + u16 command, pmctrl; + u32 identity; + int ret, pm; + + if (!host->reset_root_port || !bridge || !pci_is_root_bus(bridge->bus)) + return -EOPNOTSUPP; + pm = endpoint->pm_cap; + if (!pm) + return -ENODEV; + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + list_for_each_entry(sibling, &endpoint->bus->devices, bus_list) { + if (sibling != endpoint) { + ret = -EBUSY; + goto out; + } + } + ret = pci_save_state(bridge) ?: pci_save_state(endpoint); + if (ret) + goto out; + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, PCI_COMMAND, &command)); + if (ret) + goto out; + if (!centauri_t6050_stop_dma(endpoint)) { + ret = -EBUSY; + goto stopped; + } + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, pm + PCI_PM_CTRL, &pmctrl)); + if (ret) + goto stopped; + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + (pmctrl & ~PCI_PM_CTRL_STATE_MASK) | PCI_D3hot | PCI_PM_CTRL_PME_ENABLE)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + + ret = host->reset_root_port(host, bridge); + if (ret) + goto stopped; + pci_restore_state(bridge); + ret = pcibios_err_to_errno(pci_read_config_dword(endpoint, PCI_VENDOR_ID, &identity)); + if (!ret && identity != 0x1900106b && identity != 0x1901106b) + ret = -ENODEV; + if (ret) + goto stopped; + pci_restore_state(endpoint); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + pmctrl & ~PCI_PM_CTRL_STATE_MASK)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, PCI_COMMAND, command)); + if (!ret) + goto out; +stopped: + /* A partial transition never restores endpoint DMA or bridge decoding. */ + pci_clear_master(endpoint); + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_err(endpoint, "N1 port cycle stopped: %d\n", ret); +out: + pci_unlock_rescan_remove(); + return ret; +} + +static int centauri_t6050_cycle(struct centauri_t6050 *c) +{ + int ret; + + if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) + return -EBUSY; + c->link_ready = false; + ret = centauri_t6050_port_cycle(c->pdev); + if (ret == -EAGAIN) { + c->link_ready = true; + } else if (!ret) { + pci_restore_msi_state(c->pdev); + c->link_ready = true; + } + return ret; +} + +static void centauri_t6050_boot_worker(struct work_struct *work) +{ + struct centauri_t6050 *c = container_of(to_delayed_work(work), struct centauri_t6050, boot_work); + bool retry = false; + int ret; + + mutex_lock(&c->boot_lock); + while (!READ_ONCE(c->removing)) { + switch (c->boot_phase) { + case CEN_BOOT_ROM: + ret = centauri_t6050_rom_boot(c); + if (!ret) + c->boot_phase = readl(c->bar + 0x8000) == 1 ? + CEN_BOOT_IPC : CEN_BOOT_CYCLE; + break; + case CEN_BOOT_CYCLE: + ret = centauri_t6050_cycle(c); + if (!ret) + c->boot_phase = CEN_BOOT_IPC; + break; + case CEN_BOOT_IPC: + ret = centauri_t6050_start_ipc(c); + if (!ret) + c->boot_phase = CEN_BOOT_READY; + break; + default: + goto out; + } + if (ret == -EAGAIN && (c->boot_phase == CEN_BOOT_CYCLE || + (c->boot_phase == CEN_BOOT_IPC && !c->ipc))) { + retry = true; + break; + } + if (ret) { + dev_err(&c->pdev->dev, "N1 automatic startup failed in phase%u: %d\n", + c->boot_phase, ret); + c->boot_phase = CEN_BOOT_FAILED; + break; + } + } +out: + if (READ_ONCE(c->removing)) + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); + if (retry && !READ_ONCE(c->removing)) + queue_delayed_work(system_long_wq, &c->boot_work, msecs_to_jiffies(20)); +} + +static int centauri_t6050_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri_t6050 *c; + int ret; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + if (!bridge || !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie")) + return -ENODEV; + if (PCI_FUNC(pdev->devfn) != 0) + return -ENODEV; + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + INIT_DELAYED_WORK(&c->boot_work, centauri_t6050_boot_worker); + c->boot_phase = CEN_BOOT_ROM; + ret = device_property_read_string(&pdev->dev, "firmware-name", &c->firmware_name); + if (ret) + return dev_err_probe(&pdev->dev, ret, "missing personalized firmware name\n"); + ret = device_property_read_u32(&pdev->dev, "apple,centauri-platform-id", &c->platform_id); + if (ret) + return ret; + ret = device_property_read_u32(&pdev->dev, "apple,centauri-protocol", &c->protocol); + if (ret || c->protocol != 27) + return -EINVAL; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < 0x9080) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri_t6050"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + queue_delayed_work(system_long_wq, &c->boot_work, 0); + return 0; +} + +static void centauri_t6050_remove(struct pci_dev *pdev) +{ + struct centauri_t6050 *c = pci_get_drvdata(pdev); + + WRITE_ONCE(c->removing, true); + cancel_delayed_work_sync(&c->boot_work); + mutex_lock(&c->boot_lock); + centauri_t6050_wifi_unregister(c); + centauri_t6050_free_boot(c); + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); +} + +static int centauri_t6050_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct centauri_t6050 *c = pci_get_drvdata(pdev); + + guard(mutex)(&c->boot_lock); + if (c->boot_phase != CEN_BOOT_READY) + return -EBUSY; + return pci_save_state(pdev); +} + +static int centauri_t6050_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_t6050_pm_ops, centauri_t6050_suspend, centauri_t6050_resume); + +static const struct pci_device_id centauri_t6050_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900) }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901) }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_t6050_ids); + +static struct pci_driver centauri_t6050_driver = { + .name = "apple_centauri_t6050", + .id_table = centauri_t6050_ids, + .probe = centauri_t6050_probe, + .remove = centauri_t6050_remove, + .shutdown = centauri_t6050_remove, + .driver.pm = pm_sleep_ptr(¢auri_t6050_pm_ops), +}; + +module_pci_driver(centauri_t6050_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri PCI firmware and ACIPC transport"); diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h b/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h new file mode 100644 index 0000000000000..dfc9c013596fb --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_t6050.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef AURORA_CENTAURI_T6050_H +#define AURORA_CENTAURI_T6050_H +#include +#include +#include +#include +#include +#include + +#define CEN_ALPHA_WINDOW_SIZE SZ_32M +#define CEN_ALPHA_SCRATCH_OFFSET SZ_8M +#define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) + +struct centauri_t6050_ipc; +struct centauri_t6050_wifi; +struct centauri_t6050_irqs { + struct pci_dev *pdev; + void *data; + unsigned int count; +}; + +struct cen_alpha_resources { + struct pci_dev *pdev; + void *context, *messages, *window; + dma_addr_t context_dma, messages_dma, window_dma; + void __iomem *bar; + u32 index[4], mtr, mcr; +}; +enum centauri_t6050_boot_phase { + CEN_BOOT_ROM, + CEN_BOOT_CYCLE, + CEN_BOOT_IPC, + CEN_BOOT_READY, + CEN_BOOT_FAILED, + CEN_BOOT_STOPPED, +}; + +struct centauri_t6050 { + struct pci_dev *pdev; + void __iomem *bar; + resource_size_t size; + u32 platform_id, protocol; + const char *firmware_name; + struct dentry *debug; + struct mutex boot_lock; + struct delayed_work boot_work; + enum centauri_t6050_boot_phase boot_phase; + bool removing, ready; + void *firmware, *descriptor; + struct centauri_t6050_irqs irqs, alpha_irqs; + dma_addr_t firmware_dma, descriptor_dma; + size_t firmware_size; + u32 image_response; + bool boot_attempted; + bool link_ready; + struct centauri_t6050_ipc *ipc; + struct centauri_t6050_wifi *wifi; +}; + +int centauri_t6050_request_irqs(struct centauri_t6050_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data); +void centauri_t6050_free_irqs(struct centauri_t6050_irqs *irqs); + +int centauri_t6050_start_ipc(struct centauri_t6050 *c); +void centauri_t6050_free_ipc(struct centauri_t6050 *c); +int centauri_t6050_alpha_resources(struct centauri_t6050 *c, struct cen_alpha_resources *resources); +bool centauri_t6050_stop_dma(struct pci_dev *pdev); + +#if IS_ENABLED(CONFIG_APPLE_CENTAURI_T6050_WIFI) +irqreturn_t centauri_t6050_alpha_irq(int irq, void *data); +int centauri_t6050_wifi_register(struct centauri_t6050 *c); +void centauri_t6050_wifi_unregister(struct centauri_t6050 *c); +#else +static inline irqreturn_t centauri_t6050_alpha_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} +static inline int centauri_t6050_wifi_register(struct centauri_t6050 *c) +{ + return 0; +} +static inline void centauri_t6050_wifi_unregister(struct centauri_t6050 *c) +{ +} +#endif +#endif From 6ff5f42b7248e0085439fc8cb64934ea3c1a3283 Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:19 -0400 Subject: [PATCH 325/392] wifi: apple: Add N1 protocol-27 station support Add the N1 protocol-27 cfg80211 station interface, Alpha command/data rings, scanning, and WPA2-PSK/CCMP connections. The shared firmware and PCIe transport are supplied by the sibling core PR. Source snapshot: t6050-gpu ee68abb105200e99636a5edfc647a5cfc1d89954 Integration base: 9cbdd0734d9e186c5f5f5949a21b909d10c2571b Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/net/wireless/apple/Kconfig | 7 + .../apple/centauri/t6050/centauri_wifi.c | 1736 +++++++++++++++++ 2 files changed, 1743 insertions(+) create mode 100644 drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig index 344099f427c5e..d9b13a9785de2 100644 --- a/drivers/net/wireless/apple/Kconfig +++ b/drivers/net/wireless/apple/Kconfig @@ -34,4 +34,11 @@ config APPLE_CENTAURI_T6050 platform and protocol properties. It binds only behind a T6050 host; the J813/J873 transport remains available independently. +config APPLE_CENTAURI_T6050_WIFI + bool "T6050 N1 Centauri cfg80211 station interface" + depends on APPLE_CENTAURI_T6050 + default y + help + Add scanning and station connections for the T6050 protocol profile. + endif diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c new file mode 100644 index 0000000000000..6779271fc83f4 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -0,0 +1,1736 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri_t6050.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 + +#define DATA_RX_BUFFER SZ_16M +#define DATA_TX_BUFFER (SZ_16M + SZ_8M) +#define DATA_RX_SLOTS 2048 +#define DATA_TX_SLOTS 2048 +#define DATA_TX_QUEUE_LIMIT DATA_TX_SLOTS + +static_assert(DATA_TX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER + DATA_RX_SLOTS * 4096 <= DATA_TX_BUFFER); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= + CEN_ALPHA_WINDOW_SIZE); + +struct centauri_t6050_wifi { + struct centauri_t6050 *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct work_struct link_work; + bool irq_ready; + struct cfg80211_scan_request *scan_request; + struct mutex lock; /* Serializes firmware commands and ring consumers. */ + + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + bool stopping, command_failed; + bool scan_abort, connect_abort; + u8 scan_sequence; + bool connecting, connected, connect_event, disconnect_event; + bool disconnect_pending; + u16 connect_status; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + size_t connect_size, request_ie_size; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + s8 station_signal; + bool station_signal_valid; + unsigned long station_signal_at, connected_at; + struct net_device_stats station_start; + u8 country[3]; + bool country_valid; + u32 data_rings_open; + bool data_failed; + struct ieee80211_supported_band bands[2]; + struct ieee80211_channel channels[39]; +}; + +static int query_link_status(struct centauri_t6050_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const u16 five_ghz[] = { + 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, + 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, + { .bitrate = 110 }, { .bitrate = 60 }, { .bitrate = 90 }, + { .bitrate = 120 }, { .bitrate = 180 }, { .bitrate = 240 }, + { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; + +static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK }; + +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_AES_CMAC, +}; + +static u16 index_read(struct centauri_t6050_wifi *wifi, unsigned int array, + unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*( + __le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_t6050_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE( + *(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), + cpu_to_le16(value)); +} + +static void completions(struct centauri_t6050_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_t6050_wifi *wifi, const void *message, + size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, + descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_t6050_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, + cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, + tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_t6050_wifi *wifi, const u8 *payload, + size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band; + + if (!wifi->registered || size < 0x60 + 36) + return; + + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && + !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; + ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || + ie[0] == WLAN_EID_HT_OPERATION) && + ie[1]) + number = ie[2]; + } + if (!number) + return; + band = number <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max_t(s8, payload[0x16], payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, + frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_t6050_wifi *wifi, unsigned int ring, + u16 slot) +{ + u8 *descriptor = + wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +static bool data_rx_frame(const u8 *packet, unsigned int capacity, + unsigned int length, u32 footer, unsigned int *offset, + unsigned int *frame_length) +{ + unsigned int padding = (footer >> 12) & 15; + + if (!(footer & BIT(7))) + return false; + if (length < padding + ETH_HLEN || length > capacity) + return false; + if (footer & BIT(24)) { + if (capacity < 4) + return false; + *offset = get_unaligned_le16(packet + 2); + } else { + *offset = padding; + } + *frame_length = length - padding; + return *offset <= capacity && *frame_length <= capacity - *offset; +} + +static void drain_data_rx(struct centauri_t6050_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_rings_open & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= DATA_RX_SLOTS || tail >= DATA_RX_SLOTS || + post >= DATA_RX_SLOTS) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < (DATA_RX_SLOTS - 1)) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % DATA_RX_SLOTS; + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || + cookie > (DATA_RX_SLOTS - 1) || length > 4096 || + get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int offset, frame_length; + struct sk_buff *skb = NULL; + + if (data_rx_frame(packet, 4096, length, footer, &offset, + &frame_length)) + skb = netdev_alloc_skb_ip_align(wifi->netdev, + frame_length); + if (skb) { + skb_put_data(skb, packet + offset, + frame_length); + skb->protocol = + eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += frame_length; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % DATA_RX_SLOTS; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +static bool data_tx_completion_span(u16 cookie, u16 wire_count, u8 flags, + unsigned int *first, u16 *count) +{ + bool counted = (flags & BIT(0)) || (flags & 0x0e) == 8; + + if (!cookie || cookie > DATA_TX_SLOTS || counted != !!wire_count || + wire_count >= DATA_TX_SLOTS) + return false; + *count = counted ? wire_count : 1; + *first = (cookie + DATA_TX_SLOTS - *count) % DATA_TX_SLOTS; + return true; +} + +static void drain_data_tx(struct centauri_t6050_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = DATA_TX_SLOTS; + bool posted = false, completed = false; + + if (wifi->command_failed || !(wifi->data_rings_open & BIT(5)) || + readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + if (!data_tx_completion_span(cookie, count, entry[1], + &first, &count)) { + wifi->data_tx_bad++; + break; + } + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % + DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += + wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +static bool link_signal_read(const u8 *packet, size_t size, s8 *signal) +{ + size_t pos = 32; + + while (pos <= size && size - pos >= 4) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + + if (length > size - pos - 4) + return false; + if (tag == 268 && length >= 115) { + *signal = (s8)packet[pos + 5]; + return *signal != 0; + } + pos += 4 + length; + } + return false; +} + +static void drain(struct centauri_t6050_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command, wire_command; + u8 *descriptor = wifi->a.window + TR_BASE + + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + wire_command = get_unaligned_le32(packet + 0x14); + command = wire_command & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) + wifi->scans_done++; + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000003 || + command == 0x80000004) + wifi->disconnect_event = true; + if (command == 0x80000004) + wifi->disconnect_pending = false; + + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + + wifi->reply_command = wire_command; + if (command == 10) { + wifi->country_valid = !status && + size >= 27; + if (wifi->country_valid) + memcpy(wifi->country, + packet + 24, 3); + } + + if (command == 20 && !status) { + wifi->station_signal_valid = + link_signal_read( + packet, size, + &wifi->station_signal); + wifi->station_signal_at = jiffies; + } + } +refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = + (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +/* Caller holds wifi->lock. Unknown completion keeps DMA payloads owned. */ +static void command_fault(struct centauri_t6050_wifi *wifi) +{ + wifi->command_failed = true; + wifi->configured = false; + wifi->data_failed = true; + netif_stop_queue(wifi->netdev); + if (smp_load_acquire(&wifi->irq_ready)) + schedule_work(&wifi->link_work); +} + +static bool command_payload_consumed(struct centauri_t6050_wifi *wifi, u16 slot) +{ + if (slot >= ring_size[1] || + index_read(wifi, 1, 1) != (slot + 1) % ring_size[1]) + return false; + dma_rmb(); + return true; +} + +static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload, + size_t size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + u32 stage; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (wifi->command_failed) + return -EIO; + if (size > BUFFER_SIZE - 24) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + if (READ_ONCE(wifi->stopping) || + READ_ONCE(wifi->control->removing)) { + command_fault(wifi); + return -ECANCELED; + } + drain(wifi); + stage = readl(wifi->a.bar + 8); + if (stage != 2) { + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x lost transport: stage=%#x sequence=%u rx=%u\n", + id, stage, sequence, wifi->rx_seen[2]); + command_fault(wifi); + return -EIO; + } + + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + return wifi->reply_status ? -EREMOTEIO : 0; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + command_fault(wifi); + return -ETIMEDOUT; +} + +static int station_configure(struct centauri_t6050_wifi *wifi, const u8 *address) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + int ret; + + ret = command(wifi, 1, address, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + return ret; +} + +static int radio_start(struct centauri_t6050_wifi *wifi) +{ + unsigned int ring, slot; + int ret; + + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = station_configure(wifi, wifi->netdev->dev_addr); + if (ret) + return ret; + msleep(1000); + wifi->configured = true; + return 0; +} + +#define CEN_SCAN_CHANNELS_MAX 59 +static int scan_profile27(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_scan_request *request = wifi->scan_request; + u8 scan[14 + 5 + 4 * CEN_SCAN_CHANNELS_MAX + 9 + 4 + 2 + 32]; + unsigned int next = 0; + + if (!request || !request->n_channels || request->n_ssids > 1) + return -EINVAL; + while (next < request->n_channels) { + u8 channels[1 + 4 * CEN_SCAN_CHANNELS_MAX] = {}; + u8 options[5] = { 0, 0, 1, 0, 0 }; + unsigned int count = 0; + u32 done = wifi->scans_done; + size_t size = 14; + unsigned long deadline; + int ret; + bool abort_sent = false; + + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + memset(scan, 0, sizeof(scan)); + scan[0] = (++wifi->scan_sequence) & 0x1f; + scan[3] = 0x80; + scan[5] = 2; + scan[7] = 0x10; + + put_unaligned_le16(40, scan + 8); + put_unaligned_le16(40, scan + 10); + put_unaligned_le16(110, scan + 12); + while (next < request->n_channels && + count < CEN_SCAN_CHANNELS_MAX) { + struct ieee80211_channel *channel = + request->channels[next++]; + unsigned int number; + + /* The passive-scan wire mode is not established for this profile. */ + if (channel->flags & + (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR)) + continue; + if (channel->band != NL80211_BAND_2GHZ && + channel->band != NL80211_BAND_5GHZ) + return -EOPNOTSUPP; + number = ieee80211_frequency_to_channel( + channel->center_freq); + if (!number || number > 255) + return -EINVAL; + channels[1 + 4 * count] = number; + channels[2 + 4 * count] = channel->band == + NL80211_BAND_5GHZ; + count++; + } + if (!count) + continue; + channels[0] = count; + put_unaligned_le16(260, scan + size); + put_unaligned_le16(1 + 4 * count, scan + size + 2); + memcpy(scan + size + 4, channels, 1 + 4 * count); + size += 5 + 4 * count; + put_unaligned_le16(357, scan + size); + put_unaligned_le16(sizeof(options), scan + size + 2); + memcpy(scan + size + 4, options, sizeof(options)); + size += 4 + sizeof(options); + if (request->n_ssids) { + const struct cfg80211_ssid *ssid = &request->ssids[0]; + + if (ssid->ssid_len > 32) + return -EINVAL; + + scan[2] = !!ssid->ssid_len; + put_unaligned_le16(258, scan + size); + put_unaligned_le16(2 + ssid->ssid_len, scan + size + 2); + scan[size + 4] = ssid->ssid_len ? 5 : 4; + scan[size + 5] = ssid->ssid_len; + memcpy(scan + size + 6, ssid->ssid, ssid->ssid_len); + size += 6 + ssid->ssid_len; + } + ret = command(wifi, 65551, scan, size); + if (ret) + return ret; + deadline = jiffies + + msecs_to_jiffies(max(8000U, count * 220 + 2000)); + do { + drain(wifi); + if (READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2) + return -EIO; + if (wifi->scans_done != done) + break; + if (READ_ONCE(wifi->scan_abort) && !abort_sent) { + ret = command(wifi, 65537, NULL, 0); + if (ret) + return ret; + abort_sent = true; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (wifi->scans_done == done) { + command(wifi, 65537, NULL, 0); + return -ETIMEDOUT; + } + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + } + return 0; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->scan_abort) || READ_ONCE(wifi->stopping)) { + ret = -ECANCELED; + goto completed; + } + { + ret = scan_profile27(wifi); + goto completed; + } +completed: + info.aborted = !!ret || READ_ONCE(wifi->scan_abort) || + READ_ONCE(wifi->stopping); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) + ret = -ENODEV; + else if (wifi->command_failed) + ret = -EIO; + else if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting || + wifi->disconnect_pending) + ret = -EBUSY; + else { + WRITE_ONCE(wifi->scan_abort, false); + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + + if (wdev == &wifi->wdev) + WRITE_ONCE(wifi->scan_abort, true); +} + +static void cancel_scan(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_scan_info info = { .aborted = true }; + struct cfg80211_scan_request *request; + + WRITE_ONCE(wifi->scan_abort, true); + cancel_work_sync(&wifi->scan_work); + mutex_lock(&wifi->lock); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + + if (request) + cfg80211_scan_done(request, &info); +} + +static void cancel_connect_locked(struct centauri_t6050_wifi *wifi) +{ + if (!wifi->connecting) + return; + wifi->connecting = false; + wifi->connected = false; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +static int rsn_psk_offset(const u8 *body, size_t size) +{ + size_t offset, end; + u16 count; + unsigned int i; + + if (size < 8 || get_unaligned_le16(body) != 1) + return -EINVAL; + count = get_unaligned_le16(body + 6); + if (!count) + return -EINVAL; + offset = 8 + 4 * count; + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + end = offset + 2 + 4 * count; + if (!count || end + 2 > size) + return -EINVAL; + for (i = 0; i < count; i++) + if (get_unaligned_be32(body + offset + 2 + 4 * i) == + WLAN_AKM_SUITE_PSK) + return offset; + return -EOPNOTSUPP; +} + +static int select_psk_rsn(const struct cfg80211_crypto_settings *crypto, + const u8 *body, size_t size, u8 *selected) +{ + size_t tail; + unsigned int i; + int offset; + + for (i = 0; i < crypto->n_akm_suites; i++) + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK) + break; + if (i == crypto->n_akm_suites) + return -EOPNOTSUPP; + offset = rsn_psk_offset(body, size); + if (offset < 0) + return offset; + + if (get_unaligned_be32(body + 2) != WLAN_CIPHER_SUITE_CCMP || + get_unaligned_le16(body + 6) != 1 || + get_unaligned_be32(body + 8) != WLAN_CIPHER_SUITE_CCMP) + return -EOPNOTSUPP; + tail = offset + 2 + 4 * get_unaligned_le16(body + offset); + memcpy(selected, body, offset); + put_unaligned_le16(1, selected + offset); + put_unaligned_be32(WLAN_AKM_SUITE_PSK, selected + offset + 2); + memcpy(selected + offset + 6, body + tail, size - tail); + return offset + 6 + size - tail; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, + u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + static const u8 mld_defaults[14] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 3, 0, 0, + }; + + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + const struct cfg80211_bss_ies *ies; + const u8 *rsn, *ap_rsn; + u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size, rsn_prefix, rsn_tail; + u16 caps = 0; + int ret = 0, offset; + + if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || + params->crypto.n_ciphers_pairwise != 1 || + params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || + params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn) + return -EINVAL; + ret = select_psk_rsn(¶ms->crypto, rsn + 2, rsn[1], rsn_body); + if (ret < 0) + return ret; + rsn_size = ret; + ret = 0; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured || wifi->connecting || wifi->connected || + wifi->scan_request || wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + + bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, + params->ssid, params->ssid_len, + IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + + rcu_read_lock(); + ies = rcu_dereference(bss->ies); + ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : + NULL; + if (!ap_rsn) { + ret = -EINVAL; + } else { + ret = rsn_psk_offset(ap_rsn + 2, ap_rsn[1]); + if (ret >= 0) { + offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); + caps = get_unaligned_le16(ap_rsn + 2 + offset); + ret = 0; + } + } + rcu_read_unlock(); + if (ret) { + cfg80211_put_bss(wiphy, bss); + goto out; + } + offset = rsn_caps_offset(rsn_body, rsn_size); + put_unaligned_le16(caps, rsn_body + offset); + pmf = !!(caps & BIT(7)); + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, bss->bssid, ETH_ALEN); + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(bss->channel->center_freq); + + payload[8] = bss->channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[0x47] = 4; + + put_unaligned_le16(4, payload + 0x48); + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + + append_tlv(payload, &size, 304, zero, 1); + append_tlv(payload, &size, 63763, connection_timeouts, + sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + + append_tlv(payload, &size, 316, &pmf, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + + rsn_prefix = rsn - params->ie; + rsn_tail = rsn_prefix + 2 + rsn[1]; + memcpy(wifi->request_ie, params->ie, rsn_prefix); + wifi->request_ie[rsn_prefix] = WLAN_EID_RSN; + wifi->request_ie[rsn_prefix + 1] = rsn_size; + memcpy(wifi->request_ie + rsn_prefix + 2, rsn_body, rsn_size); + memcpy(wifi->request_ie + rsn_prefix + 2 + rsn_size, + params->ie + rsn_tail, params->ie_len - rsn_tail); + wifi->request_ie_size = params->ie_len - rsn[1] + rsn_size; + WRITE_ONCE(wifi->connect_abort, false); + wifi->connecting = true; + wifi->connect_status = 0; + wifi->connect_event = wifi->disconnect_event = false; + cfg80211_put_bss(wiphy, bss); + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, connect_work); + unsigned long deadline; + u16 slot, status; + int ret; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) { + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + return; + } + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && + !READ_ONCE(wifi->connect_abort) && !READ_ONCE(wifi->stopping) && + time_before(jiffies, deadline)) { + drain(wifi); + + if (wifi->disconnect_event) + ret = -ENOTCONN; + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) + ret = -ECANCELED; + else if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + status = !ret && wifi->connect_status == 0 ? + WLAN_STATUS_SUCCESS : + WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + wifi->station_signal_valid = false; + if (wifi->connected) { + wifi->connected_at = jiffies; + wifi->station_signal_at = jiffies - HZ; + wifi->station_start = wifi->netdev->stats; + } + + if (command_payload_consumed(wifi, slot)) + memzero_explicit(wifi->a.window + TX_BUFFER + + slot * BUFFER_SIZE, + BUFFER_SIZE); + + if (ret == -ECANCELED || ret == -ETIMEDOUT) + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, + wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + else + cfg80211_connect_result(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, + NULL, 0, status, GFP_KERNEL); + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + cfg80211_port_authorized(wifi->netdev, wifi->bssid, NULL, 0, + GFP_KERNEL); + netif_carrier_on(wifi->netdev); + if (wifi->data_rings_open & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_work(&wifi->link_work); + } + mutex_unlock(&wifi->lock); +} + +irqreturn_t centauri_t6050_alpha_irq(int irq, void *data) +{ + struct centauri_t6050 *c = data; + struct centauri_t6050_wifi *wifi = smp_load_acquire(&c->wifi); + + if (wifi && smp_load_acquire(&wifi->irq_ready) && + !READ_ONCE(wifi->stopping)) + schedule_work(&wifi->link_work); + return IRQ_HANDLED; +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_t6050_wifi *wifi = + container_of(work, struct centauri_t6050_wifi, link_work); + bool disconnected, was_connected; + + mutex_lock(&wifi->lock); + was_connected = wifi->connected; + drain(wifi); + disconnected = wifi->command_failed || wifi->disconnect_event || + READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + if (disconnected && was_connected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, false, GFP_KERNEL); + } + mutex_unlock(&wifi->lock); +} + +static int wait_disconnect(struct centauri_t6050_wifi *wifi) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + + while (wifi->disconnect_pending) { + drain(wifi); + if (READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2) + return -EIO; + if (!wifi->disconnect_pending) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(500, 1000); + } + return 0; +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, + u16 reason) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + u16 reply_status; + int ret; + + WRITE_ONCE(wifi->connect_abort, true); + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + + put_unaligned_le32(10, payload + 12); + wifi->disconnect_pending = !READ_ONCE(wifi->stopping); + ret = READ_ONCE(wifi->stopping) ? + 0 : + command(wifi, 131073, payload, sizeof(payload)); + reply_status = wifi->reply_status; + if (!ret && wifi->disconnect_pending && (wifi->reply_command & BIT(30))) + ret = wait_disconnect(wifi); + else if (ret != -EIO && ret != -ETIMEDOUT) + wifi->disconnect_pending = false; + + connected = wifi->connected; + wifi->connected = false; + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, + GFP_KERNEL); + mutex_unlock(&wifi->lock); + cancel_work_sync(&wifi->link_work); + return ret == -EREMOTEIO && reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : + -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, + struct regulatory_request *request) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + int ret; + + if (country[0] < 'A' || country[0] > 'Z' || country[1] < 'A' || + country[1] > 'Z') + return; + mutex_lock(&wifi->lock); + if (wifi->configured) { + ret = command(wifi, 9, country, sizeof(country)); + if (ret) + dev_err(&wifi->control->pdev->dev, + "firmware country update failed: %d\n", ret); + } + mutex_unlock(&wifi->lock); +} + +static void fill_station(struct centauri_t6050_wifi *wifi, struct station_info *sinfo) +{ + const struct net_device_stats *stats = &wifi->netdev->stats; + + if (time_after_eq(jiffies, wifi->station_signal_at + HZ)) { + wifi->station_signal_valid = false; + wifi->station_signal_at = jiffies; + query_link_status(wifi); + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + sinfo->connected_time = (jiffies - wifi->connected_at) / HZ; + sinfo->rx_bytes = stats->rx_bytes - wifi->station_start.rx_bytes; + sinfo->tx_bytes = stats->tx_bytes - wifi->station_start.tx_bytes; + sinfo->rx_packets = stats->rx_packets - wifi->station_start.rx_packets; + sinfo->tx_packets = stats->tx_packets - wifi->station_start.tx_packets; + if (wifi->station_signal_valid) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = wifi->station_signal; + } +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping) || !ether_addr_equal(mac, wifi->bssid)) + ret = -ENOENT; + else + fill_station(wifi, sinfo); + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int index, u8 *mac, struct station_info *sinfo) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (index || wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping)) + ret = -ENOENT; + else { + ether_addr_copy(mac, wifi->bssid); + fill_station(wifi, sinfo); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .abort_scan = wifi_abort_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .dump_station = wifi_dump_station, +}; + +static int data_start(struct centauri_t6050_wifi *wifi); + +static int net_open(struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + int ret; + + netif_carrier_off(netdev); + netif_stop_queue(netdev); + mutex_lock(&wifi->lock); + ret = wifi->configured ? data_start(wifi) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + + cancel_scan(wifi); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + schedule_work(&wifi->link_work); + return NETDEV_TX_OK; +} + +static int station_reconfigure(struct centauri_t6050_wifi *wifi, const u8 *address) +{ + const u8 stop[5] = {}; + int ret; + + wifi->configured = false; + ret = command(wifi, 3, stop, sizeof(stop)); + if (!ret) + ret = station_configure(wifi, address); + if (!ret) + wifi->configured = true; + return ret; +} + +static int net_set_mac_address(struct net_device *netdev, void *data) +{ + struct centauri_t6050_wifi *wifi = + *(struct centauri_t6050_wifi **)netdev_priv(netdev); + struct sockaddr *address = data; + int ret, restore; + + ret = eth_prepare_mac_addr_change(netdev, data); + if (ret) + return ret; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured) { + ret = -ENETDOWN; + goto out; + } + if (wifi->connecting || wifi->connected || wifi->scan_request || + wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + if (ether_addr_equal(netdev->dev_addr, address->sa_data)) + goto out; + + ret = station_reconfigure(wifi, (const u8 *)address->sa_data); + if (!ret) { + eth_commit_mac_addr_change(netdev, data); + } else { + restore = station_reconfigure(wifi, netdev->dev_addr); + if (restore) + dev_err(&wifi->control->pdev->dev, + "Wi-Fi MAC restore failed: %d (change: %d)\n", + restore, ret); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_mac_address = net_set_mac_address, +}; + +static int data_ring_open(struct centauri_t6050_wifi *wifi, unsigned int id) +{ + u8 message[0x34 + 10] = { 0 }; + unsigned int vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + lockdep_assert_held(&wifi->lock); + if (wifi->data_failed) + return -EIO; + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = DATA_RX_SLOTS; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = id == 12 ? DATA_RX_SLOTS : DATA_TX_SLOTS; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto out; + } + if (wifi->data_rings_open & BIT(bit)) + return 0; + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (ret) { + wifi->data_failed = true; + goto out; + } + wifi->data_rings_open |= BIT(bit); + if (transfer && id == 12) { + unsigned int slot; + + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +out: + return ret; +} + +static int data_start(struct centauri_t6050_wifi *wifi) +{ + static const unsigned int rings[] = { 76, 15, 12, 25 }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(rings); i++) { + ret = data_ring_open(wifi, rings[i]); + if (ret) + return ret; + } + return 0; +} + +static int query_link_status(struct centauri_t6050_wifi *wifi) +{ + return wifi->configured ? command(wifi, 20, NULL, 0) : -ENETDOWN; +} + +int centauri_t6050_wifi_register(struct centauri_t6050 *c) +{ + struct centauri_t6050_wifi *wifi; + u8 address[ETH_ALEN]; + unsigned int i; + int ret; + + wifi = kzalloc_obj(*wifi); + if (!wifi) + return -ENOMEM; + wifi->control = c; + mutex_init(&wifi->lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_WORK(&wifi->link_work, link_worker); + smp_store_release(&c->wifi, wifi); + ret = centauri_t6050_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_t6050_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_t6050_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_t6050_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_t6050_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wifi->wiphy->akm_suites = akm_suites; + wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); + wiphy_ext_feature_set(wifi->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + for (i = 0; i < 13; i++) { + wifi->channels[i].band = NL80211_BAND_2GHZ; + wifi->channels[i].hw_value = i + 1; + wifi->channels[i].center_freq = ieee80211_channel_to_frequency( + i + 1, NL80211_BAND_2GHZ); + wifi->channels[i].max_power = 20; + } + for (i = 0; i < ARRAY_SIZE(five_ghz); i++) { + wifi->channels[13 + i].band = NL80211_BAND_5GHZ; + wifi->channels[13 + i].hw_value = five_ghz[i]; + wifi->channels[13 + i].center_freq = + ieee80211_channel_to_frequency(five_ghz[i], + NL80211_BAND_5GHZ); + wifi->channels[13 + i].max_power = 20; + } + wifi->bands[0].channels = wifi->channels; + wifi->bands[0].n_channels = 13; + wifi->bands[0].bitrates = rates; + wifi->bands[0].n_bitrates = ARRAY_SIZE(rates); + wifi->bands[1].channels = wifi->channels + 13; + wifi->bands[1].n_channels = ARRAY_SIZE(five_ghz); + wifi->bands[1].bitrates = rates + 4; + wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + + ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); + if (!ret) { + eth_hw_addr_set(wifi->netdev, address); + ether_addr_copy(wifi->wiphy->perm_addr, address); + } else if (ret == -EPROBE_DEFER) { + goto failed; + } else { + eth_hw_addr_random(wifi->netdev); + dev_warn( + &wifi->a.pdev->dev, + "No valid factory MAC property; using a random address\n"); + } + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, + "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + smp_store_release(&wifi->irq_ready, true); + schedule_work(&wifi->link_work); + + return 0; +failed: + centauri_t6050_wifi_unregister(c); + return ret; +} + +void centauri_t6050_wifi_unregister(struct centauri_t6050 *c) +{ + struct centauri_t6050_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + WRITE_ONCE(wifi->connect_abort, true); + WRITE_ONCE(wifi->scan_abort, true); + smp_store_release(&c->wifi, NULL); + + for (unsigned int i = 0; i < c->alpha_irqs.count; i++) + synchronize_irq(pci_irq_vector(c->alpha_irqs.pdev, i)); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + cancel_work_sync(&wifi->link_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); +} From 260b0bacee6015b8d2fc6bfc9afda2f633ad05b6 Mon Sep 17 00:00:00 2001 From: Dj Date: Mon, 26 Oct 2026 02:34:20 -0400 Subject: [PATCH 326/392] Bluetooth: Add T6050 N1 protocol-27 HCI and ACL transport Keep the J813/J873 transport intact. Give the T6050 driver its own module name and reject other hosts before creating a supplier link. Add the N1 protocol-27 Beta HCI/ACL transport and its Control-supplier lifetime. Carry the bounded HCI command-cache reset hook while preserving standard status and command-credit semantics. Source snapshot: t6050-gpu ee68abb105200e99636a5edfc647a5cfc1d89954 Integration base: 9cbdd0734d9e186c5f5f5949a21b909d10c2571b Signed-off-by: Dj Developed with Claude Code. Signed-off-by: Erik Bourget --- .../net/bluetooth/apple,centauri.yaml | 38 + drivers/bluetooth/Kconfig | 8 + drivers/bluetooth/Makefile | 1 + drivers/bluetooth/btcentauri_t6050.c | 942 ++++++++++++++++++ include/net/bluetooth/hci_core.h | 2 + net/bluetooth/hci_sync.c | 9 + 6 files changed, 1000 insertions(+) create mode 100644 Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml create mode 100644 drivers/bluetooth/btcentauri_t6050.c diff --git a/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml new file mode 100644 index 0000000000000..f8a1895145da4 --- /dev/null +++ b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/bluetooth/apple,centauri.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple N1 Centauri Beta Bluetooth PCI function + +maintainers: + - Dj + +description: | + PCI function two is the Beta Bluetooth HCI/ACL transport. It depends on + the Control function in the same PCI slot, which boots the shared N1 + firmware using the protocol profile described on its own DT node. + The factory address is supplied in network byte order for the firmware + address-provisioning command; HCI returns the standard Bluetooth order. + +properties: + compatible: + const: pci106b,1903 + + reg: + maxItems: 1 + description: Standard five-cell PCI function address + + local-mac-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address in network byte order + +required: + - compatible + - reg + - local-mac-address + +additionalProperties: false diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 832e2d3019ecc..002a8985ca04f 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -520,4 +520,12 @@ config BT_CENTAURI Say M to build the btcentauri module for bring-up testing. +config BT_HCICENTAURI + tristate "Apple N1 Centauri T6050 Bluetooth" + depends on PCI && APPLE_CENTAURI_T6050 + help + Protocol-27 Beta HCI and ACL transport using the T6050 Control + supplier and a device-tree factory address. This can coexist with + the J813/J873 Bluetooth transport. SCO and ISO are not implemented. + endmenu diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index eb3b4de799a3b..4f2f47e39eb43 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -59,3 +59,4 @@ obj-$(CONFIG_BT_CENTAURI) += btcentauri.o obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o CONTEXT_ANALYSIS := y +obj-$(CONFIG_BT_HCICENTAURI) += btcentauri_t6050.o diff --git a/drivers/bluetooth/btcentauri_t6050.c b/drivers/bluetooth/btcentauri_t6050.c new file mode 100644 index 0000000000000..543ca329a17bc --- /dev/null +++ b/drivers/bluetooth/btcentauri_t6050.c @@ -0,0 +1,942 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_CREATE_CONN 0xff7b +#define CEN_OP_FLUSH_DUPLICATES 0xff3f +#define CEN_PROP_CREATE_CONN BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, { 3, 9, 724, false }, { 5, 9, 260, false }, + { 6, 5, 1024, false }, { 8, 9, 2592, true }, { 9, 9, 724, true }, + { 11, 9, 268, true }, { 12, 128, 1024, true }, { 13, 17, 260, true }, + { 14, 9, 1304, false }, { 15, 9, 1304, true }, +}; + +struct cen_aux_state { + u32 offset; + u16 head, post; + bool opened; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct work_struct io_work; + unsigned int irq_count; + bool irq_vectors; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + bool ipc_requested, running; + u8 native_startup_steps; + bool connection_command_supported, connection_status_pending; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, + next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, 100, 1500000, + false, bt, 0); + if (ret) + return ret; + + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, 100, 1500000, + false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, + u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = + bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static bool cen_aux_required(u16 id) +{ + return id == 2 || id == 8; +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + if (!cen_aux_required(spec->id)) + continue; + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, + spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, + "Auxiliary ring %u open failed: %d\n", spec->id, + ret); + return ret; + } + state->opened = true; + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!state->opened || !spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + if (repost && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], + bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, + CEN_LE_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + return cen_aux_start(bt); +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, u16 ring, + u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->connection_command_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + put_unaligned_le16(CEN_OP_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->connection_status_pending = true; + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +static void cen_fixup_scan_response(u8 *event, size_t length) +{ + struct hci_ev_le_ext_adv_info *info; + size_t offset = HCI_EVENT_HDR_SIZE + 2; + u8 count; + + if (length < offset || event[0] != HCI_EV_LE_META || + event[HCI_EVENT_HDR_SIZE] != HCI_EV_LE_EXT_ADV_REPORT) + return; + count = event[HCI_EVENT_HDR_SIZE + 1]; + while (count-- && length - offset >= sizeof(*info)) { + u16 type; + + info = (void *)(event + offset); + if (info->length > length - offset - sizeof(*info)) + return; + type = get_unaligned_le16(&info->type); + if ((type & LE_EXT_ADV_EVT_TYPE_MASK) == + (LE_EXT_ADV_LEGACY_PDU | LE_EXT_ADV_SCAN_RSP)) + put_unaligned_le16(type | LE_EXT_ADV_SCAN_IND, + &info->type); + offset += sizeof(*info) + info->length; + } +} + +static void cen_io_work(struct work_struct *work) +{ + struct cen_bt *bt = container_of(work, struct cen_bt, io_work); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + if (readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (cen_aux_drain(bt, true)) + goto protocol_error; + { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + if (cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + cen_fixup_scan_response(skb->data, skb->len); + + if (bt->connection_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == + CEN_OP_CREATE_CONN) { + put_unaligned_le16(HCI_OP_CREATE_CONN, + skb->data + 4); + bt->connection_status_pending = false; + } + + if (READ_ONCE(bt->connection_command_supported) && + length == + HCI_EVENT_HDR_SIZE + + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + skb->data[11] = ACL_LINK; + } + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER, + &bt->tx_head)) + goto protocol_error; + + if ((cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, + "Beta transport stopped (exec=%u ipc=%u); interrupt processing stopped\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_work(&bt->io_work); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->connection_status_pending = false; + mutex_unlock(&bt->lock); + cancel_work_sync(&bt->io_work); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_reset_dup_filter(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + static const u8 flags; + int ret; + + ret = __hci_cmd_sync_status(hdev, CEN_OP_FLUSH_DUPLICATES, + sizeof(flags), &flags, HCI_CMD_TIMEOUT); + if (ret) { + bt_dev_warn(hdev, "Duplicate filter reset failed: %d", ret); + return ret; + } + return 0; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + WRITE_ONCE(bt->connection_command_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + if (!ret && !(features & CEN_PROP_CREATE_CONN)) + ret = -EOPNOTSUPP; + if (ret) { + bt_dev_warn(hdev, "Firmware connection command unavailable: %d", + ret); + return 0; + } + WRITE_ONCE(bt->connection_command_supported, true); + return 0; +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + if (device_property_count_u8(&bt->pdev->dev, "local-mac-address") != 6) + return -EINVAL; + ret = device_property_read_u8_array(&bt->pdev->dev, "local-mac-address", + address, sizeof(address)); + if (ret) + return ret; + if (is_zero_ether_addr(address) || is_broadcast_ether_addr(address)) + return -EINVAL; + ret = get_random_bytes_wait(seed, sizeof(seed)); + if (ret) + return ret; + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err( + hdev, + "Firmware initialization stopped after %u commands: %d", + bt->native_startup_steps, ret); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && + (skb->len < HCI_COMMAND_HDR_SIZE || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2])) + return -EMSGSIZE; + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) { + u16 opcode = get_unaligned_le16(skb->data); + + if (opcode == CEN_OP_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + schedule_work(&bt->io_work); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + + return ret; +} + +static irqreturn_t cen_irq(int irq, void *data) +{ + struct cen_bt *bt = data; + + if (READ_ONCE(bt->running)) + schedule_work(&bt->io_work); + return IRQ_HANDLED; +} + +static void cen_free_irqs(struct cen_bt *bt) +{ + while (bt->irq_count) + free_irq(pci_irq_vector(bt->pdev, --bt->irq_count), bt); + if (bt->irq_vectors) { + pci_free_irq_vectors(bt->pdev); + bt->irq_vectors = false; + } +} + +static int cen_request_irqs(struct cen_bt *bt) +{ + int ret = pci_alloc_irq_vectors(bt->pdev, 2, 2, PCI_IRQ_MSI); + + if (ret < 0) + return ret; + bt->irq_vectors = true; + for (bt->irq_count = 0; bt->irq_count < 2; bt->irq_count++) { + ret = request_irq(pci_irq_vector(bt->pdev, bt->irq_count), + cen_irq, 0, "btcentauri_t6050", bt); + if (ret) { + cen_free_irqs(bt); + return ret; + } + } + return 0; +} + +static void cen_stop(struct cen_bt *bt) +{ + u16 command; + int ret; + + pci_clear_master(bt->pdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + + ret = pci_read_config_word(bt->pdev, PCI_COMMAND, &command); + if (bt->ipc_requested || ret || (command & PCI_COMMAND_MASTER) || + !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn( + &bt->pdev->dev, + "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, + bt->dma); +} + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + struct pci_dev *control; + struct device_link *link; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + if (!bridge || !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie") || + PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + control = pci_get_domain_bus_and_slot( + pci_domain_nr(pdev->bus), pdev->bus->number, + PCI_DEVFN(PCI_SLOT(pdev->devfn), 0)); + if (!control) + return -EPROBE_DEFER; + link = device_link_add(&pdev->dev, &control->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + pci_dev_put(control); + return -ENOMEM; + } + ret = apple_centauri_control_ready(control) ? 0 : -EPROBE_DEFER; + pci_dev_put(control); + if (ret) + return ret; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + INIT_WORK(&bt->io_work, cen_io_work); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri_t6050"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe( + &pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, + GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + ret = cen_request_irqs(bt); + if (ret) + goto stop; + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", + ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + hdev->reset_dup_filter = cen_hci_reset_dup_filter; + + hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +static int cen_suspend(struct device *dev) +{ + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + {} +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri_t6050", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 protocol-27 Bluetooth transport"); diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 68ff959be8cab..0502328425df6 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -666,6 +666,8 @@ struct hci_dev { struct bt_codec *codec, __u8 *vnd_len, __u8 **vnd_data); u8 (*classify_pkt_type)(struct hci_dev *hdev, struct sk_buff *skb); + /* Called in command-sync context before enabling duplicate-filtered scanning. */ + int (*reset_dup_filter)(struct hci_dev *hdev); }; #define hci_set_quirk(hdev, nr) set_bit((nr), (hdev)->quirk_flags) diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 0b4dfda2bbed8..34027b50941ab 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -2306,6 +2306,15 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, { struct hci_cp_le_set_scan_enable cp; + /* Firmware duplicate caches may need an explicit reset between scans. */ + if (val && filter_dup != LE_SCAN_FILTER_DUP_DISABLE && + !hci_dev_test_flag(hdev, HCI_MESH) && hdev->reset_dup_filter) { + int err = hdev->reset_dup_filter(hdev); + + if (err) + return err; + } + if (use_ext_scan(hdev)) return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup); From 039a2dfccde96dde4aab3507271e9dabf4bea6c7 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Wed, 4 Nov 2026 19:50:12 -0500 Subject: [PATCH 327/392] Bluetooth: btcentauri: Drop an unused variable cen_hci_reset_dup_filter() never uses the driver data it looks up, and clang warns about it. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/bluetooth/btcentauri_t6050.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/bluetooth/btcentauri_t6050.c b/drivers/bluetooth/btcentauri_t6050.c index 543ca329a17bc..12efa0d0fd2d1 100644 --- a/drivers/bluetooth/btcentauri_t6050.c +++ b/drivers/bluetooth/btcentauri_t6050.c @@ -581,7 +581,6 @@ static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, static int cen_hci_reset_dup_filter(struct hci_dev *hdev) { - struct cen_bt *bt = hci_get_drvdata(hdev); static const u8 flags; int ret; From cd1748d180f49424757dd4b49d59c8ada0b3d39d Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 11:59:40 -0800 Subject: [PATCH 328/392] Bluetooth: btcentauri: Take the factory address from local-bd-address Boot loaders fill the standard local-bd-address property from the bluetooth0 alias. Take the factory address from it (least significant byte first), falling back to local-mac-address, and let the binding accept either. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../net/bluetooth/apple,centauri.yaml | 12 +++++++- drivers/bluetooth/btcentauri_t6050.c | 28 ++++++++++++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml index f8a1895145da4..6b1a33cac7aa4 100644 --- a/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml +++ b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml @@ -24,15 +24,25 @@ properties: maxItems: 1 description: Standard five-cell PCI function address + local-bd-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address, least significant byte + first (the standard Bluetooth property a boot loader fills) + local-mac-address: $ref: /schemas/types.yaml#/definitions/uint8-array minItems: 6 maxItems: 6 description: Factory Bluetooth public address in network byte order +oneOf: + - required: [local-bd-address] + - required: [local-mac-address] + required: - compatible - reg - - local-mac-address additionalProperties: false diff --git a/drivers/bluetooth/btcentauri_t6050.c b/drivers/bluetooth/btcentauri_t6050.c index 12efa0d0fd2d1..1d701d218a512 100644 --- a/drivers/bluetooth/btcentauri_t6050.c +++ b/drivers/bluetooth/btcentauri_t6050.c @@ -619,16 +619,36 @@ static int cen_hci_post_init(struct hci_dev *hdev) return 0; } +/* + * The factory address in network byte order, for the firmware's provisioning command: from + * the standard local-bd-address (least significant byte first, as boot loaders fill it from + * the bluetooth0 alias), else from local-mac-address (already in network order). + */ +static int cen_bt_address(struct device *dev, u8 address[6]) +{ + u8 bd[6]; + int i, ret; + + if (device_property_count_u8(dev, "local-bd-address") == 6) { + ret = device_property_read_u8_array(dev, "local-bd-address", bd, sizeof(bd)); + if (ret) + return ret; + for (i = 0; i < 6; i++) + address[i] = bd[5 - i]; + return 0; + } + if (device_property_count_u8(dev, "local-mac-address") != 6) + return -EINVAL; + return device_property_read_u8_array(dev, "local-mac-address", address, 6); +} + static int cen_hci_setup(struct hci_dev *hdev) { struct cen_bt *bt = hci_get_drvdata(hdev); u8 address[6], init_done[16], seed[24]; int ret; - if (device_property_count_u8(&bt->pdev->dev, "local-mac-address") != 6) - return -EINVAL; - ret = device_property_read_u8_array(&bt->pdev->dev, "local-mac-address", - address, sizeof(address)); + ret = cen_bt_address(&bt->pdev->dev, address); if (ret) return ret; if (is_zero_ether_addr(address) || is_broadcast_ether_addr(address)) From 992d14dd687432c9551aedfe26da22d7f9ae6a9f Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 12:42:00 -0800 Subject: [PATCH 329/392] wifi: apple: centauri: Connect to the BSS the request names A connect request carries the BSSID and channel the client chose from its own scan results. The driver looked the target up in cfg80211's scan list instead and refused the connect with -ENOENT when the entry was not there. iwd runs a requested scan as channel subsets, each with the flush flag, and at the end of each subset cfg80211 expires every BSS that subset did not see, on every band. After a last subset of 2.4 GHz channels with no access point, the list holds none of the network's BSSes while iwd's own list holds all of them, so every connect right after such a scan failed until a later unflushed scan refilled cfg80211's list. - The target is the request's BSSID and channel, or its hints. The scan entry is looked up only when the request names neither. - The RSN capabilities sent to the firmware are the station's own, from the request's RSN element (the client's MFP choice), rather than the AP's from the scan entry. - A successful connect is reported against cfg80211's entry for the target. When the client's scans expired it, one is informed from the request (BSSID, channel, SSID), and the AP's beacons, which the firmware reports once associated, fill in its elements. - Scan and connect failures name the firmware's status: a refused command, a connect or disconnect event with a non-zero status, a scan whose completion event carries one, or no event at all. command() still returns -EREMOTEIO for a refusal. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../apple/centauri/t6050/centauri_wifi.c | 164 +++++++++++++----- 1 file changed, 122 insertions(+), 42 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c index 6779271fc83f4..be1040b513f0b 100644 --- a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -54,9 +54,11 @@ struct centauri_t6050_wifi { u8 scan_sequence; bool connecting, connected, connect_event, disconnect_event; bool disconnect_pending; - u16 connect_status; + u16 connect_status, scan_status, disconnect_status; + struct ieee80211_channel *connect_channel; u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; - size_t connect_size, request_ie_size; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + size_t connect_size, request_ie_size, ssid_len; u32 data_rx_received, data_rx_reposted, data_rx_bad; struct sk_buff_head data_tx_queue; u16 data_tx_length[DATA_TX_SLOTS]; @@ -512,15 +514,19 @@ static void drain(struct centauri_t6050_wifi *wifi) } else if (command & BIT(31)) { if (command == 0x80000030) wifi->channels_seen++; - if (command == 0x80000005) + if (command == 0x80000005) { wifi->scans_done++; + wifi->scan_status = status; + } if (command == 0x80000000 && wifi->connecting) { wifi->connect_event = true; wifi->connect_status = status; } if (command == 0x80000003 || - command == 0x80000004) + command == 0x80000004) { wifi->disconnect_event = true; + wifi->disconnect_status = status; + } if (command == 0x80000004) wifi->disconnect_pending = false; @@ -632,7 +638,12 @@ static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload if (wifi->reply_valid && wifi->reply_sequence == sequence && (wifi->reply_command & ~BIT(30)) == id) { - return wifi->reply_status ? -EREMOTEIO : 0; + if (!wifi->reply_status) + return 0; + dev_dbg(&wifi->control->pdev->dev, + "Wi-Fi command %#x: firmware status %u\n", id, + wifi->reply_status); + return -EREMOTEIO; } usleep_range(500, 1000); } while (time_before(jiffies, deadline)); @@ -642,6 +653,19 @@ static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload return -ETIMEDOUT; } +/* Caller holds wifi->lock; ret is what command() returned for the request. */ +static void command_refused(struct centauri_t6050_wifi *wifi, const char *what, + int ret) +{ + if (ret == -EREMOTEIO) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi %s refused: firmware status %u\n", what, + wifi->reply_status); + else + dev_warn(&wifi->control->pdev->dev, "Wi-Fi %s failed: %d\n", + what, ret); +} + static int station_configure(struct centauri_t6050_wifi *wifi, const u8 *address) { u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; @@ -762,8 +786,10 @@ static int scan_profile27(struct centauri_t6050_wifi *wifi) size += 6 + ssid->ssid_len; } ret = command(wifi, 65551, scan, size); - if (ret) + if (ret) { + command_refused(wifi, "scan", ret); return ret; + } deadline = jiffies + msecs_to_jiffies(max(8000U, count * 220 + 2000)); do { @@ -782,9 +808,16 @@ static int scan_profile27(struct centauri_t6050_wifi *wifi) usleep_range(1000, 2000); } while (time_before(jiffies, deadline)); if (wifi->scans_done == done) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels: no completion event\n", + count); command(wifi, 65537, NULL, 0); return -ETIMEDOUT; } + if (wifi->scan_status && !abort_sent) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels ended with firmware status %u\n", + count, wifi->scan_status); if (READ_ONCE(wifi->scan_abort)) return -ECANCELED; } @@ -959,6 +992,13 @@ static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, *size += 4 + length; } +/* + * The connect request names its target: nl80211 clients pass the BSSID and + * channel they chose from their own scan results. The kernel's scan entry for + * it may already be gone (a client's flushed scan expires every BSS its last + * channel subset did not see), so the entry is looked up only when the + * request names no target. + */ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_connect_params *params) { @@ -973,13 +1013,14 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, }; struct centauri_t6050_wifi *wifi = *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); + const u8 *bssid = params->bssid ?: params->bssid_hint; + struct ieee80211_channel *channel = + params->channel ?: params->channel_hint; struct cfg80211_bss *bss; - const struct cfg80211_bss_ies *ies; - const u8 *rsn, *ap_rsn; + const u8 *rsn; u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; u8 *payload = wifi->connect_payload; size_t size = 130, rsn_size, rsn_prefix, rsn_tail; - u16 caps = 0; int ret = 0, offset; if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || @@ -995,6 +1036,12 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, if (ret < 0) return ret; rsn_size = ret; + /* The station's own RSN capabilities go to the firmware; bit 7 is MFPC. */ + offset = rsn_caps_offset(rsn_body, rsn_size); + if (offset < 0) + return -EINVAL; + pmf = !!(get_unaligned_le16(rsn_body + offset) & + BIT(7)); ret = 0; mutex_lock(&wifi->lock); if (READ_ONCE(wifi->stopping)) { @@ -1007,43 +1054,34 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, goto out; } - bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, - params->ssid, params->ssid_len, - IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); - if (!bss) { - ret = -ENOENT; - goto out; - } - - rcu_read_lock(); - ies = rcu_dereference(bss->ies); - ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : - NULL; - if (!ap_rsn) { - ret = -EINVAL; + if (bssid && channel) { + memcpy(wifi->bssid, bssid, ETH_ALEN); } else { - ret = rsn_psk_offset(ap_rsn + 2, ap_rsn[1]); - if (ret >= 0) { - offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); - caps = get_unaligned_le16(ap_rsn + 2 + offset); - ret = 0; + bss = cfg80211_get_bss(wiphy, channel, bssid, params->ssid, + params->ssid_len, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; } - } - rcu_read_unlock(); - if (ret) { + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + channel = bss->channel; cfg80211_put_bss(wiphy, bss); + } + if (channel->band != NL80211_BAND_2GHZ && + channel->band != NL80211_BAND_5GHZ) { + ret = -EOPNOTSUPP; goto out; } - offset = rsn_caps_offset(rsn_body, rsn_size); - put_unaligned_le16(caps, rsn_body + offset); - pmf = !!(caps & BIT(7)); + wifi->connect_channel = channel; + memcpy(wifi->ssid, params->ssid, params->ssid_len); + wifi->ssid_len = params->ssid_len; memset(payload, 0, sizeof(wifi->connect_payload)); payload[0] = 1; - memcpy(payload + 1, bss->bssid, ETH_ALEN); - memcpy(wifi->bssid, bss->bssid, ETH_ALEN); - payload[7] = ieee80211_frequency_to_channel(bss->channel->center_freq); + memcpy(payload + 1, wifi->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(channel->center_freq); - payload[8] = bss->channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[8] = channel->band == NL80211_BAND_2GHZ ? 0 : 1; payload[0x47] = 4; put_unaligned_le16(4, payload + 0x48); @@ -1077,13 +1115,37 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, wifi->connecting = true; wifi->connect_status = 0; wifi->connect_event = wifi->disconnect_event = false; - cfg80211_put_bss(wiphy, bss); schedule_work(&wifi->connect_work); out: mutex_unlock(&wifi->lock); return ret; } +/* + * cfg80211 records a connection against a BSS entry. When the client's scans + * have expired the target's, inform one from the request until its beacons + * (reported on ring 4 once associated) replace the elements. + */ +static struct cfg80211_bss *connected_bss(struct centauri_t6050_wifi *wifi) +{ + struct cfg80211_bss *bss; + u8 ie[2 + IEEE80211_MAX_SSID_LEN]; + + bss = cfg80211_get_bss(wifi->wiphy, wifi->connect_channel, wifi->bssid, + NULL, 0, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ANY); + if (bss) + return bss; + ie[0] = WLAN_EID_SSID; + ie[1] = wifi->ssid_len; + memcpy(ie + 2, wifi->ssid, wifi->ssid_len); + return cfg80211_inform_bss(wifi->wiphy, wifi->connect_channel, + CFG80211_BSS_FTYPE_UNKNOWN, wifi->bssid, 0, + WLAN_CAPABILITY_ESS | WLAN_CAPABILITY_PRIVACY, + 100, ie, 2 + wifi->ssid_len, -95 * 100, + GFP_KERNEL); +} + static void connect_worker(struct work_struct *work) { struct centauri_t6050_wifi *wifi = @@ -1117,6 +1179,20 @@ static void connect_worker(struct work_struct *work) ret = -ECANCELED; else if (!ret && !wifi->connect_event) ret = -ETIMEDOUT; + if (ret == -ENOTCONN) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: disconnect event, firmware status %u\n", + wifi->bssid, wifi->disconnect_status); + else if (ret == -ETIMEDOUT) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: no connect event\n", + wifi->bssid); + else if (ret && ret != -ECANCELED) + command_refused(wifi, "connect", ret); + else if (!ret && wifi->connect_status) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM failed: firmware status %u\n", + wifi->bssid, wifi->connect_status); status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; @@ -1140,9 +1216,13 @@ static void connect_worker(struct work_struct *work) wifi->request_ie_size, GFP_KERNEL, NL80211_TIMEOUT_UNSPECIFIED); else - cfg80211_connect_result(wifi->netdev, wifi->bssid, - wifi->request_ie, wifi->request_ie_size, - NULL, 0, status, GFP_KERNEL); + cfg80211_connect_bss(wifi->netdev, wifi->bssid, + status == WLAN_STATUS_SUCCESS ? + connected_bss(wifi) : + NULL, + wifi->request_ie, wifi->request_ie_size, + NULL, 0, status, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { cfg80211_port_authorized(wifi->netdev, wifi->bssid, NULL, 0, GFP_KERNEL); From 725275181fb923b03edf0aaa3e90a2b4f88b6761 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 12:42:00 -0800 Subject: [PATCH 330/392] wifi: apple: centauri: Read the firmware's channel list and capabilities The firmware owns the radio's channel list, its per-country rules and its capabilities; the driver registered a fixed 2.4/5 GHz table and never asked. Read the firmware's two reports once the interface is configured: system command 19, the channels the hardware supports, and system command 8, the system capabilities. Both are kept as read in debugfs (ieee80211/phyN/hw_channels and system_caps) and their sizes are logged. The band table is built from them next. command() gains a reply buffer, command_reply(): the payload of the reply to the command that asked is copied there, in place of the ring consumer's per-command copies. query_link_status() parses the interface status reply (20) from it; the copy of the country reply (10), which nothing requested, goes. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../apple/centauri/t6050/centauri_wifi.c | 142 ++++++++++++++---- 1 file changed, 110 insertions(+), 32 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c index be1040b513f0b..465f5743cc137 100644 --- a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include #include #include @@ -15,6 +16,7 @@ #define RING_SPAN 0x20000 #define TX_BUFFER 0x600000 #define BUFFER_SIZE 2048 +#define REPLY_SIZE (BUFFER_SIZE - 24) #define DATA_RX_BUFFER SZ_16M #define DATA_TX_BUFFER (SZ_16M + SZ_8M) @@ -46,9 +48,11 @@ struct centauri_t6050_wifi { u16 main_head, command_head, sequence; u32 rx_seen[5]; u16 rx_next[5], rx_head[5]; - u16 reply_sequence, reply_status; + u16 reply_sequence, reply_status, reply_capture; u32 reply_command, channels_seen, scans_done, beacons_seen; bool reply_valid, registered, configured; + u8 *reply_buffer; + size_t reply_capacity, reply_size; bool stopping, command_failed; bool scan_abort, connect_abort; u8 scan_sequence; @@ -69,12 +73,14 @@ struct centauri_t6050_wifi { bool station_signal_valid; unsigned long station_signal_at, connected_at; struct net_device_stats station_start; - u8 country[3]; - bool country_valid; u32 data_rings_open; bool data_failed; struct ieee80211_supported_band bands[2]; struct ieee80211_channel channels[39]; + /* The firmware's own reports, read at bring-up (system commands 19 and 8). */ + u8 hw_channels[REPLY_SIZE], system_caps[REPLY_SIZE]; + struct debugfs_blob_wrapper hw_channels_blob, system_caps_blob; + u8 link_status[REPLY_SIZE]; }; static int query_link_status(struct centauri_t6050_wifi *wifi); @@ -462,9 +468,10 @@ static void drain_data_tx(struct centauri_t6050_wifi *wifi) netif_wake_queue(wifi->netdev); } +/* The interface status reply: 8 fixed bytes, then TLVs. */ static bool link_signal_read(const u8 *packet, size_t size, s8 *signal) { - size_t pos = 32; + size_t pos = 8; while (pos <= size && size - pos >= 4) { u16 tag = get_unaligned_le16(packet + pos); @@ -536,20 +543,12 @@ static void drain(struct centauri_t6050_wifi *wifi) wifi->reply_status = status; wifi->reply_command = wire_command; - if (command == 10) { - wifi->country_valid = !status && - size >= 27; - if (wifi->country_valid) - memcpy(wifi->country, - packet + 24, 3); - } - - if (command == 20 && !status) { - wifi->station_signal_valid = - link_signal_read( - packet, size, - &wifi->station_signal); - wifi->station_signal_at = jiffies; + if (wifi->reply_buffer && + sequence == wifi->reply_capture) { + wifi->reply_size = size - 24; + memcpy(wifi->reply_buffer, packet + 24, + min(wifi->reply_size, + wifi->reply_capacity)); } } refill: @@ -588,10 +587,17 @@ static bool command_payload_consumed(struct centauri_t6050_wifi *wifi, u16 slot) return true; } -static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload, - size_t size) +/* + * Send one command and wait for its reply. With a reply buffer, the reply's + * payload is copied there (up to *reply_size bytes) and *reply_size becomes + * the payload's length. + */ +static int command_reply(struct centauri_t6050_wifi *wifi, u32 id, + const void *payload, size_t size, void *reply, + size_t *reply_size) { u16 slot = wifi->command_head, sequence = ++wifi->sequence; + int ret; u32 stage; u8 *packet, *descriptor; unsigned long deadline = jiffies + msecs_to_jiffies(1500); @@ -618,13 +624,18 @@ static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload wifi->command_head = (slot + 1) % ring_size[1]; index_write(wifi, 3, 1, wifi->command_head); wifi->reply_valid = false; + wifi->reply_buffer = reply; + wifi->reply_capacity = reply ? *reply_size : 0; + wifi->reply_capture = sequence; + wifi->reply_size = 0; dma_wmb(); writel(1, wifi->a.bar + 0x1004); do { if (READ_ONCE(wifi->stopping) || READ_ONCE(wifi->control->removing)) { command_fault(wifi); - return -ECANCELED; + ret = -ECANCELED; + goto out; } drain(wifi); stage = readl(wifi->a.bar + 8); @@ -633,24 +644,38 @@ static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload "Wi-Fi command %#x lost transport: stage=%#x sequence=%u rx=%u\n", id, stage, sequence, wifi->rx_seen[2]); command_fault(wifi); - return -EIO; + ret = -EIO; + goto out; } if (wifi->reply_valid && wifi->reply_sequence == sequence && (wifi->reply_command & ~BIT(30)) == id) { - if (!wifi->reply_status) - return 0; - dev_dbg(&wifi->control->pdev->dev, - "Wi-Fi command %#x: firmware status %u\n", id, - wifi->reply_status); - return -EREMOTEIO; + ret = 0; + if (wifi->reply_status) { + dev_dbg(&wifi->control->pdev->dev, + "Wi-Fi command %#x: firmware status %u\n", + id, wifi->reply_status); + ret = -EREMOTEIO; + } + if (reply) + *reply_size = wifi->reply_size; + goto out; } usleep_range(500, 1000); } while (time_before(jiffies, deadline)); dev_err(&wifi->control->pdev->dev, "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); command_fault(wifi); - return -ETIMEDOUT; + ret = -ETIMEDOUT; +out: + wifi->reply_buffer = NULL; + return ret; +} + +static int command(struct centauri_t6050_wifi *wifi, u32 id, const void *payload, + size_t size) +{ + return command_reply(wifi, id, payload, size, NULL, NULL); } /* Caller holds wifi->lock; ret is what command() returned for the request. */ @@ -678,6 +703,46 @@ static int station_configure(struct centauri_t6050_wifi *wifi, const u8 *address return ret; } +/* + * The firmware owns the radio's channel list and capabilities: system + * commands 19 (the channels the hardware supports) and 8 (the system + * capabilities) report them once the interface is configured. Kept as + * read, in debugfs. + */ +static void firmware_report(struct centauri_t6050_wifi *wifi, u32 id, u8 *buffer, + struct debugfs_blob_wrapper *blob) +{ + size_t size = REPLY_SIZE; + int ret; + + ret = command_reply(wifi, id, NULL, 0, buffer, &size); + if (ret) { + command_refused(wifi, id == 19 ? "channel list" : "capabilities", + ret); + size = 0; + } else if (size > REPLY_SIZE) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi command %u reply of %zu bytes truncated\n", id, + size); + size = REPLY_SIZE; + } + blob->data = buffer; + blob->size = size; +} + +static void firmware_reports(struct centauri_t6050_wifi *wifi) +{ + firmware_report(wifi, 19, wifi->hw_channels, &wifi->hw_channels_blob); + firmware_report(wifi, 8, wifi->system_caps, &wifi->system_caps_blob); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channel list: %lu bytes, capabilities: %lu bytes\n", + wifi->hw_channels_blob.size, wifi->system_caps_blob.size); + debugfs_create_blob("hw_channels", 0400, wifi->wiphy->debugfsdir, + &wifi->hw_channels_blob); + debugfs_create_blob("system_caps", 0400, wifi->wiphy->debugfsdir, + &wifi->system_caps_blob); +} + static int radio_start(struct centauri_t6050_wifi *wifi) { unsigned int ring, slot; @@ -704,8 +769,9 @@ static int radio_start(struct centauri_t6050_wifi *wifi) if (ret) return ret; msleep(1000); - wifi->configured = true; - return 0; + firmware_reports(wifi); + wifi->configured = !wifi->command_failed; + return wifi->configured ? 0 : -EIO; } #define CEN_SCAN_CHANNELS_MAX 59 @@ -1669,7 +1735,19 @@ static int data_start(struct centauri_t6050_wifi *wifi) static int query_link_status(struct centauri_t6050_wifi *wifi) { - return wifi->configured ? command(wifi, 20, NULL, 0) : -ENETDOWN; + size_t size = sizeof(wifi->link_status); + int ret; + + if (!wifi->configured) + return -ENETDOWN; + ret = command_reply(wifi, 20, NULL, 0, wifi->link_status, &size); + if (!ret) + wifi->station_signal_valid = + link_signal_read(wifi->link_status, + min(size, sizeof(wifi->link_status)), + &wifi->station_signal); + wifi->station_signal_at = jiffies; + return ret; } int centauri_t6050_wifi_register(struct centauri_t6050 *c) From f737e8de074a36e40b59b315bf2d26f6618e796f Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 13:03:01 -0800 Subject: [PATCH 331/392] wifi: apple: centauri: Take the bands from the firmware's channel list The driver registered a fixed table: 2.4 GHz channels 1-13 and 25 5 GHz channels, every one at 20 dBm, and no 6 GHz band. The firmware reports which channels the current country allows (system command 19), at each width and with passive and radar flags, and its own capability elements (system command 8). Before a country is set it reports 2.4 and 5 GHz only; under US rules it reports 167 entries, 6 GHz channels 1-233 among them. - The band plan (2.4 GHz 1-14, 5 GHz 36-177, 6 GHz 1-233) is registered once, and the firmware's list is applied over cfg80211's rules: a channel it leaves out is disabled, its passive and radar flags become NO_IR and RADAR, and a width no entry covers is not used. The power limit is left to the regulatory domain and the firmware. - The regulatory notifier sets the country (command 9, as before), then reads the list again and applies it, after every regulatory change. - 6 GHz is registered with the HE capabilities element the firmware reports, and 6 GHz band capabilities from its VHT element; without an HE element there is no 6 GHz band. The radio is started before the wiphy is registered, since the bands come from it. - One band encoding, firmware_band(), for scan, connect and beacons. 6 GHz beacons carry neither DS parameters nor HT operation, so they take their channel from the HE operation's 6 GHz information. - Scans include the listen-only channels (radar channels, and all of 6 GHz under US rules) with the firmware's passive and radar bits, as its channel list encodes them, instead of skipping them. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../apple/centauri/t6050/centauri_wifi.c | 319 ++++++++++++++---- 1 file changed, 257 insertions(+), 62 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c index 465f5743cc137..e89c280a5bdc4 100644 --- a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include #include #include #include #include +#include #include #include #include @@ -17,6 +19,8 @@ #define TX_BUFFER 0x600000 #define BUFFER_SIZE 2048 #define REPLY_SIZE (BUFFER_SIZE - 24) +/* The band plan: 2.4 GHz 1-14, 5 GHz 36-64, 100-144 and 149-177, 6 GHz 1-233. */ +#define CEN_CHANNELS (14 + 28 + 59) #define DATA_RX_BUFFER SZ_16M #define DATA_TX_BUFFER (SZ_16M + SZ_8M) @@ -75,8 +79,9 @@ struct centauri_t6050_wifi { struct net_device_stats station_start; u32 data_rings_open; bool data_failed; - struct ieee80211_supported_band bands[2]; - struct ieee80211_channel channels[39]; + struct ieee80211_supported_band bands[3]; + struct ieee80211_channel channels[CEN_CHANNELS]; + struct ieee80211_sband_iftype_data iftype_6ghz; /* The firmware's own reports, read at bring-up (system commands 19 and 8). */ u8 hw_channels[REPLY_SIZE], system_caps[REPLY_SIZE]; struct debugfs_blob_wrapper hw_channels_blob, system_caps_blob; @@ -87,9 +92,13 @@ static int query_link_status(struct centauri_t6050_wifi *wifi); static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; -static const u16 five_ghz[] = { - 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, - 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, +static const struct { + enum nl80211_band band; + u8 first, last, step; +} band_plan[] = { + { NL80211_BAND_2GHZ, 1, 14, 1 }, { NL80211_BAND_5GHZ, 36, 64, 4 }, + { NL80211_BAND_5GHZ, 100, 144, 4 }, { NL80211_BAND_5GHZ, 149, 177, 4 }, + { NL80211_BAND_6GHZ, 1, 233, 4 }, }; static struct ieee80211_rate rates[] = { { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, @@ -105,6 +114,21 @@ static const u32 cipher_suites[] = { WLAN_CIPHER_SUITE_AES_CMAC, }; +/* The firmware's band numbers: 0 is 2.4 GHz, 1 is 5 GHz, 2 is 6 GHz. */ +static int firmware_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return 0; + case NL80211_BAND_5GHZ: + return 1; + case NL80211_BAND_6GHZ: + return 2; + default: + return -EOPNOTSUPP; + } +} + static u16 index_read(struct centauri_t6050_wifi *wifi, unsigned int array, unsigned int index) { @@ -196,7 +220,7 @@ static void report_beacon(struct centauri_t6050_wifi *wifi, const u8 *payload, struct cfg80211_bss *bss; u16 frame_size; int number = 0, frequency, signal; - enum nl80211_band band; + enum nl80211_band band = NL80211_BAND_2GHZ; if (!wifi->registered || size < 0x60 + 36) return; @@ -211,12 +235,26 @@ static void report_beacon(struct centauri_t6050_wifi *wifi, const u8 *payload, ie += 2 + ie[1]) { if ((ie[0] == WLAN_EID_DS_PARAMS || ie[0] == WLAN_EID_HT_OPERATION) && - ie[1]) + ie[1]) { number = ie[2]; + band = number <= 14 ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + } else if (ie[0] == WLAN_EID_EXTENSION && ie[1] > 1 && + ie[2] == WLAN_EID_EXT_HE_OPERATION && + ie[1] - 1 >= sizeof(struct ieee80211_he_operation) && + ie[1] >= ieee80211_he_oper_size(ie + 3)) { + /* 6 GHz beacons carry neither: the HE operation names the channel. */ + const struct ieee80211_he_6ghz_oper *oper = + ieee80211_he_6ghz_oper((const void *)(ie + 3)); + + if (oper && !number) { + number = oper->primary; + band = NL80211_BAND_6GHZ; + } + } } if (!number) return; - band = number <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; frequency = ieee80211_channel_to_frequency(number, band); channel = ieee80211_get_channel(wifi->wiphy, frequency); if (!channel) @@ -730,6 +768,137 @@ static void firmware_report(struct centauri_t6050_wifi *wifi, u32 id, u8 *buffer blob->size = size; } +/* + * The channel list: a count, then one 4-byte entry per channel the current + * country allows at each width: the channel number (the center channel for + * 40-160 MHz), then the band in bits 0-1, the width (20 << n MHz) in bits 2-3 + * and the flags in bits 4-7, then two bytes this driver does not use (a + * width mask and, on 6 GHz, power-mode bits). Scan requests take the same + * band and flag bits. + */ +#define CHANNEL_PASSIVE BIT(4) +#define CHANNEL_RADAR BIT(6) + +static const u8 *channel_entry(const u8 *list, size_t size, int band, + unsigned int width, unsigned int number) +{ + unsigned int count = size ? list[0] : 0, i, span = 2 * (1 << width) - 2; + + if (1 + 4 * count > size) + count = size ? (size - 1) / 4 : 0; + for (i = 0; i < count; i++) { + const u8 *entry = list + 1 + 4 * i; + + if ((entry[1] & 3) != band || ((entry[1] >> 2) & 3) != width) + continue; + if (number + span >= entry[0] && number <= entry[0] + span) + return entry; + } + return NULL; +} + +/* + * The firmware's list decides which channels exist and how: a band-plan + * channel it leaves out is disabled, its flags mark the rest, and a width no + * entry covers is not used. Applied over what cfg80211's regulatory domain + * set, after every change of it (the notifier) and at bring-up. + */ +static void channels_apply(struct centauri_t6050_wifi *wifi) +{ + const u8 *list = wifi->hw_channels; + size_t size = wifi->hw_channels_blob.size; + unsigned int counted[3] = {}, i; + + if (!size) + return; + for (i = 0; i < CEN_CHANNELS; i++) { + struct ieee80211_channel *channel = &wifi->channels[i]; + int band = firmware_band(channel->band); + const u8 *entry = channel_entry(list, size, band, 0, + channel->hw_value); + + if (!entry) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + if (entry[1] & CHANNEL_PASSIVE) + channel->flags |= IEEE80211_CHAN_NO_IR; + if (entry[1] & CHANNEL_RADAR) + channel->flags |= IEEE80211_CHAN_RADAR; + if (!channel_entry(list, size, band, 1, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_HT40; + if (!channel_entry(list, size, band, 2, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_80MHZ; + if (!channel_entry(list, size, band, 3, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_160MHZ; + if (!(channel->flags & IEEE80211_CHAN_DISABLED)) + counted[band]++; + } + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channels: 2.4 GHz %u, 5 GHz %u, 6 GHz %u (%u entries)\n", + counted[0], counted[1], counted[2], list[0]); +} + +/* + * 6 GHz needs HE: the station's HE capabilities element as the firmware + * reports it (capabilities TLV 0xff), with the 6 GHz band capabilities taken + * from its VHT element (TLV 0xbf). Without them the band is not registered. + */ +static bool he_6ghz_from_firmware(struct centauri_t6050_wifi *wifi) +{ + struct ieee80211_sband_iftype_data *data = &wifi->iftype_6ghz; + struct ieee80211_sta_he_cap *he = &data->he_cap; + const u8 *caps = wifi->system_caps, *he_elem = NULL, *vht = NULL; + size_t size = wifi->system_caps_blob.size, pos = 0, mcs; + u32 vht_cap = 0; + + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(caps + pos); + u16 length = get_unaligned_le16(caps + pos + 2); + const u8 *value = caps + pos + 4; + + if (length > size - pos - 4) + break; + if (tag == WLAN_EID_EXTENSION && length >= 3 + 6 + 11 + 4 && + value[0] == WLAN_EID_EXTENSION && value[1] == length - 2 && + value[2] == WLAN_EID_EXT_HE_CAPABILITY) + he_elem = value; + if (tag == WLAN_EID_VHT_CAPABILITY && length >= 2 + 12 && + value[0] == WLAN_EID_VHT_CAPABILITY) + vht = value; + pos += 4 + length; + } + if (!he_elem) + return false; + memcpy(&he->he_cap_elem, he_elem + 3, sizeof(he->he_cap_elem)); + mcs = he_elem[1] - 1 - sizeof(he->he_cap_elem); + memcpy(&he->he_mcs_nss_supp, he_elem + 3 + sizeof(he->he_cap_elem), + min(mcs, sizeof(he->he_mcs_nss_supp))); + if (mcs > sizeof(he->he_mcs_nss_supp)) + memcpy(he->ppe_thres, + he_elem + 3 + sizeof(he->he_cap_elem) + + sizeof(he->he_mcs_nss_supp), + min(mcs - sizeof(he->he_mcs_nss_supp), + sizeof(he->ppe_thres))); + he->has_he = true; + if (vht) + vht_cap = get_unaligned_le32(vht + 2); + data->he_6ghz_capa.capa = cpu_to_le16( + u16_encode_bits(u32_get_bits(vht_cap, + IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK), + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | + u16_encode_bits(vht_cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK, + IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN) | + u16_encode_bits(WLAN_HT_CAP_SM_PS_DISABLED, + IEEE80211_HE_6GHZ_CAP_SM_PS) | + (vht_cap & IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS : 0) | + (vht_cap & IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS : 0)); + data->types_mask = BIT(NL80211_IFTYPE_STATION); + return true; +} + static void firmware_reports(struct centauri_t6050_wifi *wifi) { firmware_report(wifi, 19, wifi->hw_channels, &wifi->hw_channels_blob); @@ -737,10 +906,45 @@ static void firmware_reports(struct centauri_t6050_wifi *wifi) dev_info(&wifi->control->pdev->dev, "Wi-Fi firmware channel list: %lu bytes, capabilities: %lu bytes\n", wifi->hw_channels_blob.size, wifi->system_caps_blob.size); - debugfs_create_blob("hw_channels", 0400, wifi->wiphy->debugfsdir, - &wifi->hw_channels_blob); - debugfs_create_blob("system_caps", 0400, wifi->wiphy->debugfsdir, - &wifi->system_caps_blob); +} + +/* The band plan, with the firmware's legacy rates (CCK on 2.4 GHz only). */ +static void bands_init(struct centauri_t6050_wifi *wifi) +{ + unsigned int i, n = 0, c; + + for (i = 0; i < ARRAY_SIZE(band_plan); i++) { + enum nl80211_band band = band_plan[i].band; + struct ieee80211_supported_band *sband = + &wifi->bands[firmware_band(band)]; + + if (!sband->channels) + sband->channels = wifi->channels + n; + for (c = band_plan[i].first; c <= band_plan[i].last; + c += band_plan[i].step) { + wifi->channels[n].band = band; + wifi->channels[n].hw_value = c; + wifi->channels[n].center_freq = + ieee80211_channel_to_frequency(c, band); + sband->n_channels++; + n++; + } + sband->band = band; + sband->bitrates = band == NL80211_BAND_2GHZ ? rates : rates + 4; + sband->n_bitrates = band == NL80211_BAND_2GHZ ? + ARRAY_SIZE(rates) : + ARRAY_SIZE(rates) - 4; + } + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + if (he_6ghz_from_firmware(wifi)) { + _ieee80211_set_sband_iftype_data(&wifi->bands[2], + &wifi->iftype_6ghz, 1); + wifi->wiphy->bands[NL80211_BAND_6GHZ] = &wifi->bands[2]; + } else { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi: no HE capabilities from the firmware; no 6 GHz band\n"); + } } static int radio_start(struct centauri_t6050_wifi *wifi) @@ -809,21 +1013,25 @@ static int scan_profile27(struct centauri_t6050_wifi *wifi) struct ieee80211_channel *channel = request->channels[next++]; unsigned int number; + int band; - /* The passive-scan wire mode is not established for this profile. */ - if (channel->flags & - (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR)) + if (channel->flags & IEEE80211_CHAN_DISABLED) continue; - if (channel->band != NL80211_BAND_2GHZ && - channel->band != NL80211_BAND_5GHZ) - return -EOPNOTSUPP; + band = firmware_band(channel->band); + if (band < 0) + return band; number = ieee80211_frequency_to_channel( channel->center_freq); if (!number || number > 255) return -EINVAL; channels[1 + 4 * count] = number; - channels[2 + 4 * count] = channel->band == - NL80211_BAND_5GHZ; + channels[2 + 4 * count] = band; + /* Flagged as in the firmware's channel list: listen only. */ + if (channel->flags & + (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) + channels[2 + 4 * count] |= CHANNEL_PASSIVE; + if (channel->flags & IEEE80211_CHAN_RADAR) + channels[2 + 4 * count] |= CHANNEL_RADAR; count++; } if (!count) @@ -1087,7 +1295,7 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; u8 *payload = wifi->connect_payload; size_t size = 130, rsn_size, rsn_prefix, rsn_tail; - int ret = 0, offset; + int ret = 0, offset, band; if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || params->crypto.n_ciphers_pairwise != 1 || @@ -1134,11 +1342,11 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, channel = bss->channel; cfg80211_put_bss(wiphy, bss); } - if (channel->band != NL80211_BAND_2GHZ && - channel->band != NL80211_BAND_5GHZ) { - ret = -EOPNOTSUPP; + ret = firmware_band(channel->band); + if (ret < 0) goto out; - } + band = ret; + ret = 0; wifi->connect_channel = channel; memcpy(wifi->ssid, params->ssid, params->ssid_len); wifi->ssid_len = params->ssid_len; @@ -1147,7 +1355,7 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, memcpy(payload + 1, wifi->bssid, ETH_ALEN); payload[7] = ieee80211_frequency_to_channel(channel->center_freq); - payload[8] = channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[8] = band; payload[0x47] = 4; put_unaligned_le16(4, payload + 0x48); @@ -1415,18 +1623,22 @@ static void wifi_regulatory(struct wiphy *wiphy, struct centauri_t6050_wifi *wifi = *(struct centauri_t6050_wifi **)wiphy_priv(wiphy); u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + bool named = country[0] >= 'A' && country[0] <= 'Z' && + country[1] >= 'A' && country[1] <= 'Z'; int ret; - if (country[0] < 'A' || country[0] > 'Z' || country[1] < 'A' || - country[1] > 'Z') - return; mutex_lock(&wifi->lock); - if (wifi->configured) { + if (wifi->configured && named) { ret = command(wifi, 9, country, sizeof(country)); if (ret) dev_err(&wifi->control->pdev->dev, "firmware country update failed: %d\n", ret); + else + firmware_report(wifi, 19, wifi->hw_channels, + &wifi->hw_channels_blob); } + /* cfg80211 has just recomputed the channel flags from its own rules. */ + channels_apply(wifi); mutex_unlock(&wifi->lock); } @@ -1754,7 +1966,6 @@ int centauri_t6050_wifi_register(struct centauri_t6050 *c) { struct centauri_t6050_wifi *wifi; u8 address[ETH_ALEN]; - unsigned int i; int ret; wifi = kzalloc_obj(*wifi); @@ -1789,31 +2000,6 @@ int centauri_t6050_wifi_register(struct centauri_t6050 *c) wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); - for (i = 0; i < 13; i++) { - wifi->channels[i].band = NL80211_BAND_2GHZ; - wifi->channels[i].hw_value = i + 1; - wifi->channels[i].center_freq = ieee80211_channel_to_frequency( - i + 1, NL80211_BAND_2GHZ); - wifi->channels[i].max_power = 20; - } - for (i = 0; i < ARRAY_SIZE(five_ghz); i++) { - wifi->channels[13 + i].band = NL80211_BAND_5GHZ; - wifi->channels[13 + i].hw_value = five_ghz[i]; - wifi->channels[13 + i].center_freq = - ieee80211_channel_to_frequency(five_ghz[i], - NL80211_BAND_5GHZ); - wifi->channels[13 + i].max_power = 20; - } - wifi->bands[0].channels = wifi->channels; - wifi->bands[0].n_channels = 13; - wifi->bands[0].bitrates = rates; - wifi->bands[0].n_bitrates = ARRAY_SIZE(rates); - wifi->bands[1].channels = wifi->channels + 13; - wifi->bands[1].n_channels = ARRAY_SIZE(five_ghz); - wifi->bands[1].bitrates = rates + 4; - wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; - wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; - wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); if (!ret) { @@ -1827,10 +2013,25 @@ int centauri_t6050_wifi_register(struct centauri_t6050 *c) &wifi->a.pdev->dev, "No valid factory MAC property; using a random address\n"); } + /* The bands come from the firmware's reports: start the radio first. */ + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, + "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + bands_init(wifi); ret = wiphy_register(wifi->wiphy); if (ret) goto failed; wifi->registered = true; + debugfs_create_blob("hw_channels", 0400, wifi->wiphy->debugfsdir, + &wifi->hw_channels_blob); + debugfs_create_blob("system_caps", 0400, wifi->wiphy->debugfsdir, + &wifi->system_caps_blob); + rtnl_lock(); + channels_apply(wifi); + rtnl_unlock(); wifi->wdev.wiphy = wifi->wiphy; wifi->wdev.iftype = NL80211_IFTYPE_STATION; wifi->wdev.netdev = wifi->netdev; @@ -1840,12 +2041,6 @@ int centauri_t6050_wifi_register(struct centauri_t6050 *c) strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); wifi->netdev->max_mtu = ETH_DATA_LEN; - ret = radio_start(wifi); - if (ret) { - dev_err(&c->pdev->dev, - "Wi-Fi ring/interface setup failed: %d\n", ret); - goto failed; - } ret = register_netdev(wifi->netdev); if (ret) goto failed; From 5663c0e8b696bdfe3a120443d3ded63a5c35d812 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Tue, 3 Nov 2026 13:03:01 -0800 Subject: [PATCH 332/392] wifi: apple: centauri: Support WPA3-Personal through the firmware's SAE The firmware runs SAE itself from the password. Offer SAE (AKM 00-0F-AC:8, SAE offload). The connect command carries one authentication and AKM pair: a request that offers SAE with a password sends SAE authentication (3) and AKM 8, with the password in TLV 313 as a 16-bit length and its bytes, and no PMK; one that offers PSK with a PMK sends open authentication and AKM 4 as before. Management frame protection travels in the station's RSN element only. TLV 316 is the security roaming groups bitmap, not a PMF flag: send 0 (no roaming groups), and TLV 304 (the join mode) as 5 for every join. 6 GHz admits WPA3-Personal only with management frame protection, so a 6 GHz connect without both is refused. Signed-off-by: Jeff Queisser Developed with Claude Code. Signed-off-by: Erik Bourget --- .../apple/centauri/t6050/centauri_wifi.c | 95 +++++++++++++++---- 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c index e89c280a5bdc4..5deffa85c8168 100644 --- a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -107,7 +107,7 @@ static struct ieee80211_rate rates[] = { { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, }; -static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK }; +static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE }; static const u32 cipher_suites[] = { WLAN_CIPHER_SUITE_CCMP, @@ -1204,7 +1204,8 @@ static int rsn_caps_offset(const u8 *body, size_t size) return offset + 2 <= size ? offset : -EINVAL; } -static int rsn_psk_offset(const u8 *body, size_t size) +/* The offset of the RSN element's AKM list when it offers akm. */ +static int rsn_akm_offset(const u8 *body, size_t size, u32 akm) { size_t offset, end; u16 count; @@ -1223,25 +1224,25 @@ static int rsn_psk_offset(const u8 *body, size_t size) if (!count || end + 2 > size) return -EINVAL; for (i = 0; i < count; i++) - if (get_unaligned_be32(body + offset + 2 + 4 * i) == - WLAN_AKM_SUITE_PSK) + if (get_unaligned_be32(body + offset + 2 + 4 * i) == akm) return offset; return -EOPNOTSUPP; } -static int select_psk_rsn(const struct cfg80211_crypto_settings *crypto, - const u8 *body, size_t size, u8 *selected) +/* The station's RSN element with akm as its only AKM. */ +static int select_rsn(const struct cfg80211_crypto_settings *crypto, u32 akm, + const u8 *body, size_t size, u8 *selected) { size_t tail; unsigned int i; int offset; for (i = 0; i < crypto->n_akm_suites; i++) - if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK) + if (crypto->akm_suites[i] == akm) break; if (i == crypto->n_akm_suites) return -EOPNOTSUPP; - offset = rsn_psk_offset(body, size); + offset = rsn_akm_offset(body, size, akm); if (offset < 0) return offset; @@ -1252,11 +1253,38 @@ static int select_psk_rsn(const struct cfg80211_crypto_settings *crypto, tail = offset + 2 + 4 * get_unaligned_le16(body + offset); memcpy(selected, body, offset); put_unaligned_le16(1, selected + offset); - put_unaligned_be32(WLAN_AKM_SUITE_PSK, selected + offset + 2); + put_unaligned_be32(akm, selected + offset + 2); memcpy(selected + offset + 6, body + tail, size - tail); return offset + 6 + size - tail; } +/* + * The connect command carries one authentication and AKM pair, in the bytes + * at 0x46 and 0x47: WPA2-Personal is open authentication with AKM 4 and the + * PMK at 0x4c, WPA3-Personal is SAE authentication with AKM 8 and the + * password in TLV 313, which the firmware runs SAE from. + */ +#define CONNECT_AUTH_OPEN 0 +#define CONNECT_AUTH_SAE 3 +#define CONNECT_AKM_PSK 4 +#define CONNECT_AKM_SAE 8 + +/* The AKM of the client's request whose credential it supplies, SAE first. */ +static u32 connect_akm(const struct cfg80211_crypto_settings *crypto) +{ + bool psk = false; + unsigned int i; + + for (i = 0; i < crypto->n_akm_suites; i++) { + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_SAE && + crypto->sae_pwd_len) + return WLAN_AKM_SUITE_SAE; + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK && crypto->psk) + psk = true; + } + return psk ? WLAN_AKM_SUITE_PSK : 0; +} + static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length) { @@ -1281,6 +1309,10 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, 0xff, 0xff, 0xff, 0xff, 3, 0, 0, }; + /* TLV 304: the join mode, 5 for every join. */ + static const u8 join_mode = 5; + /* TLV 316: no security roaming groups. */ + static const u8 roaming; static const __le32 connection_timeouts[] = { cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), cpu_to_le32(15000), cpu_to_le32(300), @@ -1292,12 +1324,16 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, params->channel ?: params->channel_hint; struct cfg80211_bss *bss; const u8 *rsn; - u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; + u8 rsn_body[255], ssid[35]; u8 *payload = wifi->connect_payload; size_t size = 130, rsn_size, rsn_prefix, rsn_tail; int ret = 0, offset, band; + u32 akm = connect_akm(¶ms->crypto); + bool sae = akm == WLAN_AKM_SUITE_SAE, mfpc; + u8 password[2 + SAE_PASSWORD_MAX_LEN]; - if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || + if (!params->ssid_len || params->ssid_len > 32 || !akm || + params->crypto.sae_pwd_len > SAE_PASSWORD_MAX_LEN || params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || @@ -1306,16 +1342,18 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); if (!rsn) return -EINVAL; - ret = select_psk_rsn(¶ms->crypto, rsn + 2, rsn[1], rsn_body); + ret = select_rsn(¶ms->crypto, akm, rsn + 2, rsn[1], rsn_body); if (ret < 0) return ret; rsn_size = ret; - /* The station's own RSN capabilities go to the firmware; bit 7 is MFPC. */ + /* + * The station's own RSN element carries its management frame + * protection to the firmware; bit 7 of its capabilities is MFPC. + */ offset = rsn_caps_offset(rsn_body, rsn_size); if (offset < 0) return -EINVAL; - pmf = !!(get_unaligned_le16(rsn_body + offset) & - BIT(7)); + mfpc = get_unaligned_le16(rsn_body + offset) & BIT(7); ret = 0; mutex_lock(&wifi->lock); if (READ_ONCE(wifi->stopping)) { @@ -1347,6 +1385,11 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, goto out; band = ret; ret = 0; + /* 6 GHz admits WPA3-Personal only with management frame protection. */ + if (channel->band == NL80211_BAND_6GHZ && (!sae || !mfpc)) { + ret = -EOPNOTSUPP; + goto out; + } wifi->connect_channel = channel; memcpy(wifi->ssid, params->ssid, params->ssid_len); wifi->ssid_len = params->ssid_len; @@ -1356,13 +1399,16 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, payload[7] = ieee80211_frequency_to_channel(channel->center_freq); payload[8] = band; - payload[0x47] = 4; + payload[0x46] = sae ? CONNECT_AUTH_SAE : CONNECT_AUTH_OPEN; + payload[0x47] = sae ? CONNECT_AKM_SAE : CONNECT_AKM_PSK; put_unaligned_le16(4, payload + 0x48); - put_unaligned_le16(32, payload + 0x4a); - memcpy(payload + 0x4c, params->crypto.psk, 32); + if (!sae) { + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + } - append_tlv(payload, &size, 304, zero, 1); + append_tlv(payload, &size, 304, &join_mode, 1); append_tlv(payload, &size, 63763, connection_timeouts, sizeof(connection_timeouts)); ssid[0] = 1; @@ -1371,8 +1417,16 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, memcpy(ssid + 3, params->ssid, params->ssid_len); append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + if (sae) { + put_unaligned_le16(params->crypto.sae_pwd_len, password); + memcpy(password + 2, params->crypto.sae_pwd, + params->crypto.sae_pwd_len); + append_tlv(payload, &size, 313, password, + 2 + params->crypto.sae_pwd_len); + memzero_explicit(password, sizeof(password)); + } - append_tlv(payload, &size, 316, &pmf, 1); + append_tlv(payload, &size, 316, &roaming, 1); append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); wifi->connect_size = size; @@ -2000,6 +2054,7 @@ int centauri_t6050_wifi_register(struct centauri_t6050 *c) wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); if (!ret) { From b4ef30560a000e051a4784fb70f77dfef84f8386 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 05:25:55 -0500 Subject: [PATCH 333/392] wifi: apple: centauri: t6050: tell the firmware the host is awake before a join Without the host power state the N1 Alpha firmware treats the host as asleep, powers the PHYs down and traps. Before the first join of a firmware boot, send command 6 with a zero body (host power state awake), command 43 (packet lifetime) and command 52 (control profile). Each is sent once per firmware boot and retried at the next join if it fails. The host_setup parameter (default on) turns this off. This is Joshua Warren's fix from the J813/J873 N1 driver (linux-aurora 1242f12e876b), which Erik Bourget ported to the T6050 driver as aurora-silicon/linux#828. #828 targets t6050/centauri-wifi's centauri/centauri_wifi.c; with the T6050 transport now in centauri/t6050/, the same change goes here. Signed-off-by: Diljit Singh --- .../apple/centauri/t6050/centauri_wifi.c | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c index 5deffa85c8168..20d86b4e50ef1 100644 --- a/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/t6050/centauri_wifi.c @@ -36,6 +36,11 @@ static_assert(DATA_RX_BUFFER + DATA_RX_SLOTS * 4096 <= DATA_TX_BUFFER); static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= CEN_ALPHA_WINDOW_SIZE); +static bool host_setup = true; +module_param(host_setup, bool, 0644); +MODULE_PARM_DESC(host_setup, + "Before a join, tell the firmware the host is awake (command 6) and apply the reference TX defaults (commands 43 and 52); 1 = on (default)"); + struct centauri_t6050_wifi { struct centauri_t6050 *control; struct cen_alpha_resources a; @@ -79,6 +84,9 @@ struct centauri_t6050_wifi { struct net_device_stats station_start; u32 data_rings_open; bool data_failed; + /* Sent once per firmware boot, before the first join (host_setup). */ + bool host_awake_sent; + u8 tx_defaults_applied; struct ieee80211_supported_band bands[3]; struct ieee80211_channel channels[CEN_CHANNELS]; struct ieee80211_sband_iftype_data iftype_6ghz; @@ -1474,6 +1482,43 @@ static struct cfg80211_bss *connected_bss(struct centauri_t6050_wifi *wifi) GFP_KERNEL); } +/* + * Without the host power state the firmware treats the host as asleep, powers + * the PHYs down and traps. Called with wifi->lock held. + */ +static void host_setup_commands(struct centauri_t6050_wifi *wifi) +{ + static const u8 awake[3] = {}; + static const u8 control_profile[] = { 0x54, 1, 10, 0, 0, 8, 0, 0, + 0xff, 0xff, 0xff, 0xff, 1, 1 }; + struct device *dev = &wifi->control->pdev->dev; + u8 lifetime[20] = {}; + int ret, i; + + if (!host_setup) + return; + if (!wifi->host_awake_sent) { + ret = command(wifi, 6, awake, sizeof(awake)); + dev_info(dev, "host power state awake: rc=%d status=%u\n", ret, + wifi->reply_status); + wifi->host_awake_sent = !ret; + } + if (!(wifi->tx_defaults_applied & BIT(0))) { + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + if (!ret) + wifi->tx_defaults_applied |= BIT(0); + dev_info(dev, "TX packet lifetime defaults: rc=%d\n", ret); + } + if (!(wifi->tx_defaults_applied & BIT(1))) { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + if (!ret) + wifi->tx_defaults_applied |= BIT(1); + dev_info(dev, "TX control profile defaults: rc=%d\n", ret); + } +} + static void connect_worker(struct work_struct *work) { struct centauri_t6050_wifi *wifi = @@ -1488,6 +1533,7 @@ static void connect_worker(struct work_struct *work) mutex_unlock(&wifi->lock); return; } + host_setup_commands(wifi); slot = wifi->command_head; ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); From d0c84b3e211906de73e74f0c12d4416d2452bf72 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 7 Nov 2026 23:52:23 -0500 Subject: [PATCH 334/392] PCI: apple: Allow retained-link s2idle on J716c The adopted J716c radio host currently rejects every noirq suspend. Its radio drivers save PCI state without entering D3, and the native host and DART retain the running hierarchy during suspend-to-idle. Allow this path only for J716c s2idle when the controller and ports are healthy and all PCI devices are accessible in D0. Validate the retained hierarchy again on resume and block downstream configuration access if validation fails. Leave existing behavior for managed-SID hosts, other adopted platforms and deeper sleep unchanged. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/pci/controller/pcie-apple.c | 74 ++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index c19f271e40d2b..eda231e2ea0b3 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -4794,6 +4794,59 @@ static void apple_pcie_stop_for_sleep(struct device *dev) apple_pcie_tunnel_hierarchy_lost(dev); } +/* + * The J716c radio drivers save PCI state without entering D3. The PCI core + * keeps the root port in D0 as well. Its host and DART have no switchable + * power domains, and the DART saves its live translations without stopping + * them. For s2idle, retain that running hierarchy instead of resetting it. + * + * Check hardware PMCSR rather than current_state: the PCI core may mark a + * driverless root port PCI_UNKNOWN even though its hardware remains in D0. + */ +static int apple_pcie_check_d0(struct pci_dev *pdev, void *data) +{ + int *error = data; + u16 value; + + if (pci_read_config_word(pdev, PCI_VENDOR_ID, &value) || + value != pdev->vendor) { + *error = -ENODEV; + } else if (pdev->pm_cap && + (pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &value) || + (value & PCI_PM_CTRL_STATE_MASK) != PCI_D0)) { + *error = -EBUSY; + } + if (*error) + pci_err(pdev, "adopted link requires accessible D0 devices for s2idle\n"); + return *error; +} + +static int apple_pcie_check_adopted_s2idle(struct apple_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + int error = 0; + + /* Other adopted platforms and deeper sleep remain unqualified. */ + if (!of_machine_is_compatible("apple,j716c") || !pm_suspend_no_platform()) + return -EBUSY; + + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN) || + !(readl(pcie->base + CORE_RC_STAT) & CORE_RC_STAT_READY)) + return -ENOLINK; + + list_for_each_entry(port, &pcie->ports, entry) { + if (READ_ONCE(port->link_failed) || + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY) || + (apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP || + (apple_pcie_port_readl(port, PORT_INTSTAT) & BIT(PORT_INT_LINK_DOWN))) + return -ENOLINK; + } + pci_walk_bus(bridge->bus, apple_pcie_check_d0, &error); + return error; +} + static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -4804,8 +4857,12 @@ static int apple_pcie_suspend_noirq(struct device *dev) */ if (pcie->managed_sids) return pm_suspend_target_state == PM_SUSPEND_TO_IDLE ? 0 : -EBUSY; - if (pcie->hw->adopt_link) - return -EBUSY; + if (pcie->hw->adopt_link) { + int ret = apple_pcie_check_adopted_s2idle(pcie); + + pcie->link_kept = !ret; + return ret; + } if (!pcie->hw->tunneled) return 0; pcie->reset_on_resume = false; @@ -4862,6 +4919,19 @@ static int apple_pcie_resume_noirq(struct device *dev) struct apple_pcie_port *port; int ret; + if (pcie->hw->adopt_link) { + if (!pcie->link_kept) + return 0; + pcie->link_kept = false; + ret = apple_pcie_check_adopted_s2idle(pcie); + if (ret) { + /* Block downstream config access if the retained link was lost. */ + list_for_each_entry(port, &pcie->ports, entry) + WRITE_ONCE(port->link_failed, true); + dev_err(dev, "adopted link lost during suspend-to-idle: %d\n", ret); + } + return ret; + } if (!pcie->hw->tunneled) return 0; /* The domain's hold for the sleep ends first, whatever follows. */ From 4df5c8c23e0e1b633796efe4061c4852b0754ec9 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 8 Nov 2026 14:49:09 -0500 Subject: [PATCH 335/392] arm64: dts: apple: Describe the J716c camera capture resources Provide the J716c ISP's named register windows, host DART streams, sensor preset and power gates. Reuse the existing hardware inventory nodes and add the aliases needed by the loader. Leave the camera and its DARTs disabled until the loader reserves the required memory and supplies the runtime properties. Developed with Claude Code. Signed-off-by: Erik Bourget --- .../arm64/boot/dts/apple/t6050-j716c-isp.dtsi | 146 ++++++++++++++++++ arch/arm64/boot/dts/apple/t6050-j716c.dts | 2 + 2 files changed, 148 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6050-j716c-isp.dtsi diff --git a/arch/arm64/boot/dts/apple/t6050-j716c-isp.dtsi b/arch/arm64/boot/dts/apple/t6050-j716c-isp.dtsi new file mode 100644 index 0000000000000..b6b1675d26022 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j716c-isp.dtsi @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* J716c camera: enabled only after the loader reserves the current firmware. */ +/ { + aliases { + isp = &isp0; + isp-dart0 = &isp0_dart; + isp-dart1 = &isp0_dart1; + isp-dart2 = &isp0_dart2; + }; +}; + +&isp0 { + /delete-property/ clocks; + compatible = "apple,t6050-isp", "apple,isp"; + reg = <0x5 0x80000000 0 0x1604000>, + <0x5 0x82924000 0 0x40>, + <0x5 0x829241d0 0 0x20>, + <0x5 0x82924464 0 0x10>, + <0x5 0xc3b50000 0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "mbox2", "wdt"; + interrupts = ; + iommus = <&isp0_dart 0>, <&isp0_dart 12>, + <&isp0_dart1 0>, <&isp0_dart1 12>, + <&isp0_dart2 0>, <&isp0_dart2 12>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_ispsens0>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_be>, <&ps_isp_clr>; + apple,dart-vm-size = /bits/ 64 <0xde000000>; + status = "disabled"; + sensor-presets { + preset0 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; +}; + +&isp0_dart { + compatible = "apple,t6050-isp-dart", "apple,t8110-dart"; + reg = <0x5 0x828c0000 0 0xc000>, <0x5 0x828d0000 0 0x4000>; + reg-names = "dart", "dapf"; + #iommu-cells = <1>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; +}; + +&isp0_dart1 { + compatible = "apple,t6050-isp-dart", "apple,t8110-dart"; + reg = <0x5 0x828e0000 0 0xc000>; + reg-names = "dart"; + #iommu-cells = <1>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; +}; + +&isp0_dart2 { + compatible = "apple,t6050-isp-dart", "apple,t8110-dart"; + reg = <0x5 0x82900000 0 0xc000>; + reg-names = "dart"; + #iommu-cells = <1>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + status = "disabled"; +}; + +&pmgr1 { + ps_isp_cpu: power-controller@3b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; + ps_isp_fe: power-controller@3b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; + ps_isp_dprx: power-controller@3d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_dprx"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; + ps_isp_secure: power-controller@3e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_secure"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; + ps_isp_be: power-controller@3e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; + ps_isp_clr: power-controller@3f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + apple,no-auto-pm; + power-domains = <&ps_isp_sys>; + }; +}; + +&ps_fab9_soc { + /delete-property/ power-domains; + apple,no-auto-pm; + apple,firmware-owned; + status = "okay"; +}; + +&ps_isp_sys { + apple,no-auto-pm; + apple,always-on; + status = "okay"; +}; + +&ps_ispsens0 { + apple,no-auto-pm; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index 55af2f818a954..25ba638862ca0 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -621,3 +621,5 @@ apple,tunable = <0x140 0x1 0x1>, <0x2a0 0x7e 0xa>; }; + +#include "t6050-j716c-isp.dtsi" From 955b69739397d6bf969fb8877bbb2cd90b87fc09 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 04:30:48 -0500 Subject: [PATCH 336/392] arm64: dts: apple: Describe the J714s camera capture resources The 14-inch M5 Pro has the same T6050 camera as J716c. The J714sAP ADT describes the same ISP registers, the three DARTs with host streams 0 and 12, the same DAPF filter mask (0xcc3) and the same two firmware segments. Only the sensor type differs (0x74 instead of 0x75), and the loader supplies that. Include the J716c camera description. Tested on J714s: the ISP and its DARTs bind, and the camera streams 1920x1080 NV12 at 30 fps. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index d98f6ac1d63a0..b95609c723f32 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -46,3 +46,6 @@ #define T6050_J714S #include "t6050-peripherals.dtsi" + +/* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ +#include "t6050-j716c-isp.dtsi" From eade37eae073ee1fda8c171a7c92af84efe46539 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 12:03:38 -0500 Subject: [PATCH 337/392] arm64: dts: apple: t8142-j813: share the SoC nodes J815 also has Move the CPU performance states, cluster controllers, PMC, SMC and MTP nodes out of the J813 files into t8142-j813-j815.dtsi, which also pulls in the J813 CPU idle, NVMe and power includes. The J813 and J815 ADTs (25G83) describe all of these identically. Each board names its own trackpad firmware and keeps its own MTP DART. The J813 DTB keeps the same nodes, properties and values. Only node order and phandle numbers change, and dtbs_check reports the same warnings as before. The J813 power include now includes the SPMI binding header itself, because it no longer follows the USB include. Signed-off-by: Diljit Singh --- .../arm64/boot/dts/apple/t8142-j813-j815.dtsi | 302 ++++++++++++++++++ .../dts/apple/t8142-j813-peripherals.dtsi | 97 ------ .../boot/dts/apple/t8142-j813-power.dtsi | 1 + arch/arm64/boot/dts/apple/t8142-j813.dts | 195 +---------- 4 files changed, 310 insertions(+), 285 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-j815.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-j815.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-j815.dtsi new file mode 100644 index 0000000000000..811e017460685 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-j815.dtsi @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Shared by the 13- and 15-inch M5 MacBook Air (J813, J815). + * + * The J813 and J815 ADTs (25G83) describe the same CPU performance states, + * cluster controllers, PMC, SMC, MTP, ANS, watchdog and PMU. Each board + * supplies the CPUs, AIC, PMGR and MTP DART these nodes use, and names its + * trackpad firmware (the multi-touch hid-merge-personality differs). + */ +#include + +/ { + aliases { + mtp = &mtp; + keyboard = &keyboard; + }; + + /* State 1 is 300 MHz; firmware SRAM OPP tables begin at state 2. */ + /* Cap each cluster at its measured J813 firmware handoff state. */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + + opp-1584000000 { + opp-hz = /bits/ 64 <1584000000>; + opp-level = <4>; + }; + + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <5>; + }; + + opp-2352000000 { + opp-hz = /bits/ 64 <2352000000>; + opp-level = <6>; + }; + + opp-2700000000 { + opp-hz = /bits/ 64 <2700000000>; + opp-level = <7>; + }; + + opp-2964000000 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <8>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-1308000000 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <2>; + }; + + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-level = <3>; + }; + + opp-1932000000 { + opp-hz = /bits/ 64 <1932000000>; + opp-level = <4>; + }; + + opp-2220000000 { + opp-hz = /bits/ 64 <2220000000>; + opp-level = <5>; + }; + + opp-2508000000 { + opp-hz = /bits/ 64 <2508000000>; + opp-level = <6>; + }; + + opp-2820000000 { + opp-hz = /bits/ 64 <2820000000>; + opp-level = <7>; + }; + + opp-3072000000 { + opp-hz = /bits/ 64 <3072000000>; + opp-level = <8>; + }; + + opp-3300000000 { + opp-hz = /bits/ 64 <3300000000>; + opp-level = <9>; + }; + + opp-3516000000 { + opp-hz = /bits/ 64 <3516000000>; + opp-level = <10>; + }; + + /* ADT levels 11-20, measured on J813 (level 19, 4416 MHz, is below + * level 18 and left out). The SoC throttles them when hot. */ + opp-3720000000 { + opp-hz = /bits/ 64 <3720000000>; + opp-level = <11>; + }; + + opp-3876000000 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <12>; + }; + + opp-4020000000 { + opp-hz = /bits/ 64 <4020000000>; + opp-level = <13>; + }; + + opp-4176000000 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <14>; + }; + + opp-4248000000 { + opp-hz = /bits/ 64 <4248000000>; + opp-level = <15>; + }; + + opp-4272000000 { + opp-hz = /bits/ 64 <4272000000>; + opp-level = <16>; + }; + + opp-4332000000 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + + opp-4428000000 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + + opp-4464000000 { + opp-hz = /bits/ 64 <4464000000>; + opp-level = <20>; + }; + }; +}; + +&{/soc} { + cpufreq_e: performance-controller@210e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: performance-controller@211e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + /* + * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): + * floors at 0x382700000 + agent * 0x4000, per-agent states at + * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): + * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so + * no vote is honoured and DRAM and fabric stay at their lowest + * states (CPU DRAM latency 171 ns instead of 70 ns). + */ + pmc: power-management@382700000 { + compatible = "apple,t8142-pmc"; + reg = <0x3 0x82700000 0x0 0x10000>, + <0x3 0x82806000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8142-clpc"; + reg = <1>; + }; + }; + + smc_mbox: mailbox@38c608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x8c608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8142-smc", "apple,t8103-smc"; + reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + }; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + }; + + mtp: mtp@394600000 { + compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@394608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x94608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dockchannel: fifo@394b14000 { + compatible = "apple,t8142-dockchannel", "apple,dockchannel"; + reg = <3 0x94b14000 0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 3 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + /* + * ADT dockchannel-mtp/mtp-transport/multi-touch: + * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28. + * The MTP firmware cannot reach the AFE on its own ("No + * HINT_L"): after macOS left it running the trackpad + * worked, after a Linux warm reboot it stayed in reset. + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; + + stm { }; + + /* Each board names its own trackpad firmware. */ + multi-touch { + apple,power-method = <2>; + }; + + /* Layout is filled by the loader using this machine's own data. */ + keyboard: keyboard { }; + }; + }; +}; + +#include "t8142-j813-psci-idle.dtsi" +#include "t8142-j813-nvme.dtsi" +#include "t8142-j813-power.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi index 5ca6fed62568a..32415bdaa874b 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -1,11 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT -#include #include / { aliases { - mtp = &mtp; - keyboard = &keyboard; nvram = &nvram; }; @@ -172,56 +169,6 @@ #pwm-cells = <2>; }; - smc_mbox: mailbox@38c608000 { - compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <3 0x8c608000 0 0x4000>; - interrupts = , - , - , - ; - interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; - #mbox-cells = <0>; - }; - - smc: smc@38c600000 { - compatible = "apple,t8142-smc", "apple,t8103-smc"; - reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; - reg-names = "smc", "sram"; - mboxes = <&smc_mbox>; - - hwmon { - compatible = "apple,smc-hwmon"; - apple,discover-temperatures; - }; - - smc_gpio: gpio { - compatible = "apple,smc-gpio"; - gpio-controller; - #gpio-cells = <2>; - }; - }; - - mtp: mtp@394600000 { - compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", - "apple,mtp", "apple,rtk-helper-asc4"; - /* The second region is firmware SRAM, not the ASC control window. */ - reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; - reg-names = "asc", "sram"; - mboxes = <&mtp_mbox>; - iommus = <&mtp_dart 0>; - }; - - mtp_mbox: mailbox@394608000 { - compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <3 0x94608000 0 0x4000>; - interrupts = , - , - , - ; - interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; - #mbox-cells = <0>; - }; - mtp_dart: iommu@394800000 { compatible = "apple,t8142-dart", "apple,t8110-dart"; reg = <3 0x94800000 0 0xc000>; @@ -230,49 +177,5 @@ /* Keep host allocations clear of firmware's low IOVA mappings. */ apple,dma-range = <0 0x02000000 0 0x10000000>; }; - - mtp_dockchannel: fifo@394b14000 { - compatible = "apple,t8142-dockchannel", "apple,dockchannel"; - reg = <3 0x94b14000 0 0x1000>; - reg-names = "irq"; - interrupts = ; - ranges = <0 3 0x94b28000 0x20000>; - nonposted-mmio; - #address-cells = <1>; - #size-cells = <1>; - interrupt-controller; - #interrupt-cells = <2>; - - mtp_hid: input@8000 { - compatible = "apple,dockchannel-hid"; - reg = <0x8000 0x1000>, <0xc000 0x1000>, - <0x0000 0x1000>, <0x4000 0x1000>; - reg-names = "config", "data", "rmt-config", "rmt-data"; - iommus = <&mtp_dart 0>; - interrupt-parent = <&mtp_dockchannel>; - interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "tx", "rx"; - apple,fifo-size = <0x800>; - apple,helper-cpu = <&mtp>; - /* - * ADT dockchannel-mtp/mtp-transport/multi-touch: - * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28. - * The MTP firmware cannot reach the AFE on its own ("No - * HINT_L"): after macOS left it running the trackpad - * worked, after a Linux warm reboot it stayed in reset. - */ - apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; - - stm { }; - - multi-touch { - firmware-name = "apple/tpmtfw-j813.bin"; - apple,power-method = <2>; - }; - - /* Layout is filled by the loader using this machine's own data. */ - keyboard: keyboard { }; - }; - }; }; #include "t8142-j813-pmp-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi index fa080be75e8fa..353cb55763765 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi @@ -3,6 +3,7 @@ * Apple MacBook Air (13-inch, M5) J813: SMC power off / restart, the PMU on * SPMI and the watchdog. */ +#include &{/soc} { /* ADT /arm-io/wdt (wdt,t8142): restart (and the SMC's fallback). */ diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 59788190b1933..21b1688e83f8a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -165,155 +165,6 @@ interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; }; - /* State 1 is 300 MHz; firmware SRAM OPP tables begin at state 2. */ - /* Cap each cluster at its measured J813 firmware handoff state. */ - ecluster_opp: opp-table-0 { - compatible = "operating-points-v2"; - opp-shared; - - opp-300000000 { - opp-hz = /bits/ 64 <300000000>; - opp-level = <1>; - }; - - opp-972000000 { - opp-hz = /bits/ 64 <972000000>; - opp-level = <2>; - }; - - opp-1152000000 { - opp-hz = /bits/ 64 <1152000000>; - opp-level = <3>; - }; - - opp-1584000000 { - opp-hz = /bits/ 64 <1584000000>; - opp-level = <4>; - }; - - opp-1992000000 { - opp-hz = /bits/ 64 <1992000000>; - opp-level = <5>; - }; - - opp-2352000000 { - opp-hz = /bits/ 64 <2352000000>; - opp-level = <6>; - }; - - opp-2700000000 { - opp-hz = /bits/ 64 <2700000000>; - opp-level = <7>; - }; - - opp-2964000000 { - opp-hz = /bits/ 64 <2964000000>; - opp-level = <8>; - }; - }; - - pcluster_opp: opp-table-1 { - compatible = "operating-points-v2"; - opp-shared; - - opp-300000000 { - opp-hz = /bits/ 64 <300000000>; - opp-level = <1>; - }; - - opp-1308000000 { - opp-hz = /bits/ 64 <1308000000>; - opp-level = <2>; - }; - - opp-1608000000 { - opp-hz = /bits/ 64 <1608000000>; - opp-level = <3>; - }; - - opp-1932000000 { - opp-hz = /bits/ 64 <1932000000>; - opp-level = <4>; - }; - - opp-2220000000 { - opp-hz = /bits/ 64 <2220000000>; - opp-level = <5>; - }; - - opp-2508000000 { - opp-hz = /bits/ 64 <2508000000>; - opp-level = <6>; - }; - - opp-2820000000 { - opp-hz = /bits/ 64 <2820000000>; - opp-level = <7>; - }; - - opp-3072000000 { - opp-hz = /bits/ 64 <3072000000>; - opp-level = <8>; - }; - - opp-3300000000 { - opp-hz = /bits/ 64 <3300000000>; - opp-level = <9>; - }; - - opp-3516000000 { - opp-hz = /bits/ 64 <3516000000>; - opp-level = <10>; - }; - - /* ADT levels 11-20, measured on J813 (level 19, 4416 MHz, is below - * level 18 and left out). The SoC throttles them when hot. */ - opp-3720000000 { - opp-hz = /bits/ 64 <3720000000>; - opp-level = <11>; - }; - - opp-3876000000 { - opp-hz = /bits/ 64 <3876000000>; - opp-level = <12>; - }; - - opp-4020000000 { - opp-hz = /bits/ 64 <4020000000>; - opp-level = <13>; - }; - - opp-4176000000 { - opp-hz = /bits/ 64 <4176000000>; - opp-level = <14>; - }; - - opp-4248000000 { - opp-hz = /bits/ 64 <4248000000>; - opp-level = <15>; - }; - - opp-4272000000 { - opp-hz = /bits/ 64 <4272000000>; - opp-level = <16>; - }; - - opp-4332000000 { - opp-hz = /bits/ 64 <4332000000>; - opp-level = <17>; - }; - - opp-4428000000 { - opp-hz = /bits/ 64 <4428000000>; - opp-level = <18>; - }; - - opp-4464000000 { - opp-hz = /bits/ 64 <4464000000>; - opp-level = <20>; - }; - }; - clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; @@ -327,42 +178,6 @@ ranges; nonposted-mmio; - cpufreq_e: performance-controller@210e20000 { - compatible = "apple,t8142-cluster-cpufreq"; - reg = <0x2 0x10e20000 0 0x1000>; - #performance-domain-cells = <0>; - }; - - cpufreq_p: performance-controller@211e20000 { - compatible = "apple,t8142-cluster-cpufreq"; - reg = <0x2 0x11e20000 0 0x1000>; - #performance-domain-cells = <0>; - }; - - /* - * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): - * floors at 0x382700000 + agent * 0x4000, per-agent states at - * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): - * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so - * no vote is honoured and DRAM and fabric stay at their lowest - * states (CPU DRAM latency 171 ns instead of 70 ns). - */ - pmc: power-management@382700000 { - compatible = "apple,t8142-pmc"; - reg = <0x3 0x82700000 0x0 0x10000>, - <0x3 0x82806000 0x0 0x3000>; - reg-names = "voters", "rails"; - #address-cells = <1>; - #size-cells = <0>; - /* PMP, CLPC and AP agents */ - apple,static-voters = <0 1 3>; - - clpc: performance-controller@1 { - compatible = "apple,t8142-clpc"; - reg = <1>; - }; - }; - aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; @@ -396,15 +211,19 @@ #include "t8142-j813-display.dtsi" #include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" +#include "t8142-j813-j815.dtsi" #include "t8142-j813-usb.dtsi" #include "t8142-j813-avd.dtsi" #include "t8142-j813-isp.dtsi" -#include "t8142-j813-psci-idle.dtsi" -#include "t8142-j813-nvme.dtsi" #include "t8142-j813-n1.dtsi" -#include "t8142-j813-power.dtsi" #include "t8142-j813-audio.dtsi" #include "t8142-peripherals.dtsi" +&mtp_hid { + multi-touch { + firmware-name = "apple/tpmtfw-j813.bin"; + }; +}; + /* Unexposed chassis port instances remain described and disabled. */ From 491bc1b67bd0922821cd5b855dd08bc158906da9 Mon Sep 17 00:00:00 2001 From: Diljit Singh Date: Mon, 9 Nov 2026 12:03:38 -0500 Subject: [PATCH 338/392] arm64: dts: apple: t8142-j815: boot with the keyboard and trackpad The J815 tree only had the T8142 inventory: CPUs, the timer and the AIC were disabled and there was no /chosen. aura stops on the missing /chosen, and Linux could not run without a timer or an interrupt controller, so no J815 has booted from this tree. Enable the CPUs, timer, AIC, DockChannel UART, PMGR and MTP DART from the inventory. Add /chosen with the boot framebuffer and the J813 UART aliases. Include the nodes shared with J813: CPU performance states, PMC, SMC, MTP, NVMe, watchdog and PMU. The trackpad uses apple/tpmtfw-j815.bin: the J815 multi-touch hid-merge-personality is C1FD3,3, against C1FD1,4 on J813. The model name comes from the J815 ADT product description. Audio, display, USB, camera, GPU and radios stay undescribed. Their J815 wiring has not been checked, and the speaker layout differs from J813's. Not tested on J815 hardware. Signed-off-by: Diljit Singh --- .../devicetree/bindings/arm/apple.yaml | 2 +- arch/arm64/boot/dts/apple/t8142-j815.dts | 122 +++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 691716f025522..c1ba1ac03780b 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -368,7 +368,7 @@ properties: - enum: - apple,j704 # MacBook Pro (14-inch, M5, 2025) - apple,j813 # MacBook Air (13-inch, M5, 2026) - - apple,j815 # MacBook Air (J815) + - apple,j815 # MacBook Air (15-inch, M5) - const: apple,t8142 - const: apple,arm-platform diff --git a/arch/arm64/boot/dts/apple/t8142-j815.dts b/arch/arm64/boot/dts/apple/t8142-j815.dts index 383ad27316095..d7ef154a20c6c 100644 --- a/arch/arm64/boot/dts/apple/t8142-j815.dts +++ b/arch/arm64/boot/dts/apple/t8142-j815.dts @@ -1,12 +1,132 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (15-inch, M5), J815. + * + * Boots on the T8142 inventory with the resources its ADT (25G83) shares + * with J813: CPUs, timer, AIC, DockChannel UART, PMGR, CPU performance + * states, PMC, SMC, the MTP keyboard and trackpad, ANS NVMe, watchdog and + * PMU. The loader supplies usable memory, CPU release addresses and + * framebuffer geometry. Audio, display, USB, camera, GPU and radios are not + * described yet: their J815 wiring has not been checked. + */ /dts-v1/; #include "t8142.dtsi" / { - model = "Apple MacBook Air (J815)"; compatible = "apple,j815", "apple,t8142", "apple,arm-platform"; + model = "Apple MacBook Air (15-inch, M5)"; chassis-type = "laptop"; + + aliases { + serial0 = &dockchannel; + serial1 = &serial0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:1500000n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "disabled"; + }; + }; +}; + +&cpu0 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu1 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu2 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu3 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu4 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu5 { + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; + status = "okay"; +}; + +&cpu6 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu7 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu8 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&cpu9 { + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; + status = "okay"; +}; + +&{/timer} { + status = "okay"; +}; + +&aic { + status = "okay"; +}; + +&dockchannel { + status = "okay"; +}; + +&pmgr { + status = "okay"; +}; + +&mtp_dart { + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + status = "okay"; +}; + +#include "t8142-j813-j815.dtsi" + +&mtp_hid { + multi-touch { + /* ADT hid-merge-personality C1FD3,3 (J813: C1FD1,4). */ + firmware-name = "apple/tpmtfw-j815.bin"; + }; }; /* Unexposed chassis port instances remain described and disabled. */ From 11cbadb66413b29e9376bb80ddb38f43d0cc723a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:17:12 +1100 Subject: [PATCH 339/392] arm64: dts: apple: t600x-j314-j316: add CPU temperatures and thermal zones The M1 Pro and M1 Max MacBook Pros expose no CPU temperature, though their SMC reports one per CPU sensor site and macOS reads them. Add the SoC die maximum (TCMz) and the E cluster (Te00-Te02) and P cluster (Tp00-Tp0c) keys as hwmon channels, after the existing ones so their numbers stay. The keys were mapped on a J316s by loading each cluster in turn: Tp00-Tp0I follow the first P cluster, Tp0K-Tp0c the second and Te00-Te02 the E cluster. Each site has a reading, the same plus a fixed offset, and a hotspot estimate; TCMz is the hottest hotspot key. Make the SMC hwmon a thermal sensor provider and the CPUs cooling devices, and give each site a passive power allocator zone on its hotspot key that throttles only its own cluster: on at 75 C (85 C for the E site, which also warms with the P clusters), control at 105 C. A 180 s load on every CPU peaks at 95 C without throttling. There is no hot or critical trip; the SMC drives the fans and keeps its own protection. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 592 ++++++++++++++++++ 1 file changed, 592 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 9a5f76f4fb069..50643249ddb5a 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -10,6 +10,7 @@ */ #include +#include / { chassis-type = "laptop"; @@ -1028,6 +1029,597 @@ apple,no-data-below-millicelsius = <15000>; }; }; + +/* + * CPU temperatures, after the channels above so their numbers do not + * change. Each sensor site has three keys: the reading (TpX0), the same + * plus a fixed offset (TpX1: 9 C, Te01: 13.4 C), and a hotspot estimate + * (TpX2) above both. TCMz is the hottest of the hotspot keys. Under + * load on one cluster at a time, Tp00-Tp0I follow the first P cluster, + * Tp0K-Tp0c the second, and Te00-Te02 the E cluster. + */ +&smc_hwmon { + #thermal-sensor-cells = <1>; + + temperature-TCMz { + apple,key-id = "TCMz"; + label = "SoC Die Temperature"; + }; + + temperature-Te00 { + apple,key-id = "Te00"; + label = "E-cluster Te00"; + }; + + temperature-Te01 { + apple,key-id = "Te01"; + label = "E-cluster Te01"; + }; + + temperature-Te02 { + apple,key-id = "Te02"; + label = "E-cluster Te02"; + }; + + temperature-Tp00 { + apple,key-id = "Tp00"; + label = "P0-cluster Tp00"; + }; + + temperature-Tp01 { + apple,key-id = "Tp01"; + label = "P0-cluster Tp01"; + }; + + temperature-Tp02 { + apple,key-id = "Tp02"; + label = "P0-cluster Tp02"; + }; + + temperature-Tp04 { + apple,key-id = "Tp04"; + label = "P0-cluster Tp04"; + }; + + temperature-Tp05 { + apple,key-id = "Tp05"; + label = "P0-cluster Tp05"; + }; + + temperature-Tp06 { + apple,key-id = "Tp06"; + label = "P0-cluster Tp06"; + }; + + temperature-Tp08 { + apple,key-id = "Tp08"; + label = "P0-cluster Tp08"; + }; + + temperature-Tp09 { + apple,key-id = "Tp09"; + label = "P0-cluster Tp09"; + }; + + temperature-Tp0A { + apple,key-id = "Tp0A"; + label = "P0-cluster Tp0A"; + }; + + temperature-Tp0C { + apple,key-id = "Tp0C"; + label = "P0-cluster Tp0C"; + }; + + temperature-Tp0D { + apple,key-id = "Tp0D"; + label = "P0-cluster Tp0D"; + }; + + temperature-Tp0E { + apple,key-id = "Tp0E"; + label = "P0-cluster Tp0E"; + }; + + temperature-Tp0G { + apple,key-id = "Tp0G"; + label = "P0-cluster Tp0G"; + }; + + temperature-Tp0H { + apple,key-id = "Tp0H"; + label = "P0-cluster Tp0H"; + }; + + temperature-Tp0I { + apple,key-id = "Tp0I"; + label = "P0-cluster Tp0I"; + }; + + temperature-Tp0K { + apple,key-id = "Tp0K"; + label = "P1-cluster Tp0K"; + }; + + temperature-Tp0L { + apple,key-id = "Tp0L"; + label = "P1-cluster Tp0L"; + }; + + temperature-Tp0M { + apple,key-id = "Tp0M"; + label = "P1-cluster Tp0M"; + }; + + temperature-Tp0O { + apple,key-id = "Tp0O"; + label = "P1-cluster Tp0O"; + }; + + temperature-Tp0P { + apple,key-id = "Tp0P"; + label = "P1-cluster Tp0P"; + }; + + temperature-Tp0Q { + apple,key-id = "Tp0Q"; + label = "P1-cluster Tp0Q"; + }; + + temperature-Tp0S { + apple,key-id = "Tp0S"; + label = "P1-cluster Tp0S"; + }; + + temperature-Tp0T { + apple,key-id = "Tp0T"; + label = "P1-cluster Tp0T"; + }; + + temperature-Tp0U { + apple,key-id = "Tp0U"; + label = "P1-cluster Tp0U"; + }; + + temperature-Tp0W { + apple,key-id = "Tp0W"; + label = "P1-cluster Tp0W"; + }; + + temperature-Tp0X { + apple,key-id = "Tp0X"; + label = "P1-cluster Tp0X"; + }; + + temperature-Tp0Y { + apple,key-id = "Tp0Y"; + label = "P1-cluster Tp0Y"; + }; + + temperature-Tp0a { + apple,key-id = "Tp0a"; + label = "P1-cluster Tp0a"; + }; + + temperature-Tp0b { + apple,key-id = "Tp0b"; + label = "P1-cluster Tp0b"; + }; + + temperature-Tp0c { + apple,key-id = "Tp0c"; + label = "P1-cluster Tp0c"; + }; +}; + +/* The zones below throttle the clusters through cpufreq. */ +&cpu_e00 { + #cooling-cells = <2>; +}; + +&cpu_e01 { + #cooling-cells = <2>; +}; + +&cpu_p00 { + #cooling-cells = <2>; +}; + +&cpu_p01 { + #cooling-cells = <2>; +}; + +&cpu_p02 { + #cooling-cells = <2>; +}; + +&cpu_p03 { + #cooling-cells = <2>; +}; + +&cpu_p10 { + #cooling-cells = <2>; +}; + +&cpu_p11 { + #cooling-cells = <2>; +}; + +&cpu_p12 { + #cooling-cells = <2>; +}; + +&cpu_p13 { + #cooling-cells = <2>; +}; + +/ { + /* + * One zone per sensor site on its hotspot key, throttling only the + * cluster the site sits on. The power allocator starts at 75 C, above + * the 46-66 C idle readings, and controls at 105 C. The E site also + * warms with the P clusters, so its zone starts at 85 C: the governor + * grants no power to an idle cluster and would park E at its lowest + * speed. A 180 s load on every CPU, with the fans under SMC control, + * peaks at 95 C on the hottest site and 87 C on the E site without + * any throttling, so these zones are a backstop. There is no hot or + * critical trip: the SMC drives the fans and keeps its own + * protection. sustainable-power is the energy model's four cores at + * 2448 MHz per P cluster and both cores at full speed for E. + */ + thermal-zones { + cpu-e0-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <800>; + thermal-sensors = <&smc_hwmon 11>; /* Te02 */ + + trips { + cpu_e0_switch_on: trip-switch-on { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e0_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_e00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_e01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p00-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 14>; /* Tp02 */ + + trips { + cpu_p00_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p00_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p00_control>; + cooling-device = + <&cpu_p00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p02 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p03 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p01-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 17>; /* Tp06 */ + + trips { + cpu_p01_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p01_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p01_control>; + cooling-device = + <&cpu_p00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p02 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p03 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p02-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 20>; /* Tp0A */ + + trips { + cpu_p02_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p02_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p02_control>; + cooling-device = + <&cpu_p00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p02 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p03 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p03-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 23>; /* Tp0E */ + + trips { + cpu_p03_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p03_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p03_control>; + cooling-device = + <&cpu_p00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p02 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p03 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p04-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 26>; /* Tp0I */ + + trips { + cpu_p04_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p04_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p04_control>; + cooling-device = + <&cpu_p00 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p01 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p02 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p03 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p10-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 29>; /* Tp0M */ + + trips { + cpu_p10_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p10_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p10_control>; + cooling-device = + <&cpu_p10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p12 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p13 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p11-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 32>; /* Tp0Q */ + + trips { + cpu_p11_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p11_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p11_control>; + cooling-device = + <&cpu_p10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p12 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p13 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p12-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 35>; /* Tp0U */ + + trips { + cpu_p12_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p12_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p12_control>; + cooling-device = + <&cpu_p10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p12 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p13 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p13-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 38>; /* Tp0Y */ + + trips { + cpu_p13_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p13_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p13_control>; + cooling-device = + <&cpu_p10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p12 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p13 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + cpu-p14-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + sustainable-power = <11600>; + thermal-sensors = <&smc_hwmon 41>; /* Tp0c */ + + trips { + cpu_p14_switch_on: trip-switch-on { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p14_control: trip-control { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p14_control>; + cooling-device = + <&cpu_p10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p12 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p13 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + }; +}; + #include "isp-imx558.dtsi" &isp { From 3a9734ec2d4e5b914e48a8973f88a8552101c47f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:28:03 +1100 Subject: [PATCH 340/392] arm64: configs: asahi: build the Apple debug and register views as modules Enable CONFIG_APPLE_DBGVIEW and CONFIG_APPLE_REGVIEW as modules. Both drivers only expose root-only, read-only debugfs files (AIC timebase, debug dock status, memory cache controller registers) and nothing loads them by default, so building them changes nothing on a running system until someone loads the module to inspect the hardware. Assisted-by: LLM --- arch/arm64/configs/asahi.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 76919c3e257d8..126f60eee6c74 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -105,6 +105,8 @@ CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_APPLE_DBGVIEW=m +CONFIG_APPLE_REGVIEW=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y CONFIG_APPLE_MAILBOX=y From b38fc1559aea784394fd034dea417642c61cfaf2 Mon Sep 17 00:00:00 2001 From: AxxzyWasTaken Date: Mon, 9 Nov 2026 17:33:17 +0000 Subject: [PATCH 341/392] arm64: dts: apple: t6041-j616c: enable the ISP iBoot places the ISP firmware differently on each boot here, so the J616s placement does not hold. The loader fills the firmware regions from the ADT, so add them as placeholders with the J616s addresses and enable the ISP: its alias, which aura needs to open the DAPF, the firmware regions and heap, the node with its platform ID, camera connection scheme and two IMX958 presets (the 720p one keeps the full frame, as #1005 did for the other boards), and the three DARTs. This goes in the J616c board file, not in t6041-j614c-j616c.dtsi: there is no J614c ADT yet, so its camera setup is unchecked. Signed-off-by: AxxzyWasTaken --- arch/arm64/boot/dts/apple/t6041-j616c.dts | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6041-j616c.dts b/arch/arm64/boot/dts/apple/t6041-j616c.dts index 9e122229805c5..f6a2ad6e49e21 100644 --- a/arch/arm64/boot/dts/apple/t6041-j616c.dts +++ b/arch/arm64/boot/dts/apple/t6041-j616c.dts @@ -16,6 +16,35 @@ / { compatible = "apple,j616c", "apple,t6041", "apple,arm-platform"; model = "Apple MacBook Pro (16-inch, M4 Max)"; + + /* aura opens a co-processor's DAPF only if it finds its alias. */ + aliases { + isp = &isp; + }; + + reserved-memory { + /* + * iBoot moves the ISP firmware each boot: the loader fills + * these placeholders from the ADT. It runs at 0x100_0000_0000 + * onwards, in its 32 MiB main memory. + */ + isp_fw_text: isp-firmware-text@10000c44000 { + reg = <0x100 0x00c44000 0x0 0xca8000>; + iommu-addresses = <&isp 0x100 0x0 0x0 0xca8000>; + no-map; + }; + + isp_fw_data: isp-firmware-data@10001e7c000 { + reg = <0x100 0x01e7c000 0x0 0x1344000>; + iommu-addresses = <&isp 0x100 0xca8000 0x0 0x1344000>; + no-map; + }; + + /* the firmware's main memory, the driver allocates above it */ + isp_heap: isp-heap { + iommu-addresses = <&isp 0x100 0x0 0x0 0x2000000>; + }; + }; }; &mtp_hid { @@ -56,3 +85,42 @@ &mesa { firmware-name = "apple/mesacal-j616c.bin", "apple/msrk-j616c.bin"; }; + +&isp { + memory-region = <&isp_fw_text>, <&isp_fw_data>, <&isp_heap>; + memory-region-names = "text", "data", "heap"; + apple,platform-id = <5>; + /* ADT isp0 cam-connections-scheme */ + apple,cam-connections-scheme = <0x10>; + status = "okay"; + + /* The front camera, an IMX958: firmware configuration 5. */ + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1280x720 */ + preset1 { + apple,config-index = <5>; + apple,input-size = <1920 2160>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + }; +}; + +&isp_dart0 { + status = "okay"; +}; + +&isp_dart1 { + status = "okay"; +}; + +&isp_dart2 { + status = "okay"; +}; From 7e00a518668471a8cd42255ea21b39e797d0d862 Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:04:55 +0100 Subject: [PATCH 342/392] arm64: dts: apple: t8132-j604: cold-start the MTP The J713 tree keeps the MTP on DART stream 1, for the MTP its iBoot leaves running. The J604's iBoot leaves the MTP stopped: its ASC CPU_CONTROL already reads 0 in the first-stage loader. No loader programmed the dart-mtp DAPF (aura's T8132 handoff ran only on the J713, and the MTP has no PMGR power state there, so it is skipped), and Linux could not start it ("RTKit: IOP power acknowledgment failed: -62"): no internal keyboard or trackpad. Ask the loader to program the dart-mtp filter for the halted MTP (linux-enablement-mac,dapf-halted-handoff, aura's halted-consumer writer) and let Linux cold-start it on stream 0, as the M4 Pro and Max trees do: mtp and mtp_hid on stream 0, and streams 1-3 remapped to 0, as the J604 ADT's dart-mtp "remap" (1 2 3) says. Tested on a J604 (26.6.2): the MTP starts, the trackpad firmware loads, and the internal keyboard and trackpad work. Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8132-j604.dts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts index 09a2730f1c442..9d01836e856a9 100644 --- a/arch/arm64/boot/dts/apple/t8132-j604.dts +++ b/arch/arm64/boot/dts/apple/t8132-j604.dts @@ -22,6 +22,15 @@ compatible = "apple,j604", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M4, 2024)"; + /* + * iBoot leaves the J604's MTP stopped, where the J713's runs it, so the + * loader programs its DART filter as for a halted consumer and Linux + * cold-starts it. + */ + chosen { + linux-enablement-mac,dapf-halted-handoff = "/arm-io/mtp"; + }; + /* * The AOP firmware runs the stream: the J616s frame, six playback and * twelve sense slots, one serializer for all six amps. The codecs are @@ -309,7 +318,22 @@ pwren-gpios = <&smc_gpio 0x19 GPIO_ACTIVE_HIGH>; }; +/* + * The cold-started MTP runs on DART stream 0, as on the M4 Pro and Max, + * instead of the stream 1 the J713's running MTP is handed over on. + * ADT "remap": streams 1 2 3 are stream 0. + */ +&mtp_dart { + apple,remap-streams = <1 0>, <2 0>, <3 0>; +}; + +&mtp { + iommus = <&mtp_dart 0>; +}; + &mtp_hid { + iommus = <&mtp_dart 0>; + multi-touch { firmware-name = "apple/tpmtfw-j604.bin"; }; From 397dfec2c3cf3b260ea95106774e9cd324423568 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 12:42:25 -0500 Subject: [PATCH 343/392] arm64: dts: apple: t6050-j714s: Run the PMP and report the GPU to it Enable the PMP on J714s with its boot arguments and firmware properties (J714sAP ADT), its report, and the PMC's agent 0, which the PMC now honours once the PMP runs. Add the GPU's request (AGX, soc-device 16) to the T6050 report on the GPU's power state, and let the GPU take it as its domain, as T6041 does; the GPU and the report are enabled together so the GPU never waits on a disabled report. With the report the PMP drops iBoot's static vote (DCS 5, fabric 1) to 0 and acknowledges every request. Told the GPU is on, it votes DCS 5 and the fabric 1-3 while the GPU works and 0 when it idles; the PMC takes the higher of its vote and the CLPC's. Signed-off-by: Diljit Singh --- .../dts/apple/t6050-j714s-pmp-tunables.dtsi | 157 ++++++++++++++++++ arch/arm64/boot/dts/apple/t6050-j714s.dts | 59 ++++++- .../boot/dts/apple/t6050-peripherals.dtsi | 14 ++ 3 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t6050-j714s-pmp-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t6050-j714s-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t6050-j714s-pmp-tunables.dtsi new file mode 100644 index 0000000000000..030479030c80f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714s-pmp-tunables.dtsi @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J714s PMP firmware properties: the ADT /arm-io/pmp0/iop-pmp0-nub values the + * firmware queries by name at boot (pmp.rs serves apple,tunable-). + * From the J714sAP 27.0 ADT; raw ADT bytes. + */ + +&pmp0 { + apple,tunable-afc-turbo-enable = [00 00 00 00]; + apple,tunable-afi-bw-enable = [01 00 00 00]; + apple,tunable-afi-bw-threshold = [3d 0a 57 00 00 00 00 00 00 00 00 00 03 00 00 00 cc 4c ae 00 00 00 00 00 3d 0a 56 00 03 00 00 00 00 00 ff ff 00 00 00 00 cc 4c ad 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-afi-sld-bw-threshold = [66 66 26 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 25 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-agx-af-bw-constraint = [00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00]; + apple,tunable-agx-af-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-af-bw-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 eb 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 42 01 00 00 00 00 5c cf 07 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 40 6a 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 c0 ad 00 00 00 00 00 00 00 00 00 06 00 00 00 00 80 42 01 00 00 00 00 f5 e8 c2 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 40 6a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 80 70 00 00 00 00 00 00 00 00 00 06 00 00 00 00 80 42 01 00 00 00 00 00 00 7e 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 40 6a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 5e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 d2 00 00 00 00 00 a3 30 69 00 06 00 00 00 00 00 ff ff 00 00 00 00 a3 b0 eb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 80 07 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 d2 00 00 00 00 00 47 e1 07 00 06 00 00 00 00 00 ff ff 00 00 00 00 a3 b0 eb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 d2 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 a3 b0 eb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 d2 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 a3 b0 eb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 d2 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 a3 b0 eb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 af 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 5c 4f c4 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-agx-af-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-af-lat-enable = [00 00 00 00]; + apple,tunable-agx-afr-bw-constraint = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 01 00 00 00 01 00 00 00 0c 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 03 00 00 00 03 00 00 00 0c 00 00 00 04 00 00 00 04 00 00 00 0c 00 00 00 05 00 00 00 05 00 00 00 0c 00 00 00 06 00 00 00 06 00 00 00 0c 00 00 00 07 00 00 00 07 00 00 00 0c 00 00 00 08 00 00 00 08 00 00 00 0c 00 00 00 09 00 00 00 09 00 00 00 0c 00 00 00 0a 00 00 00 0a 00 00 00 0c 00 00 00 0b 00 00 00 0b 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00]; + apple,tunable-agx-afr-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-afr-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-afr-bw-enable = [01 00 00 00]; + apple,tunable-agx-afr-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 ae 47 4b 00 00 00 00 00 00 00 00 00 00 00 00 00 33 f3 74 00 00 00 00 00 ae c7 4a 00 00 00 00 00 a3 70 a4 00 00 00 00 00 33 73 74 00 00 00 00 00 7a 14 dc 00 00 00 00 00 a3 f0 a3 00 00 00 00 00 14 6e 05 01 00 00 00 00 7a 94 db 00 00 00 00 00 eb 91 34 01 00 00 00 00 14 ee 04 01 00 00 00 00 e1 fa 65 01 00 00 00 00 eb 11 34 01 00 00 00 00 ae 87 9b 01 00 00 00 00 e1 7a 65 01 00 00 00 00 99 d9 d3 01 00 00 00 00 ae 07 9b 01 00 00 00 00 b8 9e 0f 02 00 00 00 00 99 59 d3 01 00 00 00 00 eb d1 4e 02 00 00 00 00 b8 1e 0f 02 00 00 00 00 8f 82 96 02 00 00 00 00 eb 51 4e 02 00 00 00 00 00 00 ff ff 00 00 00 00 8f 02 96 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 8f 82 68 00 00 00 00 00 00 00 00 00 00 00 00 00 f5 28 94 00 00 00 00 00 8f 02 68 00 00 00 00 00 33 b3 c7 00 00 00 00 00 f5 a8 93 00 00 00 00 00 a3 b0 ee 00 00 00 00 00 33 33 c7 00 00 00 00 00 ae 47 1b 01 00 00 00 00 a3 30 ee 00 00 00 00 00 e1 3a 4a 01 00 00 00 00 ae c7 1a 01 00 00 00 00 1e 45 7d 01 00 00 00 00 e1 ba 49 01 00 00 00 00 47 21 b3 01 00 00 00 00 1e c5 7c 01 00 00 00 00 a3 70 ec 01 00 00 00 00 47 a1 b2 01 00 00 00 00 a3 30 29 02 00 00 00 00 a3 f0 eb 01 00 00 00 00 28 1c 6e 02 00 00 00 00 a3 b0 28 02 00 00 00 00 00 00 ff ff 00 00 00 00 28 9c 6d 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 d7 63 8a 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 bb 00 00 00 00 00 d7 e3 89 00 00 00 00 00 cc 0c e1 00 00 00 00 00 51 f8 ba 00 00 00 00 00 99 19 0c 01 00 00 00 00 cc 8c e0 00 00 00 00 00 7a 94 39 01 00 00 00 00 99 99 0b 01 00 00 00 00 28 1c 6b 01 00 00 00 00 7a 14 39 01 00 00 00 00 00 80 9f 01 00 00 00 00 28 9c 6a 01 00 00 00 00 0a 57 d7 01 00 00 00 00 00 00 9f 01 00 00 00 00 28 9c 12 02 00 00 00 00 0a d7 d6 01 00 00 00 00 b8 de 55 02 00 00 00 00 28 1c 12 02 00 00 00 00 00 00 ff ff 00 00 00 00 b8 5e 55 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 eb 51 b3 00 00 00 00 00 00 00 00 00 00 00 00 00 c2 f5 d7 00 00 00 00 00 eb d1 b2 00 00 00 00 00 70 fd 01 01 00 00 00 00 c2 75 d7 00 00 00 00 00 e1 7a 2e 01 00 00 00 00 70 7d 01 01 00 00 00 00 8f 02 5f 01 00 00 00 00 e1 fa 2d 01 00 00 00 00 85 6b 92 01 00 00 00 00 8f 82 5e 01 00 00 00 00 1e 45 c9 01 00 00 00 00 85 eb 91 01 00 00 00 00 5c 8f 03 02 00 00 00 00 1e c5 c8 01 00 00 00 00 c2 b5 45 02 00 00 00 00 5c 0f 03 02 00 00 00 00 00 00 ff ff 00 00 00 00 c2 35 45 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 c2 f5 d7 00 00 00 00 00 00 00 00 00 00 00 00 00 70 fd 01 01 00 00 00 00 c2 75 d7 00 00 00 00 00 e1 7a 2e 01 00 00 00 00 70 7d 01 01 00 00 00 00 8f 02 5f 01 00 00 00 00 e1 fa 2d 01 00 00 00 00 85 6b 92 01 00 00 00 00 8f 82 5e 01 00 00 00 00 1e 45 c9 01 00 00 00 00 85 eb 91 01 00 00 00 00 5c 8f 03 02 00 00 00 00 1e c5 c8 01 00 00 00 00 c2 b5 45 02 00 00 00 00 5c 0f 03 02 00 00 00 00 00 00 ff ff 00 00 00 00 c2 35 45 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 b8 de f7 00 00 00 00 00 00 00 00 00 00 00 00 00 47 61 23 01 00 00 00 00 b8 5e f7 00 00 00 00 00 66 e6 52 01 00 00 00 00 47 e1 22 01 00 00 00 00 7a 54 85 01 00 00 00 00 66 66 52 01 00 00 00 00 33 33 bb 01 00 00 00 00 7a d4 84 01 00 00 00 00 8f 82 f4 01 00 00 00 00 33 b3 ba 01 00 00 00 00 5c 8f 35 02 00 00 00 00 8f 02 f4 01 00 00 00 00 00 00 ff ff 00 00 00 00 5c 0f 35 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ae 47 18 01 00 00 00 00 00 00 00 00 00 00 00 00 cc cc 46 01 00 00 00 00 ae c7 17 01 00 00 00 00 70 3d 78 01 00 00 00 00 cc 4c 46 01 00 00 00 00 47 21 ad 01 00 00 00 00 70 bd 77 01 00 00 00 00 33 73 e5 01 00 00 00 00 47 a1 ac 01 00 00 00 00 66 66 25 02 00 00 00 00 33 f3 e4 01 00 00 00 00 00 00 ff ff 00 00 00 00 66 e6 24 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 33 b3 3a 01 00 00 00 00 00 00 00 00 00 00 00 00 f5 28 6b 01 00 00 00 00 33 33 3a 01 00 00 00 00 5c 0f 9f 01 00 00 00 00 f5 a8 6a 01 00 00 00 00 66 66 d6 01 00 00 00 00 5c 8f 9e 01 00 00 00 00 70 3d 15 02 00 00 00 00 66 e6 d5 01 00 00 00 00 00 00 ff ff 00 00 00 00 70 bd 14 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 eb 11 5e 01 00 00 00 00 00 00 00 00 00 00 00 00 70 fd 90 01 00 00 00 00 eb 91 5d 01 00 00 00 00 99 59 c7 01 00 00 00 00 70 7d 90 01 00 00 00 00 7a 14 05 02 00 00 00 00 99 d9 c6 01 00 00 00 00 00 00 ff ff 00 00 00 00 7a 94 04 02 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 85 eb 82 01 00 00 00 00 00 00 00 00 00 00 00 00 3d 4a b8 01 00 00 00 00 85 6b 82 01 00 00 00 00 85 eb f4 01 00 00 00 00 3d ca b7 01 00 00 00 00 00 00 ff ff 00 00 00 00 85 6b f4 01 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 f5 a8 92 01 00 00 00 00 00 00 00 00 00 00 00 00 1e 85 cc 01 00 00 00 00 f5 28 92 01 00 00 00 00 00 00 ff ff 00 00 00 00 1e 05 cc 01 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 47 21 a4 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 47 a1 a3 01 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-agx-afr-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-afr-lat-enable = [00 00 00 00]; + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 02 00 00 00 06 00 00 00 03 00 00 00 03 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00]; + apple,tunable-agx-dcs-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-dcs-bw-enable = [01 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 80 22 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 48 00 00 00 00 00 00 80 21 00 00 00 00 00 00 00 69 00 00 00 00 00 00 00 47 00 00 00 00 00 66 e6 a3 00 00 00 00 00 00 00 68 00 00 00 00 00 00 00 ff ff 00 00 00 00 66 e6 a2 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 17 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 16 00 06 00 00 00 00 00 69 00 00 00 00 00 00 00 47 00 06 00 00 00 66 e6 a3 00 00 00 00 00 00 00 68 00 06 00 00 00 00 00 ff ff 00 00 00 00 66 e6 a2 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 5f 00 00 00 00 00 00 00 47 00 06 00 00 00 66 e6 a3 00 00 00 00 00 00 00 5e 00 06 00 00 00 00 00 c7 00 00 00 00 00 66 e6 a2 00 06 00 00 00 00 80 e3 00 00 00 00 00 00 00 c6 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 e2 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 5f 00 00 00 00 00 00 00 47 00 06 00 00 00 66 e6 a3 00 00 00 00 00 00 00 5e 00 06 00 00 00 00 00 c7 00 00 00 00 00 66 e6 a2 00 06 00 00 00 00 80 e3 00 00 00 00 00 00 00 c6 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 e2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 43 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 50 00 00 00 00 00 00 00 42 00 06 00 00 00 66 e6 a3 00 00 00 00 00 00 00 4f 00 06 00 00 00 00 00 c7 00 00 00 00 00 66 e6 a2 00 06 00 00 00 00 80 e3 00 00 00 00 00 00 00 c6 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 e2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 04 00 06 00 00 00 70 7d 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 c1 00 00 00 00 00 70 7d 8b 00 06 00 00 00 00 80 e3 00 00 00 00 00 00 00 c0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 e2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 04 00 06 00 00 00 70 7d 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 c1 00 00 00 00 00 70 7d 8b 00 06 00 00 00 00 80 e3 00 00 00 00 00 00 00 c0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 e2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 80 34 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 47 00 00 00 00 00 00 80 33 00 06 00 00 00 00 80 5c 00 00 00 00 00 00 00 46 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 5b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3d 00 00 00 00 00 00 00 04 00 06 00 00 00 00 80 51 00 00 00 00 00 00 00 3c 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 80 50 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 46 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 46 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 04 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 04 00 06 00 00 00]; + apple,tunable-agx-dcs-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-dcs-lat-enable = [00 00 00 00]; + apple,tunable-agx-dvfs-state-voltages = [7d 00 00 00 5d 02 00 00 7b 02 00 00 8f 02 00 00 c1 02 00 00 cb 02 00 00 e9 02 00 00 02 03 00 00 20 03 00 00 48 03 00 00 70 03 00 00 9d 03 00 00 e8 03 00 00 10 04 00 00 7d 00 00 00 7d 00 00 00]; + apple,tunable-ane-af-bw-enable = [00 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-dcs-bw-enable = [01 00 00 00]; + apple,tunable-ane-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 1d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 46 00 00 00 00 00 00 00 13 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 46 00 00 00 00 00 00 00 13 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 1d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 47 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 48 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 55 00 00 00 00 00 00 00 47 00 06 00 00 00 00 00 60 00 00 00 00 00 00 00 54 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5f 00 06 00 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ans-dcs-bw-enable = [01 00 00 00]; + apple,tunable-ans-dcs-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 99 19 00 00 05 00 00 00 5c 0f 00 00 03 00 00 00 99 19 00 00 05 00 00 00 5c 0f 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 5c 0f 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-dma-af-bw-enable = [01 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-nand-af-bw-enable = [01 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-enable = [01 00 00 00]; + apple,tunable-atc0-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc1-af-bw-enable = [01 00 00 00]; + apple,tunable-atc1-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc1-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc2-af-bw-enable = [01 00 00 00]; + apple,tunable-atc2-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc2-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc3-af-bw-enable = [01 00 00 00]; + apple,tunable-atc3-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc3-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-auss-af-bw-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-auss-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-enable = [00 00 00 00]; + apple,tunable-ave0-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave1-af-bw-enable = [00 00 00 00]; + apple,tunable-ave1-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave1-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-controller = [01 00 00 00 10 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 0f 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 4e 41 4e 44 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 4d 41 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 17 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 21 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 53 4c 44 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 20 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-dcs-bw-enable = [01 00 00 00]; + apple,tunable-dcs-bw-threshold = [47 e1 27 00 01 00 00 00 00 00 00 00 00 00 00 00 ae c7 4f 00 02 00 00 00 47 e1 26 00 00 00 00 00 3d 0a 74 00 03 00 00 00 ae c7 4e 00 00 00 00 00 7a 14 ae 00 04 00 00 00 3d 0a 73 00 00 00 00 00 7a 14 cb 00 05 00 00 00 7a 14 ad 00 00 00 00 00 99 19 e8 00 06 00 00 00 7a 14 ca 00 00 00 00 00 00 00 ff ff 07 00 00 00 99 19 e7 00 00 00 00 00]; + apple,tunable-disable-vlpm-feature = [01 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 07 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 0d 00 00 00 21 00 00 00 41 46 52 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 04 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 49 4e 54 00 00 00 00 00 00 00 00 05 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 45 58 54 00 00 00 00 00 00 00 00]; + apple,tunable-dvfs-domain-afc-bin = [01 00 00 00 07 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 0d 00 00 00 21 00 00 00 41 46 52 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 49 4e 54 00 00 00 00 00 00 00 00 05 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 45 58 54 00 00 00 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-firmware-name = [74 36 30 35 30 70 6d 70 00]; + apple,tunable-jpeg-af-bw-enable = [00 00 00 00]; + apple,tunable-jpeg-af-bw-threshold = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-jpeg-af-bw-threshold-a = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-lts-acsk-tddb-config = [01 00 00 00 69 00 00 00 10 14 24 00 00 00 00 00 95 c2 0f 00 1a 04 00 00 56 04 00 00 00 00 00 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 d2 03 00 00 00 00 66 00 33 33 66 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-lts-config = [03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 01 00 00 00 69 00 00 00 bf d9 01 00 00 00 00 00 da fc 0f 00 6e 00 00 00 66 00 00 00 00 00 11 00 33 b3 00 00 f9 fe c3 00 02 00 00 00 3b 01 00 00 cd f1 09 00 00 00 00 00 12 ef 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 2b 87 4f 00]; + apple,tunable-lts-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 0d 00 00 00 09 00 00 00 4d 41 43 43 30 00 00 00 02 00 00 00 03 00 00 00 0e 00 00 00 0a 00 00 00 4d 41 43 43 31 00 00 00 03 00 00 00 03 00 00 00 0f 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 04 00 00 00 01 00 00 00 11 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 05 00 00 00 03 00 00 00 10 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-mcache-stream = [1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 28 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00]; + apple,tunable-pre-loaded = [01 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 4e 55 4c 4c 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 10 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 0c 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 10 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 18 00 00 00 20 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 38 00 00 00 48 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 08 00 00 00 80 00 00 00 10 00 00 00 00 00 00 00 50 4d 53 2d 50 57 52 53 2d 50 4b 54 00 00 00 00 09 00 00 00 90 00 00 00 50 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 e0 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 e8 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 f0 01 00 00 30 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 20 02 00 00 1a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 3a 02 00 00 02 00 00 00 00 00 00 00 44 56 46 53 2d 44 49 45 2d 54 4f 2d 44 49 45 00 0f 00 00 00 3c 02 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 10 00 00 00 58 02 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 11 00 00 00 5a 02 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 12 00 00 00 61 02 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 13 00 00 00 6a 02 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 18 00 00 00 70 02 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 19 00 00 00 75 02 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 1a 00 00 00 7a 02 00 00 20 00 00 00 00 00 00 00 47 46 58 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 1b 00 00 00 9a 02 00 00 09 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 1c 00 00 00 a3 02 00 00 01 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 53 4d 43 00 00 1d 00 00 00 a4 02 00 00 08 00 00 00 00 00 00 00 41 4e 45 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 20 00 00 00 b4 02 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 21 00 00 00 b5 02 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 22 00 00 00 b6 02 00 00 08 00 00 00 00 00 00 00 50 4d 55 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 28 00 00 00 20 03 00 00 14 00 00 00 00 00 00 00 50 4d 53 2d 56 49 52 54 2d 43 4e 54 52 00 00 00]; + apple,tunable-region-base = [00 00 50 84 02 00 00 00]; + apple,tunable-region-size = [00 00 10 00 00 00 00 00]; + apple,tunable-segment-names = [5f 5f 54 45 58 54 3b 5f 5f 44 41 54 41 3b 5f 5f 45 54 45 58 54 00]; + apple,tunable-segment-ranges = [00 00 50 84 02 00 00 00 00 00 00 01 00 00 00 00 00 00 50 84 02 00 00 00 00 80 05 00 03 00 00 00 00 80 55 84 02 00 00 00 00 80 05 01 00 00 00 00 00 80 55 84 02 00 00 00 00 b0 09 00 06 00 00 00 00 80 d0 01 00 01 00 00 00 00 10 01 00 00 00 00 00 00 00 00 00 01 00 00 00 00 01 00 01 00 00 00]; + apple,tunable-soc-chip-variant = [00 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 03 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 04 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 05 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 06 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 08 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 09 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 00 0a 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 00 0b 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 00 0d 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 0e 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 0f 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 10 00 00 00 5f 58 46 47 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 11 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 12 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 13 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 49 4e 54 00 14 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 15 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 16 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 32 17 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 33 18 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 00 00 00 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 00 00 00 00 1a 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 1b 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 30 00 00 00 00 1c 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 31 00 00 00 00 1d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 1e 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1f 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 52 4f 52 45 53 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 22 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 23 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 24 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 00 00 00 00 25 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00]; + apple,tunable-soc-device-ps-group = [00 f0 ff ff ff 01 00 00]; + apple,tunable-soc-ni0-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni0-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni0-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni1-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni1-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni1-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni2-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni2-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni2-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni3-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni4-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni4-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni4-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni5-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni6-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni6-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-push-thr = [24 00 00 00]; + apple,tunable-soc-ni7-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni7-bw-threshold = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni7-bw-threshold-a = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni7-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni8-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni8-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni8-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni8-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni9-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni9-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-push-thr = [f5 06 00 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 19 00 00 00 1a 00 00 00 1b 00 00 00 1c 00 00 00 1d 00 00 00 1e 00 00 00 1f 00 00 00 20 00 00 00 21 00 00 00 22 00 00 00 23 00 00 00 24 00 00 00 25 00 00 00 26 00 00 00 27 00 00 00 28 00 00 00 29 00 00 00 2a 00 00 00 2b 00 00 00 2c 00 00 00 2d 00 00 00 2e 00 00 00 2f 00 00 00 30 00 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00 35 00 00 00 36 00 00 00 37 00 00 00 38 00 00 00 39 00 00 00 3a 00 00 00 3b 00 00 00 3c 00 00 00 3d 00 00 00 3e 00 00 00 3f 00 00 00 40 00 00 00 41 00 00 00 42 00 00 00 43 00 00 00 44 00 00 00 45 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 63 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 64 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 66 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 67 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 65 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 32 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 68 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 35 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 0a 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 73 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 62 69 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 38 69 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 33 69 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 34 69 00 00 00 00 00 00 00 00 00 00 20 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 30 69 00 00 00 00 00 00 00 00 00 00 21 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 31 69 00 00 00 00 00 00 00 00 00 00 22 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 36 69 00 00 00 00 00 00 00 00 00 00 23 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 37 69 00 00 00 00 00 00 00 00 00 00 24 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 35 69 00 00 00 00 00 00 00 00 00 00 25 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 32 69 00 00 00 00 00 00 00 00 00 00 26 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 38 69 00 00 00 00 00 00 00 00 00 00 27 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 28 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 29 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 39 69 00 00 00 00 00 00 00 00 00 00 2a 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 61 69 00 00 00 00 00 00 00 00 00 00 2b 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 63 69 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 64 69 00 00 00 00 00 00 00 00 00 00 2d 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 66 69 00 00 00 00 00 00 00 00 00 00 2e 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 67 69 00 00 00 00 00 00 00 00 00 00 2f 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 62 69 00 00 00 00 00 00 00 00 00 00 30 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 65 69 00 00 00 00 00 00 00 00 00 00 31 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 68 69 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 33 69 00 00 00 00 00 00 00 00 00 00 33 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 34 69 00 00 00 00 00 00 00 00 00 00 34 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 30 69 00 00 00 00 00 00 00 00 00 00 35 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 31 69 00 00 00 00 00 00 00 00 00 00 36 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 36 69 00 00 00 00 00 00 00 00 00 00 37 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 37 69 00 00 00 00 00 00 00 00 00 00 38 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 35 69 00 00 00 00 00 00 00 00 00 00 39 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 32 69 00 00 00 00 00 00 00 00 00 00 3a 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 38 69 00 00 00 00 00 00 00 00 00 00 3b 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6c 69 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6d 69 00 00 00 00 00 00 00 00 00 00 3d 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 39 69 00 00 00 00 00 00 00 00 00 00 3e 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 61 69 00 00 00 00 00 00 00 00 00 00 3f 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 63 69 00 00 00 00 00 00 00 00 00 00 40 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 64 69 00 00 00 00 00 00 00 00 00 00 41 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 66 69 00 00 00 00 00 00 00 00 00 00 42 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 67 69 00 00 00 00 00 00 00 00 00 00 43 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 62 69 00 00 00 00 00 00 00 00 00 00 44 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 65 69 00 00 00 00 00 00 00 00 00 00 45 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 68 69 00 00 00 00 00 00 00 00 00 00 46 00 00 00 01 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-uuid = [45 39 31 38 44 44 31 39 2d 37 41 42 46 2d 33 46 41 39 2d 41 38 45 45 2d 45 33 35 42 36 37 39 46 30 44 46 45 00]; + apple,tunable-vlpm-threshold = [55 01 00 00]; + apple,tunable-vlpm-timer-limit = [64 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index b95609c723f32..6fc1b66157762 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -4,9 +4,9 @@ * * target-type: J714s * - * One T6050 die (t6050-max.dtsi). Nobody has booted Linux on this board - * yet: beyond the SoC this file only adds the board's hardware inventory - * (t6050-peripherals.dtsi), all of it disabled, from the board's ADT. + * One T6050 die (t6050-max.dtsi). Beyond the SoC this file adds the board's + * hardware inventory (t6050-peripherals.dtsi) from the board's ADT, disabled + * except for the PMP below. * * Copyright The Asahi Linux Contributors */ @@ -47,5 +47,58 @@ #define T6050_J714S #include "t6050-peripherals.dtsi" +/* + * The PMP (ADT pmp0, /chosen, /arm-io/pmgr). iBoot loads its firmware and the driver starts it + * with these boot arguments and the firmware properties below. Its report tells it which devices + * are on; it then votes DCS and the fabric for GPU load itself, and the PMC honours that next + * to the CLPC. The report, the PMP's vote and the GPU's request are enabled together, so the + * GPU never waits on a disabled report. + */ +&ps_pms { + status = "okay"; +}; + +&ps_pmp { + status = "okay"; +}; + +&ps_pms_sram { + status = "okay"; +}; + +&pmp0_dart { + status = "okay"; +}; + +&pmp0_mbox { + status = "okay"; +}; + +&pmp0 { + /* ADT /chosen board-id, dram-vendor-id */ + apple,board-id = <8>; + apple,dram-vendor-id = <255>; + /* ADT /arm-io/pmgr pmc, pmc-pmgr */ + apple,pmc = <1>; + apple,pmc-pmgr = <63>; + status = "okay"; +}; + +&pmp0_report { + status = "okay"; +}; + +&pmc { + /* PMP, CLPC and AP agents, once the PMP runs */ + apple,static-voters = <0 1 3>; + apple,pmp = <&pmp0>; +}; + +&sgx { + power-domains = <&pmp0_gfx>; +}; + +#include "t6050-j714s-pmp-tunables.dtsi" + /* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ #include "t6050-j716c-isp.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index 2c82b0024ebe4..d3524080347f6 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2249,6 +2249,20 @@ apple,always-on; }; + /* + * The GPU (AGX), on the GPU's own power state, so the PMP is told the GPU is on only + * while it is powered (a request with the GPU off crashes the T6041 PMP). The GPU takes + * this domain on a board that enables the report. + */ + pmp0_gfx: report@f { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xf>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + /* The internal display (DISPINT) */ report@12 { compatible = "apple,t6050-pmp-v2-report-entry", From 7f6562857f7070531954058d0af9043595823576 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:51:10 +1100 Subject: [PATCH 344/392] arm64: dts: apple: t8112-j413: add the remaining CPU temperatures The J413 describes five of its CPU temperature keys (TCMz, Te05 and the offset keys of four P sensor sites), but its SMC reports 31, and the hottest site, Tp0o, is not among them. Add the other 26 as read-only hwmon channels after the GPU ones, so no existing channel or thermal zone index moves: TCMb, Te04 and Te06 for the E cluster and every Tp key for the P cluster. They were mapped by loading each cluster in turn: Te04-Te06 rise 8 C with the E cluster, the Tp keys 15-22 C with the P cluster. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- arch/arm64/boot/dts/apple/t8112-j413.dts | 140 +++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index f9b4883ec2c96..28d8d8e75bb3c 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -321,6 +321,146 @@ apple,no-data-below-millicelsius = <10000>; }; }; + +/* + * The other CPU temperatures, read-only, after the GPU channels. Each + * sensor site has a reading, the same plus a fixed offset and a hotspot + * estimate; the zones above use the offset keys of four P sites and Te05. + * Te04-Te06 follow the E cluster, every Tp key the P cluster, TCMz is the + * hottest hotspot key and TCMb the hottest offset key. + */ +&smc_hwmon { + temperature-TCMb { + apple,key-id = "TCMb"; + label = "SoC Die TCMb"; + }; + + temperature-Te04 { + apple,key-id = "Te04"; + label = "E-cluster Te04"; + }; + + temperature-Te06 { + apple,key-id = "Te06"; + label = "E-cluster Te06"; + }; + + temperature-Tp00 { + apple,key-id = "Tp00"; + label = "P-cluster Tp00"; + }; + + temperature-Tp02 { + apple,key-id = "Tp02"; + label = "P-cluster Tp02"; + }; + + temperature-Tp04 { + apple,key-id = "Tp04"; + label = "P-cluster Tp04"; + }; + + temperature-Tp06 { + apple,key-id = "Tp06"; + label = "P-cluster Tp06"; + }; + + temperature-Tp08 { + apple,key-id = "Tp08"; + label = "P-cluster Tp08"; + }; + + temperature-Tp0A { + apple,key-id = "Tp0A"; + label = "P-cluster Tp0A"; + }; + + temperature-Tp0C { + apple,key-id = "Tp0C"; + label = "P-cluster Tp0C"; + }; + + temperature-Tp0E { + apple,key-id = "Tp0E"; + label = "P-cluster Tp0E"; + }; + + temperature-Tp0a { + apple,key-id = "Tp0a"; + label = "P-cluster Tp0a"; + }; + + temperature-Tp0b { + apple,key-id = "Tp0b"; + label = "P-cluster Tp0b"; + }; + + temperature-Tp0c { + apple,key-id = "Tp0c"; + label = "P-cluster Tp0c"; + }; + + temperature-Tp0e { + apple,key-id = "Tp0e"; + label = "P-cluster Tp0e"; + }; + + temperature-Tp0f { + apple,key-id = "Tp0f"; + label = "P-cluster Tp0f"; + }; + + temperature-Tp0g { + apple,key-id = "Tp0g"; + label = "P-cluster Tp0g"; + }; + + temperature-Tp0i { + apple,key-id = "Tp0i"; + label = "P-cluster Tp0i"; + }; + + temperature-Tp0j { + apple,key-id = "Tp0j"; + label = "P-cluster Tp0j"; + }; + + temperature-Tp0k { + apple,key-id = "Tp0k"; + label = "P-cluster Tp0k"; + }; + + temperature-Tp0m { + apple,key-id = "Tp0m"; + label = "P-cluster Tp0m"; + }; + + temperature-Tp0n { + apple,key-id = "Tp0n"; + label = "P-cluster Tp0n"; + }; + + temperature-Tp0o { + apple,key-id = "Tp0o"; + label = "P-cluster Tp0o"; + }; + + temperature-Tp0q { + apple,key-id = "Tp0q"; + label = "P-cluster Tp0q"; + }; + + temperature-Tp0r { + apple,key-id = "Tp0r"; + label = "P-cluster Tp0r"; + }; + + temperature-Tp0s { + apple,key-id = "Tp0s"; + label = "P-cluster Tp0s"; + }; +}; + /* * Mesa fingerprint sensor on spi2. The J413 platform device tree * (/arm-io/spi2/mesa) describes it as J415's does: power on AP GPIO 178, From ca707626b31e353b3f9bb5ceb39cd1f511512fe3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 12:16:36 -0500 Subject: [PATCH 345/392] Remove the APIF platform drivers APIF is old SPTM support that doesn't need to be in tree. Drop the APIF MMIO transport, its DART and SART providers, the APIF mailbox profile, its config fragment and selftests, and the headers only APIF consumers used. SART goes back to its MMIO path. Signed-off-by: Diljit Singh --- arch/arm64/configs/aurora-apif.config | 6 - drivers/firmware/Kconfig | 10 - drivers/firmware/Makefile | 2 - drivers/firmware/aurora-apif-mmio.c | 255 -- drivers/iommu/Kconfig | 12 - drivers/iommu/Makefile | 2 - drivers/iommu/apple-dart-apif.c | 2077 ----------------- drivers/soc/apple/Kconfig | 10 - drivers/soc/apple/Makefile | 2 - drivers/soc/apple/apple-sart-apif.c | 167 -- drivers/soc/apple/mailbox.c | 27 - drivers/soc/apple/sart-backend.h | 21 - drivers/soc/apple/sart.c | 32 - include/linux/apple-dart-apif.h | 18 - include/linux/pci-apple-apif.h | 16 - .../linux/soc/apple/display-event-debugfs.h | 85 - include/linux/soc/apple/display-event-log.h | 213 -- include/linux/soc/aurora/apif-mmio.h | 66 - tools/testing/selftests/Makefile | 1 - tools/testing/selftests/aurora-apif/Makefile | 5 - .../selftests/aurora-apif/apif_mmio.py | 409 ---- tools/testing/selftests/aurora-apif/common.py | 19 - .../selftests/aurora-apif/dart_retained.py | 53 - 23 files changed, 3508 deletions(-) delete mode 100644 arch/arm64/configs/aurora-apif.config delete mode 100644 drivers/firmware/aurora-apif-mmio.c delete mode 100644 drivers/iommu/apple-dart-apif.c delete mode 100644 drivers/soc/apple/apple-sart-apif.c delete mode 100644 drivers/soc/apple/sart-backend.h delete mode 100644 include/linux/apple-dart-apif.h delete mode 100644 include/linux/pci-apple-apif.h delete mode 100644 include/linux/soc/apple/display-event-debugfs.h delete mode 100644 include/linux/soc/apple/display-event-log.h delete mode 100644 include/linux/soc/aurora/apif-mmio.h delete mode 100644 tools/testing/selftests/aurora-apif/Makefile delete mode 100755 tools/testing/selftests/aurora-apif/apif_mmio.py delete mode 100644 tools/testing/selftests/aurora-apif/common.py delete mode 100755 tools/testing/selftests/aurora-apif/dart_retained.py diff --git a/arch/arm64/configs/aurora-apif.config b/arch/arm64/configs/aurora-apif.config deleted file mode 100644 index dbc1e64573992..0000000000000 --- a/arch/arm64/configs/aurora-apif.config +++ /dev/null @@ -1,6 +0,0 @@ -# Common T6050 APIF providers. Add the platform-wiring PR for all consumers. -CONFIG_ARM64_16K_PAGES=y -CONFIG_AURORA_APIF_MMIO=y -CONFIG_APPLE_DART_APIF=y -CONFIG_APPLE_SART=y -CONFIG_APPLE_SART_APIF=y diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index c4f9285d3f362..b7cc11e4fbfa6 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -6,16 +6,6 @@ menu "Firmware Drivers" -config AURORA_APIF_MMIO - tristate "Aurora platform MMIO interface" - depends on ARM64 && OF - help - Submit synchronous, batched platform operations to an Aura monitor. - The interface forwards native SPTM selectors, arguments and results - without interpreting peripheral policy. Clients own their buffers. - Submission is synchronous and supports atomic-context consumers. - - source "drivers/firmware/arm_scmi/Kconfig" config ARM_SCPI_PROTOCOL diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 8d5c56e4cfbf2..be46f1e1dc77f 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -39,5 +39,3 @@ obj-y += samsung/ obj-y += smccc/ obj-y += tegra/ obj-y += xilinx/ - -obj-$(CONFIG_AURORA_APIF_MMIO) += aurora-apif-mmio.o diff --git a/drivers/firmware/aurora-apif-mmio.c b/drivers/firmware/aurora-apif-mmio.c deleted file mode 100644 index aa6ec65c6248c..0000000000000 --- a/drivers/firmware/aurora-apif-mmio.c +++ /dev/null @@ -1,255 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Aurora Platform Interface over Memory-Mapped I/O. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define APIF_MAGIC 0x4f494d4d46495041ULL -#define APIF_VERSION 2 -#define APIF_REG_MAGIC 0x000 -#define APIF_REG_VERSION 0x008 -#define APIF_REG_MAX_OPS 0x010 -#define APIF_REG_FEATURES 0x018 -#define APIF_REG_DOORBELL 0x100 -#define APIF_FEAT_NATIVE BIT_ULL(0) -#define APIF_BATCH_BYTES SZ_16K - -struct apif_batch { - u32 count, flags; - s32 status; - u32 done; - struct aurora_apif_op ops[]; -}; - -#define APIF_BATCH_MAX \ - ((APIF_BATCH_BYTES - sizeof(struct apif_batch)) / \ - sizeof(struct aurora_apif_op)) - -struct apif_provider { - void __iomem *regs; - u32 max_ops; -}; - -struct aurora_apif { - struct kref refs; - raw_spinlock_t lock; - bool live; - void __iomem *doorbell; - struct apif_batch *batch; - u32 max_ops; -}; - -static struct platform_driver apif_driver; -static_assert(sizeof(struct apif_batch) == 16); -static_assert(sizeof(struct aurora_apif_op) == 64); -static_assert(sizeof(struct aurora_apif_boot_iommu) == 64); - -/* SPTM redispatch restores SP_EL1 from its exception-stack contract. Keep the - * Linux stack in a preserved GPR across the trapping store. The monitor must - * preserve all GPRs, including x9, and return synchronously after the batch. - */ -static void apif_ring(struct aurora_apif *apif) -{ - u64 pa = virt_to_phys(apif->batch); - - /* Publish the request before its synchronous doorbell. */ - wmb(); - asm volatile("mov x9, sp\n" - "str %x[pa], [%[db]]\n" - "mov sp, x9\n" - : - : [pa] "r"(pa), [db] "r"(apif->doorbell) - : "x9", "memory"); - /* Observe completion and returned fields after redispatch. */ - rmb(); -} - -int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, - u32 count, u32 *done) -{ - unsigned long flags; - u32 total = 0; - int ret = 0; - - if (done) - *done = 0; - if (!apif || (count && !ops)) - return -EINVAL; - raw_spin_lock_irqsave(&apif->lock, flags); - if (!apif->live) { - ret = -ENODEV; - goto out; - } - while (total < count) { - u32 n = min(count - total, apif->max_ops); - u32 reported; - - apif->batch->count = n; - apif->batch->flags = 0; - apif->batch->status = -EINPROGRESS; - apif->batch->done = 0; - memcpy(apif->batch->ops, ops + total, n * sizeof(*ops)); - apif_ring(apif); - reported = READ_ONCE(apif->batch->done); - if (reported > n) { - ret = -EIO; - break; - } - memcpy(ops + total, apif->batch->ops, n * sizeof(*ops)); - total += reported; - ret = READ_ONCE(apif->batch->status); - if (ret || reported != n) { - if (ret >= 0 || ret == -EINPROGRESS) - ret = -EIO; - break; - } - } -out: - raw_spin_unlock_irqrestore(&apif->lock, flags); - if (done) - *done = total; - return ret; -} -EXPORT_SYMBOL_GPL(aurora_apif_submit); - -static void apif_free(struct kref *refs) -{ - kfree(container_of(refs, struct aurora_apif, refs)); -} - -void aurora_apif_put(struct aurora_apif *apif) -{ - kref_put(&apif->refs, apif_free); -} -EXPORT_SYMBOL_GPL(aurora_apif_put); - -static void apif_revoke(void *data) -{ - struct aurora_apif *apif = data; - unsigned long flags; - - raw_spin_lock_irqsave(&apif->lock, flags); - apif->live = false; - apif->doorbell = NULL; - apif->batch = NULL; - raw_spin_unlock_irqrestore(&apif->lock, flags); - aurora_apif_put(apif); -} - -struct aurora_apif *aurora_apif_get(struct device *dev) -{ - struct device_node *node; - struct platform_device *supplier; - struct apif_provider *provider; - struct aurora_apif *apif; - struct device_link *link; - int ret; - - node = of_parse_phandle(dev->of_node, "aurora,apif", 0); - if (!node) - return ERR_PTR(-ENODEV); - supplier = of_find_device_by_node(node); - of_node_put(node); - if (!supplier) - return ERR_PTR(-EPROBE_DEFER); - if (supplier->dev.driver != &apif_driver.driver) { - ret = -EPROBE_DEFER; - goto put_device; - } - link = device_link_add(dev, &supplier->dev, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!link) { - ret = -ENOMEM; - goto put_device; - } - provider = platform_get_drvdata(supplier); - if (!provider) { - ret = -EPROBE_DEFER; - goto remove_link; - } - apif = kzalloc(sizeof(*apif), GFP_KERNEL); - if (!apif) { - ret = -ENOMEM; - goto remove_link; - } - kref_init(&apif->refs); - raw_spin_lock_init(&apif->lock); - apif->batch = devm_kzalloc(dev, APIF_BATCH_BYTES, GFP_KERNEL); - if (!apif->batch) { - ret = -ENOMEM; - aurora_apif_put(apif); - goto remove_link; - } - apif->max_ops = provider->max_ops; - apif->doorbell = provider->regs + APIF_REG_DOORBELL; - apif->live = true; - kref_get(&apif->refs); - ret = devm_add_action_or_reset(dev, apif_revoke, apif); - if (ret) { - aurora_apif_put(apif); - goto remove_link; - } - put_device(&supplier->dev); - return apif; -remove_link: - device_link_del(link); -put_device: - put_device(&supplier->dev); - return ERR_PTR(ret); -} -EXPORT_SYMBOL_GPL(aurora_apif_get); - -static int apif_probe(struct platform_device *pdev) -{ - struct apif_provider *provider; - struct resource *resource; - u64 max_ops; - - resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!resource || - resource_size(resource) < APIF_REG_DOORBELL + sizeof(u64)) - return -EINVAL; - provider = devm_kzalloc(&pdev->dev, sizeof(*provider), GFP_KERNEL); - if (!provider) - return -ENOMEM; - provider->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(provider->regs)) - return PTR_ERR(provider->regs); - if (readq(provider->regs + APIF_REG_MAGIC) != APIF_MAGIC || - readq(provider->regs + APIF_REG_VERSION) != APIF_VERSION || - !(readq(provider->regs + APIF_REG_FEATURES) & APIF_FEAT_NATIVE)) - return dev_err_probe(&pdev->dev, -ENODEV, - "unsupported APIF interface\n"); - max_ops = readq(provider->regs + APIF_REG_MAX_OPS); - if (!max_ops) - return -EINVAL; - provider->max_ops = min_t(u64, max_ops, APIF_BATCH_MAX); - platform_set_drvdata(pdev, provider); - return 0; -} - -static const struct of_device_id apif_of_match[] = { - { .compatible = "aurora,apif-mmio" }, - {} -}; -MODULE_DEVICE_TABLE(of, apif_of_match); - -static struct platform_driver apif_driver = { - .driver = { - .name = "aurora-apif-mmio", - .of_match_table = apif_of_match, - }, - .probe = apif_probe, -}; -module_platform_driver(apif_driver); - -MODULE_DESCRIPTION("Aurora Platform Interface over Memory-Mapped I/O"); -MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index dd2dd23043b54..4233b253b06e2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -329,18 +329,6 @@ config APPLE_DART_J713_DCP Leave this disabled for ordinary DART use, including APPLE_DART=m. -config APPLE_DART_APIF - bool "Apple DART over the Aurora platform interface" - depends on ARM64_16K_PAGES && OF - depends on ARCH_APPLE || COMPILE_TEST - select AURORA_APIF_MMIO - select IOMMU_API - help - Apple DART backend for a trusted Linux guest under Aura and SPTM. - Native mapping and retyping policy lives in Linux; the monitor - provides call transport, frame ownership and bootstrap metadata. - Existing raw-MMIO DART drivers continue to serve other systems. - config APPLE_DART_KUNIT_TEST bool "KUnit tests for Apple DART domains and commands" if !KUNIT_ALL_TESTS depends on APPLE_DART=y && KUNIT=y diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 43893f2c03dbd..2f05725eaab18 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -38,5 +38,3 @@ obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o - -obj-$(CONFIG_APPLE_DART_APIF) += apple-dart-apif.o diff --git a/drivers/iommu/apple-dart-apif.c b/drivers/iommu/apple-dart-apif.c deleted file mode 100644 index 8b5792c1efd7e..0000000000000 --- a/drivers/iommu/apple-dart-apif.c +++ /dev/null @@ -1,2077 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Apple DART IOMMU backend using Aura's native SPTM call transport. - * Linux owns stream topology, translation-table setup, DMA apertures and - * transactional mapping bookkeeping. Aura forwards native selectors 1:1. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dma-iommu.h" - -/* Internal operations used by the IOMMU bookkeeping; not the APIF wire ABI. */ -#define APIF_DART_INIT 1 /* () INIT + POWERUP */ -#define APIF_DART_ATTACH 2 /* (sid, dva, level) */ -#define APIF_DART_MAP 3 /* (sid, dva, ipa, size, prot) */ -#define APIF_DART_UNMAP 4 /* (sid, dva, size) */ -#define APIF_DART_ENABLE 5 /* (sid) */ -#define APIF_DART_DISABLE 6 /* (sid) */ - -#define APIF_DART_RETAINED_INFO 9 -#define APIF_DART_RETAINED_RUN 10 - -struct apif_op { - u32 op; - u32 flags; - u64 arg[6]; - u64 ret; -}; - -struct apif_batch { - u32 count; - u32 flags; - s32 status; - u32 done; - struct apif_op op[]; -}; - -#define DART_APIF_PAGE_SHIFT 14 -#define DART_APIF_PAGE_SIZE (1UL << DART_APIF_PAGE_SHIFT) /* 16 KiB */ -/* apif's DART MAP forwards one contiguous run within a single 32 MiB leaf-table - * span, so every MAP op must stay inside one aligned block. - */ -#define DART_APIF_BLOCK_SHIFT 25 -#define DART_APIF_BLOCK_SIZE (1UL << DART_APIF_BLOCK_SHIFT) /* 32 MiB */ - -#define DART_APIF_MAX_STREAMS 256 -#define DART_APIF_APERTURE_END (SZ_4G - 1) -#define DART_APIF_NATIVE_APERTURE_END GENMASK_ULL(41, 0) - -/* Preserve batched per-block operations and the existing rollback accounting. */ -#define APIF_BATCH_MAX_OPS \ - ((PAGE_SIZE - sizeof(struct apif_batch)) / sizeof(struct apif_op)) - -static const struct iommu_ops apple_dart_apif_ops; - -struct apif_retained_run { - u64 dva, ipa, size; -}; -struct apif_retained_sid { - unsigned int root_level, count; - struct apif_retained_run *runs; -}; - -struct apple_dart_apif { - struct device *dev; - struct iommu_device iommu; - struct aurora_apif *apif; - struct aurora_apif_op *native_ops; - struct aurora_apif_boot_iommu *boot_info; - u64 *page_lists[APIF_BATCH_MAX_OPS]; - u64 page_list_pa[APIF_BATCH_MAX_OPS]; - u32 dart_id; - u32 num_streams; - u64 iova_force; /* Native address force mask, hidden from DMA API/device addresses. */ - u64 dma_min, dma_max; - bool reserve_last_page; - bool inited; - bool boot_mappings; /* U-Boot retained the running MTP's DMA tables. */ - bool ownership_uncertain; /* A table operation needs a fresh boot after lost completion. */ - struct apif_retained_sid *retained[DART_APIF_MAX_STREAMS]; - struct mutex init_lock; /* serialises one-time init */ - /* Protected by apple_dart_apif_groups_lock; group-owned lifetime. */ - struct iommu_group *sid2group[DART_APIF_MAX_STREAMS]; - - struct apif_batch *batch; /* internal mapping-operation buffer */ - spinlock_t batch_lock; /* protects operation buffers and stream updates */ -}; - -/* Match the native DART driver's capacity for shared display DMA domains. */ -#define APIF_MAX_DARTS 8 - -struct apif_stream_map { - struct apple_dart_apif *dart; - DECLARE_BITMAP(sids, DART_APIF_MAX_STREAMS); -}; - -struct apple_dart_apif_master { - unsigned int num_darts; - struct apif_stream_map streams[APIF_MAX_DARTS]; -}; - -struct apple_dart_apif_domain { - struct iommu_domain domain; - unsigned int num_darts; - struct apif_stream_map streams[APIF_MAX_DARTS]; - struct mutex lock; /* serialises domain attachments */ - /* Mapping calls may run concurrently on distinct ranges. An attachment - * must replay a stable mapping set before publishing its new streams. - */ - rwlock_t streams_lock; - /* Key: dart-id:8, sid:8, block:40. SPTM tables cannot be attached twice. */ - struct xarray l2_attached; - bool has_retained; - /* One canonical mapping set and IOVA allocator for every DART. */ - struct xarray mappings; -}; - -static struct apif_stream_map * -apif_find_stream(struct apple_dart_apif_domain *dom, - struct apple_dart_apif *dart) -{ - if (dom) - for (unsigned int i = 0; i < dom->num_darts; i++) - if (dom->streams[i].dart == dart) - return &dom->streams[i]; - return NULL; -} - -/* Domain topology is fixed before the first attachment. Later devices can - * add SIDs on these DARTs, but cannot change an active allocator's aperture. - */ -static int apif_domain_configure(struct apple_dart_apif_domain *dom, - const struct apple_dart_apif_master *master) -{ - u64 start = 0, end = DART_APIF_NATIVE_APERTURE_END; - - if (!master || !master->num_darts) - return -ENODEV; - if (dom->num_darts) { - for (unsigned int i = 0; i < master->num_darts; i++) - if (!apif_find_stream(dom, master->streams[i].dart)) - return -EINVAL; - return 0; - } - for (unsigned int i = 0; i < master->num_darts; i++) { - start = max(start, master->streams[i].dart->dma_min); - end = min(end, master->streams[i].dart->dma_max); - } - if (start > end) - return -EINVAL; - for (unsigned int i = 0; i < master->num_darts; i++) - dom->streams[i].dart = master->streams[i].dart; - dom->domain.geometry.aperture_start = start; - dom->domain.geometry.aperture_end = end; - dom->num_darts = master->num_darts; - return 0; -} - -static struct apple_dart_apif_domain *to_apif_domain(struct iommu_domain *dom) -{ - return container_of(dom, struct apple_dart_apif_domain, domain); -} - -/* The native DMA address space strips the force bits before returning a device DMA address - * and restores it for DART operations. Translate only the bounded transport - * batch; all live-map bookkeeping and the DMA aperture remain device-visible. - * The force bits are disjoint from the validated low-4-GiB aperture. - */ -static void apif_force_iovas(struct apple_dart_apif *dart, bool restore) -{ - if (!dart->iova_force) - return; - for (u32 i = 0; i < dart->batch->count; i++) { - struct apif_op *op = &dart->batch->op[i]; - - switch (op->op) { - case APIF_DART_ATTACH: - case APIF_DART_MAP: - case APIF_DART_UNMAP: - if (restore) - op->arg[1] &= ~dart->iova_force; - else - op->arg[1] |= dart->iova_force; - break; - } - } -} - -static void apif_put_transport(void *data) -{ - aurora_apif_put(data); -} - -static void apif_free_page_lists(void *data) -{ - struct apple_dart_apif *dart = data; - - for (unsigned int i = 0; i < APIF_BATCH_MAX_OPS; i++) - kfree(dart->page_lists[i]); -} - -static int apif_one(struct apple_dart_apif *dart, struct aurora_apif_op *op) -{ - return aurora_apif_submit(dart->apif, op, 1, NULL); -} - -static int apif_prepare_table(struct apple_dart_apif *dart, u64 sid, u64 dva, - u64 level) -{ - u64 base = level ? round_down(dva, 1ULL << (level == 1 ? 36 : 25)) : 0; - u64 owner = 0x4441525400000000ULL | ((u64)dart->dart_id << 16) | - (sid << 8) | level; - struct aurora_apif_op op = { - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_BOOT_TABLE), - .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, sid, dva, - level }, - }; - u64 pa; - int ret; - - if (sid >= dart->num_streams || level > 2) - return -EINVAL; - if (level == 2 && dart->retained[sid] && - !dart->retained[sid]->root_level) { - ret = apif_prepare_table(dart, sid, dva, 1); - if (ret) - return ret; - } - ret = apif_one(dart, &op); - if (ret || op.ret) - return ret; - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_FIND_FRAME), - .arg = { owner, base }, - }; - ret = apif_one(dart, &op); - if (ret || op.ret) - return ret; - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_ALLOC_FRAME), - }; - ret = apif_one(dart, &op); - if (ret) - return ret; - pa = op.ret; - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(0, 1), /* RETYPE */ - .arg = { pa, 11, 24, - (((level << 16) | (sid << 8) | dart->dart_id) << 32) | - 5 }, - }; - ret = apif_one(dart, &op); - if (ret) { - dart->ownership_uncertain = true; - return ret; /* Retain the frame until a fresh boot. */ - } - if (!op.ret) { - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR( - AURORA_APIF_SERVICE, AURORA_APIF_FREE_FRAME), - .arg = { pa }, - }; - apif_one(dart, &op); - return -EIO; - } - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(3, 0), /* DART MAP_TABLE */ - .arg = { dart->dart_id, sid, dva, level, pa }, - }; - ret = apif_one(dart, &op); - if (ret) { - dart->ownership_uncertain = true; - return ret; - } - /* Publish the opaque ownership key only after the table is attached. */ - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_TAG_FRAME), - .arg = { pa, owner, base }, - }; - ret = apif_one(dart, &op); - if (ret) - dart->ownership_uncertain = true; - return ret; -} - -static int apif_boot_query(struct apple_dart_apif *dart, - struct apif_op *request) -{ - struct apif_retained_sid *r = request->arg[0] < dart->num_streams ? - dart->retained[request->arg[0]] : NULL; - - if (dart->boot_mappings && r) { - if (request->op == APIF_DART_RETAINED_INFO) { - request->arg[1] = r->root_level; - request->arg[2] = r->count; - request->arg[3] = dart->dma_min; - request->arg[4] = dart->dma_max - dart->dma_min + 1; - request->arg[5] = 1; - } else { - const struct apif_retained_run *run; - - if (request->arg[1] >= r->count) - return -EINVAL; - run = &r->runs[request->arg[1]]; - request->arg[2] = run->dva; - request->arg[3] = run->ipa; - request->arg[4] = run->size; - request->arg[5] = 0; - } - return 0; - } - - struct aurora_apif_boot_iommu *info = dart->boot_info; - struct aurora_apif_op op = { - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_BOOT_IOMMU), - .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, request->arg[0], - request->op == APIF_DART_RETAINED_INFO ? - U64_MAX : - request->arg[1], - virt_to_phys(info), sizeof(*info) }, - }; - int ret = apif_one(dart, &op); - - if (ret) - return ret; - if (request->op == APIF_DART_RETAINED_INFO) { - request->arg[1] = info->level; - request->arg[2] = info->count; - request->arg[3] = info->base; - request->arg[4] = info->size; - request->arg[5] = 1; - } else { - request->arg[2] = info->base; - request->arg[3] = info->ipa; - request->arg[4] = info->size; - request->arg[5] = info->attrs; - } - return 0; -} - -/* Build native calls in Linux, including address lists and permission flags. - * Address translations form a separate batch, so failure cannot send a MAP - * with an untranslated pointer. Mapping batches keep the original capacity - * and prefix-completion accounting used by rollback. Caller holds batch_lock. - */ -static int apif_ring(struct apple_dart_apif *dart) -{ - struct aurora_apif_op *native = dart->native_ops; - u32 count = dart->batch->count, n = 0, translated = 0, done = 0; - u32 ends[APIF_BATCH_MAX_OPS]; - int ret = 0; - - dart->batch->done = 0; - if (dart->ownership_uncertain) { - dart->batch->status = -EIO; - return -EIO; - } - if (count > APIF_BATCH_MAX_OPS) - return -EINVAL; - apif_force_iovas(dart, false); - for (u32 i = 0; i < count; i++) { - struct apif_op *op = &dart->batch->op[i]; - - if (op->flags) { - ret = -EINVAL; - goto out; - } - if (op->op == APIF_DART_MAP) { - if (op->arg[4] & - ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | - IOMMU_NOEXEC | IOMMU_MMIO)) { - ret = -EINVAL; - goto out; - } - if (!dart->page_lists[i]) - dart->page_lists[i] = - kmalloc(SZ_16K, GFP_ATOMIC); - if (!dart->page_lists[i]) { - ret = -ENOMEM; - goto out; - } - native[translated++] = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR( - AURORA_APIF_SERVICE, - AURORA_APIF_TRANSLATE), - .arg = { op->arg[2], op->arg[3] }, - }; - native[translated++] = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR( - AURORA_APIF_SERVICE, - AURORA_APIF_TRANSLATE), - .arg = { virt_to_phys(dart->page_lists[i]), - SZ_16K }, - }; - } - } - if (translated) { - ret = aurora_apif_submit(dart->apif, native, translated, NULL); - if (ret) - goto out; - translated = 0; - for (u32 i = 0; i < count; i++) - if (dart->batch->op[i].op == APIF_DART_MAP) { - dart->batch->op[i].arg[5] = - native[translated++].ret; - dart->page_list_pa[i] = - native[translated++].ret; - } - } - for (u32 i = 0; i < count; i++) { - struct apif_op *op = &dart->batch->op[i]; - u64 *a = op->arg; - struct aurora_apif_op call = {}; - - call.arg[0] = dart->dart_id; - call.arg[1] = a[0]; - op->ret = 0; - switch (op->op) { - case APIF_DART_INIT: { - if (dart->boot_mappings) - break; - struct aurora_apif_op query = { - .selector = AURORA_APIF_SELECTOR( - AURORA_APIF_SERVICE, - AURORA_APIF_BOOT_TABLE), - .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, - 0, 0, U64_MAX }, - }; - - ret = apif_one(dart, &query); - if (ret) - goto out; - if (!query.ret) { - call.selector = - AURORA_APIF_SELECTOR(3, 6); /* INIT */ - native[n++] = call; - call.selector = AURORA_APIF_SELECTOR( - 3, 5); /* POWERUP */ - native[n++] = call; - } - break; - } - case APIF_DART_ATTACH: - ret = apif_prepare_table(dart, a[0], a[1], a[2]); - if (ret) - goto out; - break; - case APIF_DART_RETAINED_INFO: - case APIF_DART_RETAINED_RUN: - ret = apif_boot_query(dart, op); - if (ret) - goto out; - break; - case APIF_DART_MAP: { - u64 pa = a[5], size = a[3]; - - if (!size || size > DART_APIF_BLOCK_SIZE || - ((a[1] | pa | size) & (DART_APIF_PAGE_SIZE - 1)) || - (a[1] >> 25) != ((a[1] + size - 1) >> 25)) { - ret = -EINVAL; - goto out; - } - for (u64 p = 0; p < size / DART_APIF_PAGE_SIZE; p++) - dart->page_lists[i][p] = - pa + p * DART_APIF_PAGE_SIZE; - call.selector = AURORA_APIF_SELECTOR(3, 2); /* MAP */ - call.arg[2] = a[1]; - call.arg[3] = dart->page_list_pa[i]; - call.arg[4] = size; - call.arg[5] = (~a[4]) & 7; - native[n++] = call; - break; - } - case APIF_DART_UNMAP: - call.selector = AURORA_APIF_SELECTOR(3, 3); - call.arg[2] = a[1]; - call.arg[3] = a[2]; - call.arg[4] = 2; - native[n++] = call; - break; - case APIF_DART_ENABLE: - case APIF_DART_DISABLE: - if (a[0] >= dart->num_streams) { - ret = -EINVAL; - goto out; - } - /* Live firmware retains stream state across Linux domain - * attachment and teardown, just as for power transitions. - */ - if (dart->retained[a[0]]) - break; - call.selector = AURORA_APIF_SELECTOR( - 3, op->op == APIF_DART_ENABLE ? 8 : 7); - native[n++] = call; - break; - case 7: /* POWERUP */ - case 8: /* POWERDOWN */ - for (u32 sid = 0; sid < dart->num_streams; sid++) - if (dart->retained[sid]) { - if (op->op == 8) { - ret = -EBUSY; - goto out; - } - goto prepared; - } - call.selector = - AURORA_APIF_SELECTOR(3, op->op == 7 ? 5 : 4); - native[n++] = call; - break; - default: - ret = -EINVAL; - goto out; - } -prepared: - ends[i] = n; - } - ret = aurora_apif_submit(dart->apif, native, n, &done); - for (u32 i = 0; i < count && ends[i] <= done; i++) - dart->batch->done = i + 1; -out: - apif_force_iovas(dart, true); - dart->batch->status = ret; - return ret; -} - -/* Setup-only host queries. Firmware-owned mappings are reserved, never replayed - * through MAP, and never inserted in the Linux-owned mappings xarray. - */ -static int apif_load_boot_mappings(struct apple_dart_apif *dart) -{ - struct device_node *np = dart->dev->of_node; - struct apif_retained_sid *r; - u64 cells[32], sid, previous = 0; - int count, ret; - - if (!of_find_property(np, "apple,apif-boot-mappings", NULL)) - return 0; - count = of_property_count_u64_elems(np, "apple,apif-boot-mappings"); - if (count < 4 || count > ARRAY_SIZE(cells) || count % 4 || dart->iova_force) - return -EINVAL; - ret = of_property_read_u64_array(np, "apple,apif-boot-mappings", cells, count); - if (ret) - return ret; - sid = cells[0]; - if (sid >= dart->num_streams || dart->retained[sid]) - return -EINVAL; - r = devm_kzalloc(dart->dev, sizeof(*r), GFP_KERNEL); - if (!r) - return -ENOMEM; - r->root_level = 1; - r->count = count / 4; - r->runs = devm_kcalloc(dart->dev, r->count, sizeof(*r->runs), GFP_KERNEL); - if (!r->runs) - return -ENOMEM; - for (unsigned int i = 0; i < r->count; i++) { - struct apif_retained_run *run = &r->runs[i]; - - run->dva = cells[i * 4 + 1]; - run->ipa = cells[i * 4 + 2]; - run->size = cells[i * 4 + 3]; - if (cells[i * 4] != sid || !run->size || - (run->dva | run->ipa | run->size) & (DART_APIF_PAGE_SIZE - 1) || - run->dva < dart->dma_min || run->dva > dart->dma_max || - run->size - 1 > dart->dma_max - run->dva || - run->size > U64_MAX - run->ipa || (i && run->dva < previous)) - return -EINVAL; - previous = run->dva + run->size; - } - dart->retained[sid] = r; - dart->boot_mappings = true; - dev_info(dart->dev, "retaining %u boot DMA mappings on stream %llu\n", r->count, sid); - return 0; -} - -static int apif_load_retained(struct apple_dart_apif *dart) -{ - struct device_node *np = dart->dev->of_node; - int count = of_property_count_u32_elems(np, "apple,apif-retained-sids"); - int ret; - - if (!of_find_property(np, "apple,apif-retained-sids", NULL)) - return 0; - if (count < 1 || count > dart->num_streams || dart->iova_force) - return -EINVAL; - for (int i = 0; i < count; i++) { - struct apif_retained_sid *r; - u32 sid; - u64 previous = 0; - struct apif_op *op = &dart->batch->op[0]; - - if (of_property_read_u32_index(np, "apple,apif-retained-sids", - i, &sid) || - sid >= dart->num_streams || dart->retained[sid]) - return -EINVAL; - dart->batch->count = 1; - *op = (struct apif_op){ .op = APIF_DART_RETAINED_INFO, - .arg = { sid } }; - ret = apif_ring(dart); - if (ret) - return ret; - if (op->arg[1] > 1 || op->arg[2] > 131072 || op->arg[5] != 1 || - op->arg[3] != dart->dma_min || - op->arg[4] != dart->dma_max - dart->dma_min + 1) - return -EINVAL; - r = devm_kzalloc(dart->dev, sizeof(*r), GFP_KERNEL); - if (!r) - return -ENOMEM; - r->root_level = op->arg[1]; - r->count = op->arg[2]; - r->runs = devm_kcalloc(dart->dev, r->count, sizeof(*r->runs), - GFP_KERNEL); - if (r->count && !r->runs) - return -ENOMEM; - for (u64 j = 0; j < r->count; j++) { - struct apif_retained_run *run = &r->runs[j]; - - *op = (struct apif_op){ .op = APIF_DART_RETAINED_RUN, - .arg = { sid, j } }; - ret = apif_ring(dart); - if (ret) - return ret; - run->dva = op->arg[2]; - run->ipa = op->arg[3]; - run->size = op->arg[4]; - if (!run->size || - (run->dva | run->size) & - (DART_APIF_PAGE_SIZE - 1) || - run->dva < dart->dma_min || - run->dva > dart->dma_max || - run->size - 1 > dart->dma_max - run->dva || - (j && run->dva < previous) || - (run->ipa != U64_MAX && - ((run->ipa & (DART_APIF_PAGE_SIZE - 1)) || - run->size > U64_MAX - run->ipa))) - return -EINVAL; - previous = run->dva + run->size; - } - dart->retained[sid] = r; - } - return 0; -} - -static const struct apif_retained_run * -apif_retained_lookup(const struct apif_retained_sid *r, u64 dva, u64 size) -{ - unsigned int low = 0, high = r->count; - - while (low < high) { - unsigned int mid = low + (high - low) / 2; - const struct apif_retained_run *run = &r->runs[mid]; - - if (run->dva + run->size <= dva) - low = mid + 1; - else - high = mid; - } - return low < r->count && r->runs[low].dva < dva + size ? &r->runs[low] : - NULL; -} - -static bool apif_retained_overlap(struct apple_dart_apif_domain *dom, u64 dva, - u64 size) -{ - unsigned int sid; - - if (!dom->has_retained) - return false; - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { - struct apif_retained_sid *r = - stream->dart->retained[sid]; - - if (r && apif_retained_lookup(r, dva, size)) - return true; - } - } - return false; -} - -static phys_addr_t apif_retained_translate(struct apple_dart_apif_domain *dom, - u64 dva) -{ - unsigned int sid; - u64 result = 0; - - if (!dom->has_retained) - return 0; - /* Firmware may retain different mappings on different display engines. - * Only a unanimous translation is meaningful for their shared domain. - */ - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { - struct apif_retained_sid *r = - stream->dart->retained[sid]; - const struct apif_retained_run *run = - r ? apif_retained_lookup(r, dva, 1) : NULL; - u64 pa; - - if (!run || dva < run->dva || run->ipa == U64_MAX) - return 0; - pa = run->ipa + dva - run->dva; - if (result && result != pa) - return 0; - result = pa; - } - } - return result; -} - -/* Bring the DART up in SPTM exactly once, before any stream is enabled. */ -static int apple_dart_apif_ensure_init(struct apple_dart_apif *dart) -{ - int ret = 0; - - mutex_lock(&dart->init_lock); - if (!dart->inited) { - unsigned long flags; - - spin_lock_irqsave(&dart->batch_lock, flags); - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ .op = APIF_DART_INIT }; - ret = apif_ring(dart); - spin_unlock_irqrestore(&dart->batch_lock, flags); - if (ret) - ret = -EIO; - else - dart->inited = true; - } - mutex_unlock(&dart->init_lock); - return ret; -} - -/* Live mappings are indexed by IOVA page. Normal map/unmap only touches the - * requested range; a complete walk is needed solely when adding a new SID. - * Zero tagged values reserve XArray storage before any hardware mapping. - */ -#define APIF_MAP_PROT_BITS 8 - -static unsigned long apif_mapping_value(phys_addr_t pa, int prot) -{ - return ((pa >> DART_APIF_PAGE_SHIFT) << APIF_MAP_PROT_BITS) | prot; -} - -static int apif_reserve_mappings(struct apple_dart_apif_domain *dom, - unsigned long iova, size_t size, gfp_t gfp) -{ - unsigned long first = iova >> DART_APIF_PAGE_SHIFT; - unsigned long pages = size >> DART_APIF_PAGE_SHIFT; - unsigned long page; - int ret; - - for (page = 0; page < pages; page++) { - ret = xa_insert(&dom->mappings, first + page, xa_mk_value(0), - gfp); - if (ret) - goto undo; - } - return 0; -undo: - while (page) - xa_erase(&dom->mappings, first + --page); - return ret; -} - -static void apif_release_reservations(struct apple_dart_apif_domain *dom, - unsigned long iova, size_t size) -{ - unsigned long first = iova >> DART_APIF_PAGE_SHIFT; - unsigned long pages = size >> DART_APIF_PAGE_SHIFT; - - for (unsigned long page = 0; page < pages; page++) - if (xa_load(&dom->mappings, first + page) == xa_mk_value(0)) - xa_erase(&dom->mappings, first + page); -} - -static void apif_record_mapping(struct apple_dart_apif_domain *dom, u64 iova, - u64 pa, size_t size, int prot) -{ - for (size_t off = 0; off < size; off += DART_APIF_PAGE_SIZE) { - unsigned long index = (iova + off) >> DART_APIF_PAGE_SHIFT; - void *entry = xa_mk_value(apif_mapping_value(pa + off, prot)); - - /* Reserved entries retain their nodes, so replacing a reservation - * requires no allocation, even in this IRQ-disabled section. - */ - WARN_ON(xa_err( - xa_store(&dom->mappings, index, entry, GFP_NOWAIT))); - } -} - -static void apif_forget_mapping(struct apple_dart_apif_domain *dom, u64 iova, - size_t size) -{ - for (size_t off = 0; off < size; off += DART_APIF_PAGE_SIZE) - xa_erase(&dom->mappings, (iova + off) >> DART_APIF_PAGE_SHIFT); -} - -static int apif_prepare_leaf(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, unsigned int sid, - u64 iova) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned long key = ((unsigned long)dart->dart_id << 48) | - ((unsigned long)sid << 40) | - (iova >> DART_APIF_BLOCK_SHIFT); - int ret; - - if (xa_load(&dom->l2_attached, key)) - return 0; - ret = xa_insert(&dom->l2_attached, key, xa_mk_value(0), GFP_ATOMIC); - if (ret) - return ret; - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ - .op = APIF_DART_ATTACH, - .arg = { sid, iova, 2 }, - }; - ret = apif_ring(dart); - if (ret) { - xa_erase(&dom->l2_attached, key); - return ret; - } - WARN_ON(xa_err( - xa_store(&dom->l2_attached, key, xa_mk_value(1), GFP_NOWAIT))); - return 0; -} - -/* Return the next physically contiguous, same-protection run, stopping at a - * leaf-table boundary. Reservations are not yet live and are skipped. - */ -static bool apif_next_mapping(struct apple_dart_apif_domain *dom, - unsigned long *cursor, struct apif_op *op, - unsigned int sid) -{ - unsigned long index = *cursor, value, first; - void *entry; - u64 iova, pa; - size_t size; - int prot; - - for (;;) { - entry = xa_find(&dom->mappings, &index, ULONG_MAX, XA_PRESENT); - if (!entry) - return false; - value = xa_to_value(entry); - if (value) - break; - index++; - } - first = index; - prot = value & ((1UL << APIF_MAP_PROT_BITS) - 1); - pa = (value >> APIF_MAP_PROT_BITS) << DART_APIF_PAGE_SHIFT; - iova = (u64)first << DART_APIF_PAGE_SHIFT; - size = DART_APIF_PAGE_SIZE; - while ((iova + size) & (DART_APIF_BLOCK_SIZE - 1)) { - entry = xa_load(&dom->mappings, index + 1); - if (entry != xa_mk_value(apif_mapping_value(pa + size, prot))) - break; - index++; - size += DART_APIF_PAGE_SIZE; - } - *cursor = index + 1; - *op = (struct apif_op){ - .op = APIF_DART_MAP, - .arg = { sid, iova, pa, size, prot }, - }; - return true; -} - -static int apif_undo_replay(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, unsigned int sid, - u64 mapped_end) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned long cursor = 0; - struct apif_op op; - - while (apif_next_mapping(dom, &cursor, &op, sid) && - op.arg[1] < mapped_end) { - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ - .op = APIF_DART_UNMAP, - .arg = { sid, op.arg[1], - min_t(u64, op.arg[3], - mapped_end - op.arg[1]) }, - }; - if (apif_ring(dart)) { - dev_err(dart->dev, "failed to undo SID %u replay\n", - sid); - return -EIO; - } - } - return 0; -} - -/* New SID is disabled throughout replay. On failure, keep it disabled and - * remove the successful replay prefix. Existing SIDs/maps stay untouched. - */ -static int apif_replay_sid(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, unsigned int sid) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned long cursor = 0; - struct apif_op op; - u64 mapped_end = 0; - u32 n = 0; - int ret; - - while (apif_next_mapping(dom, &cursor, &op, sid)) { - if (dart->retained[sid] && - apif_retained_lookup(dart->retained[sid], op.arg[1], - op.arg[3])) - return -EBUSY; - ret = apif_prepare_leaf(dom, stream, sid, op.arg[1]); - if (ret) - return ret; - } - cursor = 0; - for (;;) { - bool have = apif_next_mapping(dom, &cursor, &op, sid); - - if (have) - dart->batch->op[n++] = op; - if (n == APIF_BATCH_MAX_OPS || (!have && n)) { - dart->batch->count = n; - ret = apif_ring(dart); - for (u32 i = 0; i < min(dart->batch->done, n); i++) - mapped_end = dart->batch->op[i].arg[1] + - dart->batch->op[i].arg[3]; - if (ret) - goto undo; - n = 0; - } - if (!have) - return 0; - } -undo: - apif_undo_replay(dom, stream, sid, mapped_end); - return ret; -} - -static int apple_dart_apif_attach_dev(struct iommu_domain *domain, - struct device *dev, - struct iommu_domain *old) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - struct apple_dart_apif_domain *dom = to_apif_domain(domain); - struct apple_dart_apif_domain *prior = NULL; - unsigned long added[APIF_MAX_DARTS] - [BITS_TO_LONGS(DART_APIF_MAX_STREAMS)] = {}; - unsigned long moved[APIF_MAX_DARTS] - [BITS_TO_LONGS(DART_APIF_MAX_STREAMS)] = {}; - unsigned long flags; - unsigned int sid; - int ret; - - if (!master || !master->num_darts) - return -ENODEV; - for (unsigned int i = 0; i < master->num_darts; i++) { - ret = apple_dart_apif_ensure_init(master->streams[i].dart); - if (ret) - return ret; - } - if (old && old != domain && old->owner == &apple_dart_apif_ops && - old->ops == apple_dart_apif_ops.default_domain_ops) - prior = to_apif_domain(old); - if (prior && (unsigned long)prior < (unsigned long)dom) { - mutex_lock(&prior->lock); - mutex_lock_nested(&dom->lock, SINGLE_DEPTH_NESTING); - } else { - mutex_lock(&dom->lock); - if (prior) - mutex_lock_nested(&prior->lock, SINGLE_DEPTH_NESTING); - } - ret = apif_domain_configure(dom, master); - if (ret) - goto unlock_mutex; - - local_irq_save(flags); - if (prior && (unsigned long)prior < (unsigned long)dom) { - write_lock(&prior->streams_lock); - write_lock_nested(&dom->streams_lock, SINGLE_DEPTH_NESTING); - } else { - write_lock(&dom->streams_lock); - if (prior) - write_lock_nested(&prior->streams_lock, - SINGLE_DEPTH_NESTING); - } - for (unsigned int i = 0; i < master->num_darts && !ret; i++) { - struct apple_dart_apif *dart = master->streams[i].dart; - struct apif_stream_map *stream = apif_find_stream(dom, dart); - struct apif_stream_map *before = apif_find_stream(prior, dart); - - spin_lock(&dart->batch_lock); - for_each_set_bit(sid, master->streams[i].sids, - dart->num_streams) { - if (test_bit(sid, stream->sids)) - continue; - if (dart->retained[sid] && before && - test_bit(sid, before->sids)) { - set_bit(sid, moved[i]); - ret = apif_undo_replay(prior, before, sid, - U64_MAX); - if (ret) - break; - } - dart->batch->count = dart->retained[sid] ? 1 : 2; - dart->batch->op[0] = (struct apif_op){ - .op = dart->retained[sid] ? - APIF_DART_RETAINED_INFO : - APIF_DART_DISABLE, - .arg = { sid }, - }; - if (!dart->retained[sid]) - dart->batch->op[1] = (struct apif_op){ - .op = APIF_DART_ATTACH, - .arg = { sid, dart->dma_min, 1 }, - }; - ret = apif_ring(dart); - if (ret) - break; - ret = apif_replay_sid(dom, stream, sid); - if (ret) - break; - /* Include this SID in unwind even when ENABLE's completion fails. */ - set_bit(sid, added[i]); - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ - .op = dart->retained[sid] ? - APIF_DART_RETAINED_INFO : - APIF_DART_ENABLE, - .arg = { sid }, - }; - ret = apif_ring(dart); - if (ret) - break; - dom->has_retained |= !!dart->retained[sid]; - set_bit(sid, stream->sids); - } - spin_unlock(&dart->batch_lock); - } - - /* Attach is one transaction across all of the master's DARTs. */ - for (unsigned int i = 0; i < master->num_darts; i++) { - struct apple_dart_apif *dart = master->streams[i].dart; - struct apif_stream_map *stream = apif_find_stream(dom, dart); - struct apif_stream_map *before = apif_find_stream(prior, dart); - - spin_lock(&dart->batch_lock); - if (ret) { - for_each_set_bit(sid, added[i], dart->num_streams) { - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ - .op = dart->retained[sid] ? - APIF_DART_RETAINED_INFO : - APIF_DART_DISABLE, - .arg = { sid }, - }; - if (apif_ring(dart)) - dev_err(dart->dev, - "failed to disable SID %u during attach unwind\n", - sid); - apif_undo_replay(dom, stream, sid, U64_MAX); - clear_bit(sid, stream->sids); - } - } - for_each_set_bit(sid, moved[i], dart->num_streams) { - if (ret) { - if (apif_replay_sid(prior, before, sid)) - dev_err(dart->dev, - "failed to restore previous domain SID %u\n", - sid); - } else { - clear_bit(sid, before->sids); - } - } - spin_unlock(&dart->batch_lock); - } - if (prior) - write_unlock(&prior->streams_lock); - write_unlock(&dom->streams_lock); - local_irq_restore(flags); -unlock_mutex: - if (prior) - mutex_unlock(&prior->lock); - mutex_unlock(&dom->lock); - return ret; -} - -struct apif_map_progress { - size_t mapped; - unsigned int streams; - struct apif_op partial; -}; - -/* Track only fully installed stream cohorts. The batch may split one cohort - * without reducing batching; keep its progress across doorbells. - */ -static int apif_flush_maps(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, - struct apif_map_progress *progress) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned int total_streams = - bitmap_weight(stream->sids, dart->num_streams); - int ret = apif_ring(dart); - - for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { - struct apif_op *op = &dart->batch->op[i]; - - progress->partial = *op; - if (++progress->streams == total_streams) { - progress->mapped += op->arg[3]; - progress->streams = 0; - } - } - if (ret && progress->streams) { - unsigned int sid; - u32 n = 0; - - for_each_set_bit(sid, stream->sids, dart->num_streams) { - if (n == progress->streams) - break; - dart->batch->op[n++] = (struct apif_op){ - .op = APIF_DART_UNMAP, - .arg = { sid, progress->partial.arg[1], - progress->partial.arg[3] }, - }; - } - dart->batch->count = n; - if (apif_ring(dart)) - dev_err(dart->dev, "failed to undo partial DART map\n"); - } - return ret; -} - -static int apif_map_streams(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int prot, gfp_t gfp, size_t *mapped) -{ - struct apple_dart_apif *dart = stream->dart; - struct apif_map_progress progress = {}; - size_t total; - unsigned long flags; - u32 n = 0; - int ret; - - *mapped = 0; - if (!dart || bitmap_empty(stream->sids, dart->num_streams)) - return -ENODEV; - if (pgsize != DART_APIF_PAGE_SIZE || !pgcount || - check_mul_overflow(pgsize, pgcount, &total) || - (iova | paddr) & (DART_APIF_PAGE_SIZE - 1) || - prot & ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | IOMMU_NOEXEC | - IOMMU_MMIO) || - !paddr || iova < dart->dma_min || iova > dart->dma_max || - total - 1 > dart->dma_max - iova || - iova + total > DART_APIF_NATIVE_APERTURE_END) - return -EINVAL; - spin_lock_irqsave(&dart->batch_lock, flags); - /* Install all leaf tables before accumulating the mapping batch. */ - for (size_t pos = 0; pos < total;) { - u64 cur = iova + pos; - size_t chunk = - min_t(size_t, total - pos, - DART_APIF_BLOCK_SIZE - - (cur & (DART_APIF_BLOCK_SIZE - 1))); - unsigned int sid; - - for_each_set_bit(sid, stream->sids, dart->num_streams) { - ret = apif_prepare_leaf(dom, stream, sid, cur); - if (ret) - goto out; - } - pos += chunk; - } - for (size_t pos = 0; pos < total;) { - u64 cur = iova + pos; - size_t chunk = - min_t(size_t, total - pos, - DART_APIF_BLOCK_SIZE - - (cur & (DART_APIF_BLOCK_SIZE - 1))); - unsigned int sid; - - for_each_set_bit(sid, stream->sids, dart->num_streams) { - dart->batch->op[n++] = (struct apif_op){ - .op = APIF_DART_MAP, - .arg = { sid, cur, paddr + pos, chunk, prot }, - }; - if (n == APIF_BATCH_MAX_OPS) { - dart->batch->count = n; - ret = apif_flush_maps(dom, stream, &progress); - if (ret) - goto out; - n = 0; - } - } - pos += chunk; - } - if (n) { - dart->batch->count = n; - ret = apif_flush_maps(dom, stream, &progress); - } -out: - spin_unlock_irqrestore(&dart->batch_lock, flags); - *mapped = progress.mapped; - return ret ? -EIO : 0; -} - -static int apif_restore_partial_unmap(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, - const struct apif_op *partial, - unsigned int streams) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned long cursor = partial->arg[1] >> DART_APIF_PAGE_SHIFT; - u64 end = partial->arg[1] + partial->arg[2]; - struct apif_op op; - u32 n = 0; - - while (apif_next_mapping(dom, &cursor, &op, 0) && op.arg[1] < end) { - unsigned int sid, count = 0; - - op.arg[3] = min_t(u64, op.arg[3], end - op.arg[1]); - for_each_set_bit(sid, stream->sids, dart->num_streams) { - if (count++ == streams) - break; - op.arg[0] = sid; - dart->batch->op[n++] = op; - if (n == APIF_BATCH_MAX_OPS) { - dart->batch->count = n; - if (apif_ring(dart)) - return -EIO; - n = 0; - } - } - } - if (!n) - return 0; - dart->batch->count = n; - return apif_ring(dart); -} - -static int apif_flush_unmaps(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, - struct apif_map_progress *progress) -{ - struct apple_dart_apif *dart = stream->dart; - unsigned int total_streams = - bitmap_weight(stream->sids, dart->num_streams); - int ret = apif_ring(dart); - - for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { - struct apif_op *op = &dart->batch->op[i]; - - progress->partial = *op; - if (++progress->streams == total_streams) { - progress->mapped += op->arg[2]; - progress->streams = 0; - } - } - if (ret && progress->streams && - apif_restore_partial_unmap(dom, stream, &progress->partial, - progress->streams)) - dev_err(dart->dev, "failed to restore partial DART unmap\n"); - return ret; -} - -static size_t apif_unmap_streams(struct apple_dart_apif_domain *dom, - struct apif_stream_map *stream, - unsigned long iova, size_t pgsize, - size_t pgcount, - struct iommu_iotlb_gather *gather) -{ - struct apple_dart_apif *dart = stream->dart; - struct apif_map_progress progress = {}; - size_t total; - unsigned long flags; - u32 n = 0; - - if (!dart || bitmap_empty(stream->sids, dart->num_streams) || - pgsize != DART_APIF_PAGE_SIZE || !pgcount || - check_mul_overflow(pgsize, pgcount, &total) || - (iova & (DART_APIF_PAGE_SIZE - 1)) || iova < dart->dma_min || - iova > dart->dma_max || total - 1 > dart->dma_max - iova) - return 0; - spin_lock_irqsave(&dart->batch_lock, flags); - for (size_t pos = 0; pos < total;) { - u64 cur = iova + pos; - size_t chunk = - min_t(size_t, total - pos, - DART_APIF_BLOCK_SIZE - - (cur & (DART_APIF_BLOCK_SIZE - 1))); - unsigned int sid; - - for_each_set_bit(sid, stream->sids, dart->num_streams) { - dart->batch->op[n++] = (struct apif_op){ - .op = APIF_DART_UNMAP, - .arg = { sid, cur, chunk }, - }; - if (n == APIF_BATCH_MAX_OPS) { - dart->batch->count = n; - if (apif_flush_unmaps(dom, stream, &progress)) - goto out; - n = 0; - } - } - pos += chunk; - } - if (n) { - dart->batch->count = n; - apif_flush_unmaps(dom, stream, &progress); - } -out: - spin_unlock_irqrestore(&dart->batch_lock, flags); - return progress.mapped; -} - -/* A buffer has one IOVA and one software mapping, regardless of how many - * display engines can scan it out. Forward whole batches to each DART; do - * not split them into per-page or per-stream transactions. - */ -static int apple_dart_apif_map_pages(struct iommu_domain *domain, - unsigned long iova, phys_addr_t paddr, - size_t pgsize, size_t pgcount, int prot, - gfp_t gfp, size_t *mapped) -{ - struct apple_dart_apif_domain *dom = to_apif_domain(domain); - size_t done[APIF_MAX_DARTS] = {}, total; - unsigned long flags; - unsigned int active = 0; - int ret; - - *mapped = 0; - if (!dom->num_darts) - return -ENODEV; - if (pgsize != DART_APIF_PAGE_SIZE || !pgcount || - check_mul_overflow(pgsize, pgcount, &total) || - (iova | paddr) & (DART_APIF_PAGE_SIZE - 1) || !paddr || - total - 1 > U64_MAX - paddr || - prot & ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | IOMMU_NOEXEC | - IOMMU_MMIO) || - iova < domain->geometry.aperture_start || - iova > domain->geometry.aperture_end || - total - 1 > domain->geometry.aperture_end - iova || - iova + total > DART_APIF_NATIVE_APERTURE_END) - return -EINVAL; - ret = apif_reserve_mappings(dom, iova, total, gfp); - if (ret) - return ret; - read_lock_irqsave(&dom->streams_lock, flags); - if (apif_retained_overlap(dom, iova, total)) { - ret = -EBUSY; - goto out; - } - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - - if (bitmap_empty(stream->sids, stream->dart->num_streams)) - continue; - active++; - ret = apif_map_streams(dom, stream, iova, paddr, pgsize, - pgcount, prot, gfp, &done[i]); - if (ret) - break; - } - if (!active) - ret = -ENODEV; - if (!ret) { - apif_record_mapping(dom, iova, paddr, total, prot); - *mapped = total; - } else { - /* A later DART failed. Undo the successful prefixes on every - * engine before letting the DMA layer reuse this IOVA. - */ - for (unsigned int i = 0; i < dom->num_darts; i++) - if (done[i] && - apif_unmap_streams(dom, &dom->streams[i], iova, - pgsize, done[i] / pgsize, - NULL) != done[i]) - dev_err(dom->streams[i].dart->dev, - "failed to unwind multi-DART mapping\n"); - } -out: - read_unlock_irqrestore(&dom->streams_lock, flags); - if (ret) - apif_release_reservations(dom, iova, total); - return ret; -} - -static size_t apple_dart_apif_unmap_pages(struct iommu_domain *domain, - unsigned long iova, size_t pgsize, - size_t pgcount, - struct iommu_iotlb_gather *gather) -{ - struct apple_dart_apif_domain *dom = to_apif_domain(domain); - size_t done[APIF_MAX_DARTS] = {}, total, common; - unsigned long flags; - unsigned int active = 0; - - if (!dom->num_darts || pgsize != DART_APIF_PAGE_SIZE || !pgcount || - check_mul_overflow(pgsize, pgcount, &total) || - (iova & (DART_APIF_PAGE_SIZE - 1)) || - iova < domain->geometry.aperture_start || - iova > domain->geometry.aperture_end || - total - 1 > domain->geometry.aperture_end - iova) - return 0; - read_lock_irqsave(&dom->streams_lock, flags); - if (apif_retained_overlap(dom, iova, total)) { - common = 0; - goto out; - } - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - - if (bitmap_empty(stream->sids, stream->dart->num_streams)) - continue; - active++; - done[i] = apif_unmap_streams(dom, stream, iova, pgsize, pgcount, - gather); - if (done[i] != total) - break; - } - common = active ? total : 0; - for (unsigned int i = 0; i < dom->num_darts; i++) - if (!bitmap_empty(dom->streams[i].sids, - dom->streams[i].dart->num_streams)) - common = min(common, done[i]); - /* Preserve the canonical mapping until all engines have agreed. On - * failure restore the suffix removed from earlier engines, retaining - * their existing leaf tables and full-sized transport batches. - */ - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - struct apple_dart_apif *dart = stream->dart; - struct apif_op suffix = { - .arg = { 0, iova + common, done[i] - common }, - }; - - if (done[i] <= common) - continue; - spin_lock(&dart->batch_lock); - if (apif_restore_partial_unmap( - dom, stream, &suffix, - bitmap_weight(stream->sids, dart->num_streams))) - dev_err(dart->dev, - "failed to restore multi-DART unmap suffix\n"); - spin_unlock(&dart->batch_lock); - } - apif_forget_mapping(dom, iova, common); -out: - read_unlock_irqrestore(&dom->streams_lock, flags); - return common; -} - -static void apple_dart_apif_iotlb_sync(struct iommu_domain *domain, - struct iommu_iotlb_gather *gather) -{ - /* apif invalidates inside every MAP/UNMAP it forwards to SPTM, so a - * separate flush is a no-op. - */ -} - -static phys_addr_t apple_dart_apif_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova) -{ - struct apple_dart_apif_domain *dom = to_apif_domain(domain); - void *entry = xa_load(&dom->mappings, iova >> DART_APIF_PAGE_SHIFT); - unsigned long value = entry ? xa_to_value(entry) : 0; - - if (!value) - return apif_retained_translate(dom, iova); - return ((value >> APIF_MAP_PROT_BITS) << DART_APIF_PAGE_SHIFT) | - (iova & (DART_APIF_PAGE_SIZE - 1)); -} - -static void apple_dart_apif_domain_free(struct iommu_domain *domain) -{ - struct apple_dart_apif_domain *dom = to_apif_domain(domain); - - if (dom->has_retained) { - for (unsigned int i = 0; i < dom->num_darts; i++) { - struct apif_stream_map *stream = &dom->streams[i]; - unsigned int sid; - unsigned long flags; - - spin_lock_irqsave(&stream->dart->batch_lock, flags); - for_each_set_bit(sid, stream->sids, - stream->dart->num_streams) - apif_undo_replay(dom, stream, sid, U64_MAX); - spin_unlock_irqrestore(&stream->dart->batch_lock, - flags); - } - } - xa_destroy(&dom->l2_attached); - xa_destroy(&dom->mappings); - kfree(dom); -} - -static struct iommu_domain * -apple_dart_apif_domain_alloc_paging(struct device *dev) -{ - struct apple_dart_apif_domain *dom; - int ret; - - dom = kzalloc_obj(*dom); - if (!dom) - return NULL; - mutex_init(&dom->lock); - rwlock_init(&dom->streams_lock); - xa_init(&dom->l2_attached); - xa_init(&dom->mappings); - dom->domain.pgsize_bitmap = DART_APIF_PAGE_SIZE; - dom->domain.geometry.aperture_end = DART_APIF_APERTURE_END; - dom->domain.geometry.force_aperture = true; - if (dev && dev_iommu_priv_get(dev)) { - ret = apif_domain_configure(dom, dev_iommu_priv_get(dev)); - if (ret) { - kfree(dom); - return ERR_PTR(ret); - } - } - return &dom->domain; -} - -/* - * The core attaches this before release_device(), so a released device no - * longer uses the domain that is about to be freed. Its SIDs leave that - * domain disabled, with the domain's mappings undone, as an attach unwind - * does. Without it the SIDs stayed enabled on the freed domain's tables, and - * the next attach of the same SIDs met that stale state. Retained SIDs keep - * their current handling. - */ -static int apple_dart_apif_release_attach(struct iommu_domain *domain, - struct device *dev, - struct iommu_domain *old) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - struct apple_dart_apif_domain *prior; - unsigned long flags; - unsigned int sid; - - if (!master || !old || old->owner != &apple_dart_apif_ops || - old->ops != apple_dart_apif_ops.default_domain_ops) - return 0; - prior = to_apif_domain(old); - mutex_lock(&prior->lock); - write_lock_irqsave(&prior->streams_lock, flags); - for (unsigned int i = 0; i < master->num_darts; i++) { - struct apple_dart_apif *dart = master->streams[i].dart; - struct apif_stream_map *before = apif_find_stream(prior, dart); - - if (!before) - continue; - spin_lock(&dart->batch_lock); - for_each_set_bit(sid, master->streams[i].sids, - dart->num_streams) { - if (dart->retained[sid] || !test_bit(sid, before->sids)) - continue; - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ - .op = APIF_DART_DISABLE, - .arg = { sid }, - }; - if (apif_ring(dart)) - dev_err(dart->dev, - "failed to disable SID %u at release\n", - sid); - apif_undo_replay(prior, before, sid, U64_MAX); - clear_bit(sid, before->sids); - } - spin_unlock(&dart->batch_lock); - } - write_unlock_irqrestore(&prior->streams_lock, flags); - mutex_unlock(&prior->lock); - return 0; -} - -static struct iommu_domain apple_dart_apif_release_domain = { - .type = IOMMU_DOMAIN_BLOCKED, - .ops = &(const struct iommu_domain_ops){ - .attach_dev = apple_dart_apif_release_attach, - }, -}; - -static struct iommu_device *apple_dart_apif_probe_device(struct device *dev) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - - if (!dev_iommu_fwspec_get(dev) || !master || !master->num_darts) - return ERR_PTR(-ENODEV); - for (unsigned int i = 0; i < master->num_darts; i++) - device_link_add(dev, master->streams[i].dart->dev, - DL_FLAG_PM_RUNTIME | - DL_FLAG_AUTOREMOVE_SUPPLIER); - return &master->streams[0].dart->iommu; -} - -static void apple_dart_apif_release_device(struct device *dev) -{ - kfree(dev_iommu_priv_get(dev)); -} - -static int apple_dart_apif_of_xlate(struct device *dev, - const struct of_phandle_args *args) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - struct platform_device *iommu_pdev; - struct apple_dart_apif *dart; - u32 sid; - - if (args->args_count != 1) - return -EINVAL; - sid = args->args[0]; - if (sid >= DART_APIF_MAX_STREAMS) - return -EINVAL; - - iommu_pdev = of_find_device_by_node(args->np); - if (!iommu_pdev) - return -ENODEV; - dart = platform_get_drvdata(iommu_pdev); - put_device(&iommu_pdev->dev); - if (!dart) - return -ENODEV; - - if (sid >= dart->num_streams) - return -EINVAL; - if (!master) { - master = kzalloc_obj(*master); - if (!master) - return -ENOMEM; - dev_iommu_priv_set(dev, master); - } - for (unsigned int i = 0; i < master->num_darts; i++) { - if (master->streams[i].dart == dart) { - set_bit(sid, master->streams[i].sids); - return 0; - } - } - if (master->num_darts == APIF_MAX_DARTS) - return -EINVAL; - master->streams[master->num_darts].dart = dart; - set_bit(sid, master->streams[master->num_darts].sids); - master->num_darts++; - return 0; -} - -/* Devices sharing a stream cannot have independent IOVA allocators. PCI - * multifunction/topology grouping is also required even with distinct SIDs. - * Mirrors the native Apple DART lifetime model: the group owns its SID union, - * and its release callback clears the reverse lookup before freeing it. - */ -static DEFINE_MUTEX(apple_dart_apif_groups_lock); - -static void apple_dart_apif_release_group(void *data) -{ - struct apple_dart_apif_master *group_master = data; - unsigned int sid; - - mutex_lock(&apple_dart_apif_groups_lock); - for (unsigned int i = 0; i < group_master->num_darts; i++) { - struct apif_stream_map *stream = &group_master->streams[i]; - - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) - stream->dart->sid2group[sid] = NULL; - } - mutex_unlock(&apple_dart_apif_groups_lock); - kfree(group_master); -} - -static int apif_merge_master(struct apple_dart_apif_master *dst, - const struct apple_dart_apif_master *src) -{ - struct apple_dart_apif_master merged = *dst; - - for (unsigned int i = 0; i < src->num_darts; i++) { - const struct apif_stream_map *from = &src->streams[i]; - unsigned int j; - - for (j = 0; j < merged.num_darts; j++) - if (merged.streams[j].dart == from->dart) - break; - if (j == merged.num_darts) { - if (j == APIF_MAX_DARTS) - return -EINVAL; - merged.streams[j] = *from; - merged.num_darts++; - } else { - bitmap_or(merged.streams[j].sids, - merged.streams[j].sids, from->sids, - from->dart->num_streams); - } - } - *dst = merged; - return 0; -} - -static struct iommu_group *apple_dart_apif_device_group(struct device *dev) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - struct apple_dart_apif_master *group_master; - struct iommu_group *group = NULL, *put_group = NULL; - struct iommu_group *result = ERR_PTR(-EINVAL); - unsigned int sid; - - if (!master || !master->num_darts) - return result; - mutex_lock(&apple_dart_apif_groups_lock); - for (unsigned int i = 0; i < master->num_darts; i++) { - struct apif_stream_map *stream = &master->streams[i]; - - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { - struct iommu_group *existing = - stream->dart->sid2group[sid]; - - if (!existing) - continue; - if (group && existing != group) - goto out; - group = existing; - } - } - if (group) { - group = iommu_group_ref_get(group); - } else { -#ifdef CONFIG_PCI - if (dev_is_pci(dev)) - group = pci_device_group(dev); - else -#endif - group = generic_device_group(dev); - } - if (IS_ERR(group)) { - result = group; - goto out; - } - if (!group) { - result = ERR_PTR(-ENOMEM); - goto out; - } - group_master = iommu_group_get_iommudata(group); - if (!group_master) { - group_master = kmemdup(master, sizeof(*master), GFP_KERNEL); - if (!group_master) { - result = ERR_PTR(-ENOMEM); - put_group = group; - goto out; - } - iommu_group_set_iommudata(group, group_master, - apple_dart_apif_release_group); - } else if (apif_merge_master(group_master, master)) { - put_group = group; - goto out; - } - for (unsigned int i = 0; i < master->num_darts; i++) { - struct apif_stream_map *stream = &master->streams[i]; - - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) - stream->dart->sid2group[sid] = group; - } - result = group; -out: - mutex_unlock(&apple_dart_apif_groups_lock); - /* Dropping the last reference can invoke release_group: do it unlocked. */ - if (put_group) - iommu_group_put(put_group); - return result; -} - -static bool apple_dart_apif_capable(struct device *dev, enum iommu_cap cap) -{ - return cap == IOMMU_CAP_CACHE_COHERENCY; -} - -/* PCIe captures the MSI doorbell before DART translation. Advertise the - * identity MSI page to the DMA layer so it never tries to map this non-RAM - * address through the SPTM transport. Other devices retain normal reservations. - * This follows apple-dart.c's hardware MSI reservation (Asahi contributors). - */ -static void apple_dart_apif_get_resv_regions(struct device *dev, - struct list_head *head) -{ - struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); - unsigned int sid; - - for (unsigned int i = 0; master && i < master->num_darts; i++) { - struct apif_stream_map *stream = &master->streams[i]; - - /* Reserve the exclusive-end granule when required by the aperture. - * Narrower native apertures declare that restriction in DT. - */ - if (stream->dart->dma_max == DART_APIF_NATIVE_APERTURE_END || - stream->dart->reserve_last_page) { - struct iommu_resv_region *region = - iommu_alloc_resv_region( - stream->dart->dma_max + 1 - - DART_APIF_PAGE_SIZE, - DART_APIF_PAGE_SIZE, 0, - IOMMU_RESV_RESERVED, GFP_KERNEL); - - if (region) - list_add_tail(®ion->list, head); - } - for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { - struct apif_retained_sid *r = - stream->dart->retained[sid]; - if (!r) - continue; - for (unsigned int i = 0; i < r->count; i++) { - struct iommu_resv_region *region = - iommu_alloc_resv_region( - r->runs[i].dva, r->runs[i].size, - 0, IOMMU_RESV_RESERVED, - GFP_KERNEL); - if (region) - list_add_tail(®ion->list, head); - } - } - } - if (dev_is_pci(dev)) { - struct pci_host_bridge *bridge = - pci_find_host_bridge(to_pci_dev(dev)->bus); - struct device_node *np = bridge->dev.parent->of_node; - struct iommu_resv_region *region; - u64 address; - - if (of_device_is_compatible(np, "apple,apif-pcie") && - !of_property_read_u64(np, "apple,msi-address", &address)) { - region = iommu_alloc_resv_region( - address & PAGE_MASK, PAGE_SIZE, - IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO, - IOMMU_RESV_MSI, GFP_KERNEL); - if (region) - list_add_tail(®ion->list, head); - } - } - iommu_dma_get_resv_regions(dev, head); -} - -static const struct iommu_ops apple_dart_apif_ops = { - .get_resv_regions = apple_dart_apif_get_resv_regions, - .capable = apple_dart_apif_capable, - .domain_alloc_paging = apple_dart_apif_domain_alloc_paging, - .probe_device = apple_dart_apif_probe_device, - .release_device = apple_dart_apif_release_device, - .release_domain = &apple_dart_apif_release_domain, - .device_group = apple_dart_apif_device_group, - .of_xlate = apple_dart_apif_of_xlate, - .owner = THIS_MODULE, - .default_domain_ops = - &(const struct iommu_domain_ops){ - .attach_dev = apple_dart_apif_attach_dev, - .map_pages = apple_dart_apif_map_pages, - .unmap_pages = apple_dart_apif_unmap_pages, - .iotlb_sync = apple_dart_apif_iotlb_sync, - .iova_to_phys = apple_dart_apif_iova_to_phys, - .free = apple_dart_apif_domain_free, - }, -}; - -/* Used only after the PCI host has stopped every endpoint on this DART. - * Native POWERDOWN/POWERUP retains the SPTM-owned tables and live mappings. - */ -static int apif_other_group_device(struct device *member, void *data) -{ - return member == data ? 0 : -EBUSY; -} - -int apple_dart_apif_set_power(struct device *dev, bool on) -{ - struct apple_dart_apif_master *master; - struct apple_dart_apif *dart; - struct iommu_group *group; - unsigned long flags; - int ret; - - if (!dev->iommu || !dev->iommu->iommu_dev || - dev->iommu->iommu_dev->ops != &apple_dart_apif_ops) - return -ENODEV; - master = dev_iommu_priv_get(dev); - if (!master || master->num_darts != 1) - return -ENODEV; - dart = master->streams[0].dart; - group = iommu_group_get(dev); - if (!group) - return -ENODEV; - ret = iommu_group_for_each_dev(group, dev, apif_other_group_device); - if (ret) - goto put_group; - mutex_lock(&apple_dart_apif_groups_lock); - /* Native DISABLE affects the entire unit. Retained firmware streams or - * another Linux group must never be stopped by one endpoint's reset. - */ - for (unsigned int sid = 0; sid < dart->num_streams; sid++) { - if (dart->retained[sid] || - (dart->sid2group[sid] && dart->sid2group[sid] != group)) { - ret = -EBUSY; - goto unlock_groups; - } - } - mutex_lock(&dart->init_lock); - if (!dart->inited || dart->ownership_uncertain) { - ret = -EINVAL; - goto out; - } - spin_lock_irqsave(&dart->batch_lock, flags); - dart->batch->count = 1; - dart->batch->op[0] = (struct apif_op){ .op = on ? 7 : 8 }; - ret = apif_ring(dart); - spin_unlock_irqrestore(&dart->batch_lock, flags); -out: - mutex_unlock(&dart->init_lock); -unlock_groups: - mutex_unlock(&apple_dart_apif_groups_lock); -put_group: - iommu_group_put(group); - return ret; -} -EXPORT_SYMBOL_GPL(apple_dart_apif_set_power); - -static int apple_dart_apif_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct apple_dart_apif *dart; - int ret; - - dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); - if (!dart) - return -ENOMEM; - - dart->dev = dev; - mutex_init(&dart->init_lock); - spin_lock_init(&dart->batch_lock); - - dart->apif = aurora_apif_get(dev); - if (IS_ERR(dart->apif)) - return dev_err_probe(dev, PTR_ERR(dart->apif), - "APIF transport unavailable\n"); - ret = devm_add_action_or_reset(dev, apif_put_transport, dart->apif); - if (ret) - return ret; - dart->native_ops = devm_kcalloc(dev, 2 * APIF_BATCH_MAX_OPS, - sizeof(*dart->native_ops), GFP_KERNEL); - if (!dart->native_ops) - return -ENOMEM; - dart->boot_info = - devm_kzalloc(dev, sizeof(*dart->boot_info), GFP_KERNEL); - if (!dart->boot_info) - return -ENOMEM; - if (of_property_read_u32(dev->of_node, "apple,apif-dart-id", - &dart->dart_id)) - return dev_err_probe(dev, -EINVAL, - "missing apple,apif-dart-id\n"); - if (dart->dart_id > 0xff) - return dev_err_probe(dev, -EINVAL, "dart-id out of range\n"); - - dart->dma_max = DART_APIF_APERTURE_END; - if (of_find_property(dev->of_node, "apple,dma-range", NULL)) { - u64 range[2], end; - - if (of_property_read_u64_array(dev->of_node, "apple,dma-range", - range, 2) || - !range[1] || - (range[0] | range[1]) & (DART_APIF_PAGE_SIZE - 1) || - check_add_overflow(range[0], range[1] - 1, &end) || - end > DART_APIF_NATIVE_APERTURE_END) - return dev_err_probe(dev, -EINVAL, "bad DMA range\n"); - dart->dma_min = range[0]; - dart->dma_max = end; - } - if (of_find_property(dev->of_node, "apple,apif-iova-force", NULL)) { - if (of_property_read_u64(dev->of_node, "apple,apif-iova-force", - &dart->iova_force) || - (dart->iova_force && dart->iova_force != BIT_ULL(40))) - return dev_err_probe(dev, -EINVAL, - "unsupported IOVA force mask\n"); - /* A native DVA aperture already contains its high address bits. - * Hidden force bits are only valid for the low-address device view. - */ - if (dart->iova_force && dart->dma_max >= dart->iova_force) - return dev_err_probe( - dev, -EINVAL, - "IOVA force overlaps DMA aperture\n"); - } - - dart->reserve_last_page = of_property_read_bool( - dev->of_node, "apple,apif-reserve-last-page"); - dart->num_streams = DART_APIF_MAX_STREAMS; - of_property_read_u32(dev->of_node, "apple,apif-num-streams", - &dart->num_streams); - if (!dart->num_streams || dart->num_streams > DART_APIF_MAX_STREAMS) - return dev_err_probe(dev, -EINVAL, "bad stream count\n"); - - dart->batch = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL); - if (!dart->batch) - return -ENOMEM; - ret = apif_load_retained(dart); - if (ret) - return dev_err_probe(dev, ret, - "retained DART discovery failed\n"); - ret = apif_load_boot_mappings(dart); - if (ret) - return dev_err_probe(dev, ret, "invalid boot DMA mappings\n"); - - ret = devm_add_action_or_reset(dev, apif_free_page_lists, dart); - if (ret) - return ret; - platform_set_drvdata(pdev, dart); - - ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, - "apple-dart-apif.%s", dev_name(dev)); - if (ret) - return ret; - - ret = iommu_device_register(&dart->iommu, &apple_dart_apif_ops, dev); - if (ret) - goto err_sysfs; - - return 0; - -err_sysfs: - iommu_device_sysfs_remove(&dart->iommu); - return ret; -} - -static void apple_dart_apif_remove(struct platform_device *pdev) -{ - struct apple_dart_apif *dart = platform_get_drvdata(pdev); - - iommu_device_unregister(&dart->iommu); - iommu_device_sysfs_remove(&dart->iommu); -} - -static const struct of_device_id apple_dart_apif_of_match[] = { - { .compatible = "apple,dart-apif" }, - {}, -}; -MODULE_DEVICE_TABLE(of, apple_dart_apif_of_match); - -static struct platform_driver apple_dart_apif_driver = { - .driver = { - .name = "apple-dart-apif", - .suppress_bind_attrs = true, - .of_match_table = apple_dart_apif_of_match, - }, - .probe = apple_dart_apif_probe, - .remove = apple_dart_apif_remove, -}; -module_platform_driver(apple_dart_apif_driver); - -MODULE_DESCRIPTION("Apple DART IOMMU over the Aurora platform interface"); -MODULE_LICENSE("GPL"); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index dbf3bff72cadd..113e09aea5c47 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -116,16 +116,6 @@ config APPLE_SART Say 'y' here if you have an Apple SoC. -config APPLE_SART_APIF - bool "Apple SART over the Aurora platform interface" - depends on ARM64 && OF && APPLE_SART=y - select AURORA_APIF_MMIO - help - SART provider for a trusted Linux guest under Aura and SPTM. - Consumers retain the existing Apple SART API. Native call selection, - region tracking and completion interpretation stay in this backend. - Existing raw-MMIO SART providers continue to serve other systems. - config APPLE_TUNABLE tristate depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 9ab327c4ec817..5ef85752f160d 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -61,5 +61,3 @@ obj-$(CONFIG_APPLE_CLPC) += clpc.o obj-$(CONFIG_APPLE_REGVIEW) += apple-regview.o apple-regview-y := regview.o - -obj-$(CONFIG_APPLE_SART_APIF) += apple-sart-apif.o diff --git a/drivers/soc/apple/apple-sart-apif.c b/drivers/soc/apple/apple-sart-apif.c deleted file mode 100644 index 3bbdaea969840..0000000000000 --- a/drivers/soc/apple/apple-sart-apif.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* SART policy stays in Linux; native table-5 calls use the common APIF client. */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sart-backend.h" - -#define SART_APIF_MAX_REGIONS 16 - -struct sart_apif { - struct device *dev; - struct aurora_apif *apif; - struct mutex lock; - bool active; - struct { - phys_addr_t address; - size_t size; - } regions[SART_APIF_MAX_REGIONS]; -}; - -static int sart_apif_region(struct apple_sart *sart, phys_addr_t address, - size_t size, bool map) -{ - struct sart_apif *state = apple_sart_backend_data(sart); - struct aurora_apif_op op; - u64 pa; - int i, ret; - - if (!size || ((address | size) & (SZ_16K - 1)) || - address > PHYS_ADDR_MAX - (size - 1)) - return -EINVAL; - mutex_lock(&state->lock); - for (i = 0; i < SART_APIF_MAX_REGIONS; i++) - if (map ? !state->regions[i].size : - state->regions[i].address == address && - state->regions[i].size == size) - break; - if (i == SART_APIF_MAX_REGIONS) { - ret = map ? -EBUSY : -ENOENT; - goto out; - } - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, - AURORA_APIF_TRANSLATE), - .arg = { dma_to_phys(state->dev, address), size }, - }; - ret = aurora_apif_submit(state->apif, &op, 1, NULL); - if (ret) - goto out; - pa = op.ret; - if (!state->active) { - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR( - 5, 0), /* SET_STATE, void result */ - .arg = { 1 }, - }; - ret = aurora_apif_submit(state->apif, &op, 1, NULL); - if (ret) - goto out; - state->active = true; - } - op = (struct aurora_apif_op){ - .selector = AURORA_APIF_SELECTOR(5, map ? 1 : 2), - .arg = { pa, size, map, map }, - }; - ret = aurora_apif_submit(state->apif, &op, 1, NULL); - /* MAP is void; UNMAP returns status. A refused unmap keeps the region - * live so the consumer cannot treat its DMA backing as released. - */ - if (!ret && !map && op.ret) - ret = -EIO; - if (!ret) { - state->regions[i].address = map ? address : 0; - state->regions[i].size = map ? size : 0; - } -out: - mutex_unlock(&state->lock); - return ret; -} - -static int sart_apif_add(struct apple_sart *sart, phys_addr_t address, - size_t size) -{ - return sart_apif_region(sart, address, size, true); -} - -static int sart_apif_remove(struct apple_sart *sart, phys_addr_t address, - size_t size) -{ - return sart_apif_region(sart, address, size, false); -} - -static const struct apple_sart_backend sart_apif_backend = { - .add = sart_apif_add, - .remove = sart_apif_remove, -}; - -static void sart_apif_put(void *data) -{ - aurora_apif_put(data); -} - -static int sart_apif_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct sart_apif *state; - struct apple_sart *sart; - int ret; - - state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - state->dev = dev; - mutex_init(&state->lock); - state->apif = aurora_apif_get(dev); - if (IS_ERR(state->apif)) - return dev_err_probe(dev, PTR_ERR(state->apif), - "APIF transport unavailable\n"); - ret = devm_add_action_or_reset(dev, sart_apif_put, state->apif); - if (ret) - return ret; - sart = apple_sart_create_backend(dev, &sart_apif_backend, state); - if (IS_ERR(sart)) - return PTR_ERR(sart); - platform_set_drvdata(pdev, sart); - return 0; -} - -static void sart_apif_shutdown(struct platform_device *pdev) -{ - struct apple_sart *sart = platform_get_drvdata(pdev); - struct sart_apif *state = apple_sart_backend_data(sart); - - /* Supplier ordering stops consumers before their Linux-owned grants. */ - for (unsigned int i = 0; i < SART_APIF_MAX_REGIONS; i++) - if (state->regions[i].size && - sart_apif_remove(sart, state->regions[i].address, - state->regions[i].size)) - dev_err(state->dev, - "failed to release DMA region at shutdown\n"); -} - -static const struct of_device_id sart_apif_of_match[] = { - { .compatible = "apple,sart-apif" }, - {} -}; -MODULE_DEVICE_TABLE(of, sart_apif_of_match); - -static struct platform_driver sart_apif_driver = { - .driver = { - .name = "apple-sart-apif", - .of_match_table = sart_apif_of_match, - .suppress_bind_attrs = true, - }, - .probe = sart_apif_probe, - .shutdown = sart_apif_shutdown, -}; -module_platform_driver(sart_apif_driver); - -MODULE_DESCRIPTION("Apple SART over the Aurora platform interface"); -MODULE_LICENSE("GPL"); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 6cfc08e15fce2..f4f674f548c5d 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -1688,15 +1688,6 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_j775d_ascwr APPLE_J775D_GFX1_WRAPPER_BASE, APPLE_J775D_GFX1_IOP_VBAR_BASE }, }; -static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6050_apif_ascwrap_v6_roles[] = { - { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, - APPLE_T8140_GFX_WRAPPER_BASE, - APPLE_T8140_GFX_IOP_VBAR_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA }, - { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, - APPLE_T8140_GFX1_WRAPPER_BASE, - APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T6050_ASCWRAP_BASE_DELTA }, -}; - static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, @@ -1729,23 +1720,6 @@ static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { .has_irq_controls = false, }; -/* Retained compatibility profile: unshifted wrapper aliases, native VBAR. */ -static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { - .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, - .is_ascwrap_v6 = true, - .ascwrap_v6_roles = apple_mbox_t6050_apif_ascwrap_v6_roles, - .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t6050_apif_ascwrap_v6_roles), - .control_full = APPLE_ASC_MBOX_CONTROL_FULL, - .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, - .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, - .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, - .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, - .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, - .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, - .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, - .has_irq_controls = false, -}; - /* * T8152 SMC post-office mailbox: counted FIFOs share the status register. * Interrupt masking is provided by the interrupt controller, not the mailbox. @@ -1771,7 +1745,6 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, - { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-j775d-ascwrap-v6", diff --git a/drivers/soc/apple/sart-backend.h b/drivers/soc/apple/sart-backend.h deleted file mode 100644 index 8a130fa48ccba..0000000000000 --- a/drivers/soc/apple/sart-backend.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef APPLE_SART_BACKEND_H -#define APPLE_SART_BACKEND_H - -#include - -/* Alternate providers retain the existing consumer API. */ -struct apple_sart_backend { - int (*add)(struct apple_sart *sart, phys_addr_t address, size_t size); - int (*remove)(struct apple_sart *sart, phys_addr_t address, - size_t size); - bool (*inherited)(struct apple_sart *sart, phys_addr_t address, - size_t size); -}; - -struct apple_sart * -apple_sart_create_backend(struct device *dev, - const struct apple_sart_backend *backend, void *data); -void *apple_sart_backend_data(struct apple_sart *sart); - -#endif diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index 8270a5a731f2c..f9e79924e53a6 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -24,7 +24,6 @@ #include #include #include -#include "sart-backend.h" #define APPLE_SART_MAX_ENTRIES 16 @@ -91,33 +90,8 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; - const struct apple_sart_backend *backend; - void *backend_data; }; -struct apple_sart *apple_sart_create_backend(struct device *dev, - const struct apple_sart_backend *backend, void *data) -{ - struct apple_sart *sart; - - if (!backend || !backend->add || !backend->remove) - return ERR_PTR(-EINVAL); - sart = devm_kzalloc(dev, sizeof(*sart), GFP_KERNEL); - if (!sart) - return ERR_PTR(-ENOMEM); - sart->dev = dev; - sart->backend = backend; - sart->backend_data = data; - return sart; -} -EXPORT_SYMBOL_GPL(apple_sart_create_backend); - -void *apple_sart_backend_data(struct apple_sart *sart) -{ - return sart->backend_data; -} -EXPORT_SYMBOL_GPL(apple_sart_backend_data); - static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size) { @@ -304,8 +278,6 @@ bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, { int i; - if (sart->backend) - return sart->backend->inherited && sart->backend->inherited(sart, paddr, size); if (!size || paddr > PHYS_ADDR_MAX - (size - 1)) return false; for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { @@ -348,8 +320,6 @@ int apple_sart_add_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i, ret; - if (sart->backend) - return sart->backend->add(sart, paddr, size); for (i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { if (test_bit(i, &sart->protected_entries)) continue; @@ -384,8 +354,6 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i; - if (sart->backend) - return sart->backend->remove(sart, paddr, size); dev_dbg(sart->dev, "will remove [paddr: %pa, size: 0x%zx] from allowed regions\n", &paddr, size); diff --git a/include/linux/apple-dart-apif.h b/include/linux/apple-dart-apif.h deleted file mode 100644 index 814b2a91aba94..0000000000000 --- a/include/linux/apple-dart-apif.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LINUX_APPLE_DART_APIF_H -#define _LINUX_APPLE_DART_APIF_H -#include -#include -#include -#if IS_REACHABLE(CONFIG_APPLE_DART_APIF) -/* Caller must quiesce every device on this DART before powering it down. - * Tables and mappings survive the native SPTM power transition. - */ -int apple_dart_apif_set_power(struct device *dev, bool on); -#else -static inline int apple_dart_apif_set_power(struct device *dev, bool on) -{ - return -EOPNOTSUPP; -} -#endif -#endif diff --git a/include/linux/pci-apple-apif.h b/include/linux/pci-apple-apif.h deleted file mode 100644 index d3094cc976a8f..0000000000000 --- a/include/linux/pci-apple-apif.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LINUX_PCI_APPLE_APIF_H -#define _LINUX_PCI_APPLE_APIF_H -#include -#include -struct pci_dev; -#if IS_REACHABLE(CONFIG_PCIE_APPLE_APIF) -/* Preboot N1 transition only, before radio function enumeration or DMA. */ -int apple_apif_pcie_cycle(struct pci_dev *pdev); -#else -static inline int apple_apif_pcie_cycle(struct pci_dev *pdev) -{ - return -EOPNOTSUPP; -} -#endif -#endif diff --git a/include/linux/soc/apple/display-event-debugfs.h b/include/linux/soc/apple/display-event-debugfs.h deleted file mode 100644 index 81322e64bf02f..0000000000000 --- a/include/linux/soc/apple/display-event-debugfs.h +++ /dev/null @@ -1,85 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef _LINUX_APPLE_DISPLAY_EVENT_DEBUGFS_H -#define _LINUX_APPLE_DISPLAY_EVENT_DEBUGFS_H -#include -#include -#include -#include -#include -#include -#include - -static int apple_display_events_show(struct seq_file *s, void *unused) -{ - const struct apple_display_event_snapshot *p = s->private; - u32 i, j; - - seq_printf(s, "apple-display-events version=1 name=%s capacity=512 clock=mono_fast\n", p->name); - seq_printf(s, "snapshot ns=%llu/%llu seq=%llu/%llu count=%u copied=%u gaps=%u complete=%u\n", - p->start_ns, p->end_ns, p->seq_start, p->seq_end, - p->record_count, p->copied, p->gaps, p->complete); - seq_printf(s, "before committed=%llu attempts=%llu busy=%llu sequence=%llu overwritten=%llu\n", - p->start.committed, p->start.attempts, p->start.busy_drops, - p->start.sequence_drops, p->start.overwritten); - seq_printf(s, "after committed=%llu attempts=%llu busy=%llu sequence=%llu overwritten=%llu\n", - p->end.committed, p->end.attempts, p->end.busy_drops, - p->end.sequence_drops, p->end.overwritten); - for (i = 0; i < p->record_count; i++) { - const struct apple_display_event_record *r = &p->records[i]; - - if (!p->valid[i]) { - seq_printf(s, "gap index=%u\n", i); - continue; - } - seq_printf(s, "event seq=%llu ns=%llu type=%u flags=%#x result=%d data=", - r->sequence, r->timestamp_ns, r->type, r->flags, r->result); - for (j = 0; j < 9; j++) - seq_printf(s, "%s%016llx", j ? "," : "", r->data[j]); - seq_putc(s, '\n'); - } - return 0; -} - -static int apple_display_events_open(struct inode *inode, struct file *file) -{ - struct apple_display_event_snapshot *snapshot; - struct dentry *dentry = file->f_path.dentry; - int ret; - - /* No owner pointer is read until debugfs removal protection is held. */ - snapshot = kvzalloc(sizeof(*snapshot), GFP_KERNEL); - if (!snapshot) - return -ENOMEM; - ret = debugfs_file_get(dentry); - if (ret) - goto free; - ret = apple_display_event_snapshot_copy(inode->i_private, snapshot); - debugfs_file_put(dentry); - if (ret) - goto free; - /* Only standalone bytes remain: formatting may outlive the device. */ - ret = single_open(file, apple_display_events_show, snapshot); - if (ret) - goto free; - return 0; -free: - kvfree(snapshot); - return ret; -} - -static int apple_display_events_release(struct inode *inode, struct file *file) -{ - struct seq_file *s = file->private_data; - - kvfree(s->private); - return single_release(inode, file); -} - -static const struct file_operations apple_display_events_fops = { - .owner = THIS_MODULE, - .open = apple_display_events_open, - .read = seq_read, - .llseek = seq_lseek, - .release = apple_display_events_release, -}; -#endif diff --git a/include/linux/soc/apple/display-event-log.h b/include/linux/soc/apple/display-event-log.h deleted file mode 100644 index 9d7cdee5c7551..0000000000000 --- a/include/linux/soc/apple/display-event-log.h +++ /dev/null @@ -1,213 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LINUX_SOC_APPLE_DISPLAY_EVENT_LOG_H -#define _LINUX_SOC_APPLE_DISPLAY_EVENT_LOG_H - -#include -#include -#include -#include -#include -#include -#include - -#define APPLE_DISPLAY_EVENT_CAPACITY 512U -#define APPLE_DISPLAY_EVENT_CHUNK 8U -#define APPLE_DISPLAY_EVENT_SCHEMA 1U - -struct apple_display_event_record { - u64 timestamp_ns; - u64 sequence; - u16 type; - u16 flags; - s32 result; - u64 data[9]; -}; -static_assert(sizeof(struct apple_display_event_record) == 96); - -/* Atomics count attempted calls, including callers which cannot take lock. */ -struct apple_display_event_log { - raw_spinlock_t lock; - char name[64]; - atomic64_t attempts; - atomic64_t busy_drops; - atomic64_t sequence_drops; - u64 committed; - struct apple_display_event_record records[APPLE_DISPLAY_EVENT_CAPACITY]; -}; - -/* Atomic counters are individually sampled, not one atomic transaction. */ -struct apple_display_event_counters { - u64 committed; - u64 attempts; - u64 busy_drops; - u64 sequence_drops; - u64 overwritten; -}; - -/* Standalone storage: no device, ring, lock or callback pointer survives copy. */ -struct apple_display_event_snapshot { - char name[64]; - struct apple_display_event_counters start; - struct apple_display_event_counters end; - u64 seq_start; - u64 seq_end; - u64 start_ns; - u64 end_ns; - u32 record_count; - u32 copied; - u32 gaps; - bool complete; - u8 valid[APPLE_DISPLAY_EVENT_CAPACITY]; - struct apple_display_event_record records[APPLE_DISPLAY_EVENT_CAPACITY]; -}; - -/* Owner allocates/initializes before publishing, and quiesces before freeing. */ -static inline void apple_display_event_log_init(struct apple_display_event_log *log, - const char *name) -{ - memset(log, 0, sizeof(*log)); - raw_spin_lock_init(&log->lock); - atomic64_set(&log->attempts, 0); - atomic64_set(&log->busy_drops, 0); - atomic64_set(&log->sequence_drops, 0); - strscpy(log->name, name, sizeof(log->name)); -} - -static inline void __apple_display_event_record(struct apple_display_event_log *log, - u16 type, s32 result, u16 flags, - const u64 data[9]) -{ - struct apple_display_event_record *record; - unsigned long irqflags; - u64 sequence; - - atomic64_inc(&log->attempts); - if (!raw_spin_trylock_irqsave(&log->lock, irqflags)) { - atomic64_inc(&log->busy_drops); - return; - } - /* Never wrap to sequence zero or relabel old records after exhaustion. */ - if (log->committed == ~(u64)0) { - atomic64_inc(&log->sequence_drops); - raw_spin_unlock_irqrestore(&log->lock, irqflags); - return; - } - sequence = log->committed + 1; - record = &log->records[(sequence - 1) % APPLE_DISPLAY_EVENT_CAPACITY]; - record->timestamp_ns = ktime_get_mono_fast_ns(); - record->sequence = sequence; - record->type = type; - record->flags = flags; - record->result = result; - memcpy(record->data, data, sizeof(record->data)); - log->committed = sequence; - raw_spin_unlock_irqrestore(&log->lock, irqflags); -} - -/* Disabled means no payload evaluation, clock read, lock or counter access. - * Payload expressions must still be POD observations, never hardware helpers. - */ -#define APPLE_DISPLAY_EVENT_RECORD(log_, type_, result_, flags_, \ - a_, b_, c_, d_, e_, f_, g_, h_, i_) do { \ - struct apple_display_event_log *__event_log = (log_); \ - if (__event_log) { \ - const u64 __event_data[9] = { (a_), (b_), (c_), (d_), (e_), \ - (f_), (g_), (h_), (i_) }; \ - __apple_display_event_record(__event_log, (type_), (result_), \ - (flags_), __event_data); \ - } \ -} while (0) - -/* Caller holds the recorder lock; no hardware or owner state is accessed. */ -static inline void __apple_display_event_counters(struct apple_display_event_log *log, - struct apple_display_event_counters *out) -{ - out->committed = log->committed; - out->attempts = (u64)atomic64_read(&log->attempts); - out->busy_drops = (u64)atomic64_read(&log->busy_drops); - out->sequence_drops = (u64)atomic64_read(&log->sequence_drops); - out->overwritten = log->committed > APPLE_DISPLAY_EVENT_CAPACITY ? - log->committed - APPLE_DISPLAY_EVENT_CAPACITY : 0; -} - -/* Allocate snapshot before acquiring any owner reference/lock. EAGAIN is not - * retried here: a reader must never spin waiting for a producer or vice versa. - */ -static inline int apple_display_event_snapshot_begin(struct apple_display_event_log *log, - struct apple_display_event_snapshot *out) -{ - unsigned long irqflags; - - memset(out, 0, sizeof(*out)); - out->start_ns = ktime_get_mono_fast_ns(); - if (!raw_spin_trylock_irqsave(&log->lock, irqflags)) - return -EAGAIN; - memcpy(out->name, log->name, sizeof(out->name)); - __apple_display_event_counters(log, &out->start); - out->seq_end = log->committed; - out->record_count = log->committed < APPLE_DISPLAY_EVENT_CAPACITY ? - log->committed : APPLE_DISPLAY_EVENT_CAPACITY; - out->seq_start = out->record_count ? - out->seq_end - out->record_count + 1 : 0; - raw_spin_unlock_irqrestore(&log->lock, irqflags); - return 0; -} - -/* Return 1 for more chunks, 0 when complete, or EAGAIN without blocking. - * The fixed begin bounds never expand to include later writes. Each requested - * sequence is checked under lock; an overwrite is explicitly retained as a gap. - */ -static inline int apple_display_event_snapshot_next(struct apple_display_event_log *log, - struct apple_display_event_snapshot *out) -{ - unsigned long irqflags; - u32 limit, i; - bool finished; - - if (out->complete) - return 0; - if (!raw_spin_trylock_irqsave(&log->lock, irqflags)) - return -EAGAIN; - limit = out->record_count - out->copied; - if (limit > APPLE_DISPLAY_EVENT_CHUNK) - limit = APPLE_DISPLAY_EVENT_CHUNK; - for (i = 0; i < limit; i++) { - u32 index = out->copied++; - u64 sequence = out->seq_start + index; - struct apple_display_event_record *source = - &log->records[(sequence - 1) % APPLE_DISPLAY_EVENT_CAPACITY]; - - if (source->sequence == sequence) { - out->records[index] = *source; - out->valid[index] = 1; - } else { - /* Cleared by begin: do not expose the replacement payload. */ - out->records[index].sequence = sequence; - out->gaps++; - } - } - finished = out->copied == out->record_count; - if (finished) - __apple_display_event_counters(log, &out->end); - raw_spin_unlock_irqrestore(&log->lock, irqflags); - if (finished) { - out->end_ns = ktime_get_mono_fast_ns(); - out->complete = true; - } - return finished ? 0 : 1; -} - -static inline int apple_display_event_snapshot_copy(struct apple_display_event_log *log, - struct apple_display_event_snapshot *out) -{ - int ret = apple_display_event_snapshot_begin(log, out); - - if (ret) - return ret; - do { - ret = apple_display_event_snapshot_next(log, out); - } while (ret > 0); - return ret; -} - -#endif /* _LINUX_SOC_APPLE_DISPLAY_EVENT_LOG_H */ diff --git a/include/linux/soc/aurora/apif-mmio.h b/include/linux/soc/aurora/apif-mmio.h deleted file mode 100644 index fbf865f3a24c6..0000000000000 --- a/include/linux/soc/aurora/apif-mmio.h +++ /dev/null @@ -1,66 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LINUX_SOC_AURORA_APIF_MMIO_H -#define _LINUX_SOC_AURORA_APIF_MMIO_H - -#include -#include -#include - -struct device; -struct aurora_apif; - -#define AURORA_APIF_SELECTOR(table, endpoint) \ - (((u64)(table) << 32) | (endpoint)) -#define AURORA_APIF_SERVICE 0xffffffffU -#define AURORA_APIF_TRANSLATE 1 -#define AURORA_APIF_ALLOC_FRAME 2 -#define AURORA_APIF_FREE_FRAME 3 -#define AURORA_APIF_MAP_IO 4 -#define AURORA_APIF_GUEST_ROOT 5 -#define AURORA_APIF_BOOT_IOMMU 6 -#define AURORA_APIF_BOOT_TABLE 7 -#define AURORA_APIF_FIND_FRAME 8 -#define AURORA_APIF_TAG_FRAME 9 -#define AURORA_APIF_BOOT_DART 0 - -struct aurora_apif_op { - u64 selector; - u64 arg[6]; - u64 ret; -}; - -struct aurora_apif_boot_iommu { - u64 level, count, base, size, pa, ipa, attrs, reserved; -}; - -/* Each client owns a buffer. Submission is synchronous and atomic-context - * safe. Unbind drains the client and subsequent calls return -ENODEV. - */ -#if IS_REACHABLE(CONFIG_AURORA_APIF_MMIO) -struct aurora_apif *aurora_apif_get(struct device *dev); -void aurora_apif_put(struct aurora_apif *apif); -/* @done counts executed operations. The transport does not interpret native - * results; endpoint-specific status and ownership belong to the caller. - */ -int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, - u32 count, u32 *done); - -#else -static inline struct aurora_apif *aurora_apif_get(struct device *dev) -{ - return ERR_PTR(-ENODEV); -} -static inline void aurora_apif_put(struct aurora_apif *apif) -{ -} -static inline int aurora_apif_submit(struct aurora_apif *apif, - struct aurora_apif_op *ops, u32 count, - u32 *done) -{ - if (done) - *done = 0; - return -ENODEV; -} -#endif - -#endif diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 1f2c4b5b33fa8..6fb32e0396fa9 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -4,7 +4,6 @@ TARGETS += alloc_tag TARGETS += alsa TARGETS += amd-pstate TARGETS += arm64 -TARGETS += aurora-apif TARGETS += bpf TARGETS += breakpoints TARGETS += cachestat diff --git a/tools/testing/selftests/aurora-apif/Makefile b/tools/testing/selftests/aurora-apif/Makefile deleted file mode 100644 index fba0d147f9246..0000000000000 --- a/tools/testing/selftests/aurora-apif/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Each sibling PR contributes its own tests to the shared base. -TEST_PROGS := $(filter-out common.py,$(wildcard *.py)) -TEST_FILES := common.py -include ../lib.mk diff --git a/tools/testing/selftests/aurora-apif/apif_mmio.py b/tools/testing/selftests/aurora-apif/apif_mmio.py deleted file mode 100755 index 8cc3484814f48..0000000000000 --- a/tools/testing/selftests/aurora-apif/apif_mmio.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only -"""Host fault injection against production APIF functions; no firmware calls.""" -import os -from pathlib import Path -import re -import shlex -import subprocess -import tempfile - -ROOT = Path(__file__).resolve().parents[4] - - -def function(path, name): - source = (ROOT / path).read_text() - match = re.search(r"^.*\b" + name + r"\([^;]*?\)\n\{", source, re.M) - if not match: - raise RuntimeError(f"missing production function {name}") - start = match.start() - pos = match.end() - 1 - depth = 0 - for end in range(pos, len(source)): - if source[end] == "{": - depth += 1 - elif source[end] == "}": - depth -= 1 - if not depth: - return source[start:end + 1] - raise RuntimeError(f"unterminated function {name}") - - -HEADER = r''' -#include -#include -#include -#include -#include -#include -#include -typedef uint16_t u16; -typedef uint32_t u32; -typedef int32_t s32; -typedef uint64_t u64; -typedef uintptr_t phys_addr_t; -#define __packed __attribute__((packed)) -#define U64_MAX UINT64_MAX -#define PHYS_ADDR_MAX UINTPTR_MAX -#define SZ_16K 16384UL -#define SZ_4G (1ULL << 32) -#define PAGE_SIZE SZ_16K -#define IOMMU_READ 1 -#define IOMMU_WRITE 2 -#define IOMMU_CACHE 4 -#define IOMMU_NOEXEC 8 -#define IOMMU_MMIO 16 -#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) -#define IS_REACHABLE(x) 1 -#define GFP_ATOMIC 0 -#define kmalloc(n,flags) malloc(n) -#define virt_to_phys(p) ((u64)(uintptr_t)(p)) -#define round_down(v,a) ((v) & ~((a)-1)) -#define min(a,b) ((a)<(b)?(a):(b)) -#define READ_ONCE(v) (v) -#define raw_spin_lock_irqsave(lock,flags) ((void)(lock), (flags)=0) -#define raw_spin_unlock_irqrestore(lock,flags) ((void)(lock), (void)(flags)) -#define mutex_lock(lock) ((void)(lock)) -#define mutex_unlock(lock) ((void)(lock)) -''' -api = (ROOT / "include/linux/soc/aurora/apif-mmio.h").read_text() -api = "\n".join(line for line in api.splitlines() if not line.startswith("#include")) -HEADER += api + "\n" - - -def transport_test(): - submit = function("drivers/firmware/aurora-apif-mmio.c", "aurora_apif_submit") - return HEADER + r''' -struct apif_batch { u32 count, flags; s32 status; u32 done; struct aurora_apif_op ops[3]; }; -struct aurora_apif { int lock; bool live; struct apif_batch *batch; u32 max_ops; }; -static unsigned rings; -static int mode; -static void apif_ring(struct aurora_apif *apif) -{ - rings++; - apif->batch->done = apif->batch->count; - apif->batch->status = 0; - for (u32 i = 0; i < apif->batch->count; i++) apif->batch->ops[i].ret = U64_MAX; - if (mode == 1) { apif->batch->done = 1; apif->batch->status = -EINVAL; } - if (mode == 2) apif->batch->done++; - if (mode == 3) { apif->batch->done = 0; apif->batch->status = -EINPROGRESS; } - if (mode == 4) { apif->batch->done = 0; apif->batch->status = 1; } - if (mode == 5) { apif->batch->done = 1; } -} -''' + submit + r''' -int main(void) -{ - struct apif_batch batch = {}; - struct aurora_apif client = { .live = true, .batch = &batch, .max_ops = 3 }; - struct aurora_apif_op ops[8] = {}; - u32 done; - assert(!aurora_apif_submit(&client, ops, 8, &done)); - assert(done == 8 && rings == 3 && ops[7].ret == U64_MAX); - assert(!aurora_apif_submit(&client, NULL, 0, &done) && !done && rings == 3); - assert(aurora_apif_submit(&client, NULL, 1, &done) == -EINVAL && !done); - mode = 1; - assert(aurora_apif_submit(&client, ops, 8, &done) == -EINVAL && done == 1); - mode = 2; - assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); - mode = 3; - assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); - mode = 4; - assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); - mode = 5; - assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && done == 1); - client.live = false; - assert(aurora_apif_submit(&client, ops, 1, &done) == -ENODEV && !done); - return 0; -} -''' - - -def dart_test(): - path = "drivers/iommu/apple-dart-apif.c" - source = (ROOT / path).read_text() - constants = source[source.index("#define APIF_DART_INIT"):source.index("static const struct iommu_ops")] - funcs = "\n".join(function(path, name) for name in - ["apif_force_iovas", "apif_one", "apif_prepare_table", "apif_boot_query", "apif_ring"]) - return HEADER + constants + r''' -struct apif_retained_run { u64 dva, ipa, size; }; -struct apif_retained_sid { unsigned root_level; u32 count; struct apif_retained_run *runs; }; -struct apple_dart_apif { - struct aurora_apif *apif; - struct aurora_apif_op native_ops[2 * APIF_BATCH_MAX_OPS]; - struct aurora_apif_boot_iommu *boot_info; - u64 *page_lists[APIF_BATCH_MAX_OPS], page_list_pa[APIF_BATCH_MAX_OPS]; - u32 dart_id, num_streams; - u64 iova_force, dma_min, dma_max; - bool boot_mappings; - bool ownership_uncertain; - struct apif_retained_sid *retained[DART_APIF_MAX_STREAMS]; - struct apif_batch *batch; -}; -static struct aurora_apif_op calls[2048]; -static unsigned ncalls; -static u64 boot_table, found_table, retype_result = 1, fail_selector; -static int fail_status; -static int native_done = -1; -int aurora_apif_submit(struct aurora_apif *client, struct aurora_apif_op *ops, u32 count, u32 *done) -{ - (void)client; - if (done) *done = 0; - for (u32 i = 0; i < count; i++) { - struct aurora_apif_op *op = &ops[i]; - calls[ncalls++] = *op; - if (fail_selector && op->selector == fail_selector) return fail_status; - if (native_done >= 0 && (op->selector >> 32) != AURORA_APIF_SERVICE && i >= (u32)native_done) - return -EIO; - op->ret = 0; - if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TRANSLATE)) - op->ret = op->arg[0] + SZ_4G; - if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_BOOT_TABLE)) - op->ret = boot_table; - if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_FIND_FRAME)) - op->ret = found_table; - if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_ALLOC_FRAME)) - op->ret = 0x90000000; - if (op->selector == AURORA_APIF_SELECTOR(0, 1)) op->ret = retype_result; - /* A void endpoint is allowed to return nonzero. */ - if (op->selector == AURORA_APIF_SELECTOR(3, 0)) op->ret = U64_MAX; - if (done) *done = i + 1; - } - return 0; -} -''' + funcs + r''' -int main(void) -{ - struct apple_dart_apif dart = { .dart_id = 5, .num_streams = 16 }; - struct aurora_apif_boot_iommu info = {}; - dart.boot_info = &info; - dart.batch = calloc(1, PAGE_SIZE); - assert(dart.batch); - boot_table = 0x80000000; - assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 1); - boot_table = 0; found_table = 0x80000000; ncalls = 0; - assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 2); - found_table = 0; ncalls = 0; - assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 6); - assert(calls[3].selector == AURORA_APIF_SELECTOR(0, 1)); - assert(calls[3].arg[0] == 0x90000000 && calls[3].arg[1] == 11 && calls[3].arg[2] == 24); - assert(calls[4].selector == AURORA_APIF_SELECTOR(3, 0)); - assert(calls[4].arg[0] == 5 && calls[4].arg[1] == 3 && calls[4].arg[4] == 0x90000000); - assert(calls[5].selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TAG_FRAME)); - retype_result = 0; ncalls = 0; - assert(apif_prepare_table(&dart, 3, 0, 2) == -EIO && ncalls == 5); - assert(calls[4].selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_FREE_FRAME)); - retype_result = 1; ncalls = 0; - fail_selector = AURORA_APIF_SELECTOR(3, 0); fail_status = -ETIMEDOUT; - assert(apif_prepare_table(&dart, 3, 0, 2) == -ETIMEDOUT && dart.ownership_uncertain); - assert(apif_ring(&dart) == -EIO && !dart.batch->done); - dart.ownership_uncertain = false; fail_selector = 0; ncalls = 0; - dart.iova_force = 1ULL << 40; - dart.batch->count = 2; - for (unsigned i = 0; i < 2; i++) { - dart.batch->op[i] = (struct apif_op){ .op = APIF_DART_MAP, - .arg = { 3, i * 0x4000, 0x800000 + i * 0x4000, 0x4000, 3 | IOMMU_NOEXEC } }; - } - assert(!apif_ring(&dart) && dart.batch->done == 2 && ncalls == 6); - assert(calls[4].selector == AURORA_APIF_SELECTOR(3, 2)); - assert(calls[4].arg[2] == (1ULL << 40)); - assert(calls[4].arg[3] == virt_to_phys(dart.page_lists[0]) + SZ_4G); - assert(dart.page_lists[0][0] == 0x100800000ULL); - assert(calls[4].arg[5] == 4 && dart.batch->op[0].arg[1] == 0); - ncalls = 0; native_done = 1; - assert(apif_ring(&dart) == -EIO && dart.batch->done == 1); - assert(dart.batch->op[0].arg[1] == 0 && dart.batch->op[1].arg[1] == 0x4000); - native_done = -1; ncalls = 0; - dart.batch->op[0].arg[4] = 256; - assert(apif_ring(&dart) == -EINVAL && !ncalls && !dart.batch->done); - for (unsigned i = 0; i < APIF_BATCH_MAX_OPS; i++) free(dart.page_lists[i]); - free(dart.batch); - return 0; -} -''' - - -def sart_test(): - return HEADER + r''' -struct device {}; -struct mutex { int unused; }; -struct sart_apif { - struct device *dev; - struct aurora_apif *apif; - struct mutex lock; - bool active; - struct { phys_addr_t address; size_t size; } regions[16]; -}; -struct apple_sart { struct sart_apif *state; }; -#define SART_APIF_MAX_REGIONS 16 -#define apple_sart_backend_data(sart) ((sart)->state) -#define dma_to_phys(dev,address) (address) -static struct aurora_apif_op calls[8]; -static unsigned ncalls; -static u64 unmap_result; -int aurora_apif_submit(struct aurora_apif *client, struct aurora_apif_op *ops, u32 count, u32 *done) -{ - (void)client; (void)done; - assert(count == 1); - calls[ncalls++] = *ops; - ops->ret = U64_MAX; - if (ops->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TRANSLATE)) - ops->ret = ops->arg[0] + SZ_4G; - if (ops->selector == AURORA_APIF_SELECTOR(5, 2)) ops->ret = unmap_result; - return 0; -} -''' + function("drivers/soc/apple/apple-sart-apif.c", "sart_apif_region") + r''' -int main(void) -{ - struct sart_apif state = {}; - struct apple_sart sart = { &state }; - assert(sart_apif_region(&sart, 0, 0, true) == -EINVAL); - assert(sart_apif_region(&sart, 1, SZ_16K, true) == -EINVAL); - assert(!sart_apif_region(&sart, 0x800000, SZ_16K, true)); - assert(ncalls == 3 && state.active && state.regions[0].size == SZ_16K); - assert(calls[2].selector == AURORA_APIF_SELECTOR(5, 1) && calls[2].arg[0] == 0x100800000ULL); - ncalls = 0; unmap_result = 1; - assert(sart_apif_region(&sart, 0x800000, SZ_16K, false) == -EIO); - assert(state.regions[0].size == SZ_16K); - ncalls = 0; unmap_result = 0; - assert(!sart_apif_region(&sart, 0x800000, SZ_16K, false) && !state.regions[0].size); - assert(sart_apif_region(&sart, 0x800000, SZ_16K, false) == -ENOENT); - return 0; -} -''' - - -def nvme_test(): - helpers = "\n".join(function("drivers/nvme/host/apple-apif.h", name) for name in - ["apple_nvme_apif_translate", "apple_nvme_apif_request"]) - return HEADER + r''' -#define SZ_4K 4096UL -#define APPLE_NVME_APIF_PAGES 257 -#define APPLE_NVME_APIF_LIST_SIZE SZ_4K -#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) -#define IS_ALIGNED(n,a) (!((n)&((a)-1))) -#define lockdep_assert_held(p) ((void)(p)) -#define dma_wmb() ((void)0) -#define cpu_to_le16(v) (v) -#define le64_to_cpu(v) (v) -typedef uint64_t __le64; -struct command { struct { struct {u64 prp1, prp2;} dptr; u32 command_id; } common; }; -struct apple_nvmmu_tcb { u32 pad; unsigned short length; char rest[122]; }; -struct apple_nvme_iod { struct command cmd; bool apif_registered; __le64 *prps; }; -struct request { struct apple_nvme_iod iod; u32 bytes; }; -struct apple_nvme_hw { u32 max_queue_depth; }; -struct apple_nvme { void *dev; int lock; struct aurora_apif *apif; struct apple_nvme_hw *hw; - struct aurora_apif_op *apif_ops; void *apif_scratch; u64 apif_scratch_dma; bool apif_uncertain; }; -struct apple_nvme_queue { struct apple_nvme *anv; bool is_adminq; struct apple_nvmmu_tcb tcbs[64]; }; -static struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) {return q->anv;} -static struct apple_nvme_iod *blk_mq_rq_to_pdu(struct request *r) {return &r->iod;} -static u32 nvme_tag_from_cid(u32 tag) {return tag;} -static u32 blk_rq_payload_bytes(struct request *r) {return r->bytes;} -static void **apple_nvme_iod_list(struct request *r) {return (void **)&r->iod.prps;} -static u64 dma_to_phys(void *dev, u64 p) {(void)dev; return p;} -static int map_calls, translate_calls, mode; -static struct aurora_apif_op mapped; -int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, u32 n, u32 *done) { - (void)apif; if(done) *done=n; - for(u32 i=0;ilength==255); - assert(translate_calls>=257); - return 0; -} -''' - - -def pcie_test(): - layout = (ROOT / "drivers/pci/controller/apple-apif/t6050_pcie_sequence.h").read_text() - gpio = (ROOT / "drivers/pci/controller/apple-apif/t6050_pcie_gpio.h").read_text() - layout = "\n".join(x for x in (layout + gpio).splitlines() if not x.startswith("#include")) - helpers = "\n".join(function("drivers/pci/controller/apple-apif/" + file, name) for file, name in - [("t6050_pcie_msi.c", "t6050_pcie_configure_msi_port0"), - ("t6050_pcie_gpio.c", "t6050_pcie_gpio_perst"), - ("t6050_pcie_disable.c", "update"), - ("t6050_pcie_disable.c", "t6050_pcie_disable_port0")]) - return HEADER + layout + "\n" + helpers + r''' -static u32 pad, address_lo, address_hi; -static int writes, fail_write, power_calls, polls; -static int fake_read(void *ctx,unsigned reg,unsigned off,u32 *v) { - (void)ctx;(void)reg;(void)off;*v=0;return 0; -} -static int fake_write(void *ctx,unsigned reg,unsigned off,u32 v) { - (void)ctx;assert(reg==T6050_PCIE_PORT0);writes++; - if(fail_write==writes) return -EIO; - if(off==0x16c) address_lo=v; - if(off==0x170) address_hi=v; - return 0; -} -static int fake_poll(void *ctx,unsigned reg,unsigned off,u32 mask,u32 value,unsigned timeout) { - (void)ctx;(void)reg;(void)off;(void)mask;(void)value; - assert(timeout>0 && timeout<=250000);polls++;return -ETIMEDOUT; -} -static void fake_delay(void *ctx,unsigned us) {(void)ctx;assert(us<=100000);} -static int power(void *ctx,bool on) {(void)ctx;assert(!on);power_calls++;return -EBUSY;} -static int pad_read(void *ctx,u32 *v) {(void)ctx;*v=pad;return 0;} -static int pad_write(void *ctx,u32 v) {(void)ctx;pad=v;writes++;return 0;} -int main(void) { - struct t6050_pcie_io io={.read=fake_read,.write=fake_write,.poll=fake_poll,.delay=fake_delay}; - assert(!t6050_pcie_configure_msi_port0(&io,0x1fffff000ULL,64)); - assert(writes==67 && address_hi==1 && address_lo==0xfffff000); - writes=0;fail_write=2; - assert(t6050_pcie_configure_msi_port0(&io,0xfffff000,32)==-EIO && writes==2); - writes=0;assert(t6050_pcie_configure_msi_port0(&io,1,32)==-EINVAL && !writes); - assert(t6050_pcie_configure_msi_port0(&io,0xfffff000,257)==-EINVAL); - assert(t6050_pcie_disable_port0(&io,power,250000)==-EBUSY); - assert(power_calls==1 && !writes && !polls); - struct t6050_gpio_pad_io g={.read=pad_read,.write=pad_write}; - pad=1U<<21;assert(t6050_pcie_gpio_perst(&g,true)==-EIO && !writes); - pad=0;assert(!t6050_pcie_gpio_perst(&g,true) && (pad&0x27f)==0x202); - assert(!t6050_pcie_gpio_perst(&g,false) && (pad&0x27f)==0x203); - return 0; -} -''' - - -def run_tests(tests): - compiler = shlex.split(os.environ.get("HOSTCC", "cc")) - print(f"TAP version 13\n1..{len(tests)}", flush=True) - with tempfile.TemporaryDirectory(prefix="apif-selftest-") as directory: - for i, (name, test) in enumerate(tests, 1): - path = Path(directory) / name - path.with_suffix(".c").write_text(test()) - subprocess.run(compiler + ["-std=gnu11", "-Wall", "-Wextra", "-Werror", - str(path.with_suffix(".c")), "-o", str(path)], check=True) - subprocess.run([str(path)], check=True) - print(f"ok {i} - {name} production function", flush=True) - - -if __name__ == "__main__": - run_tests([("transport", transport_test), ("dart", dart_test), ("sart", sart_test)]) diff --git a/tools/testing/selftests/aurora-apif/common.py b/tools/testing/selftests/aurora-apif/common.py deleted file mode 100644 index 3476376b8026d..0000000000000 --- a/tools/testing/selftests/aurora-apif/common.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -"""Production-source extraction shared by the APIF host tests.""" -import re - -def block(source, pattern): - match = re.search(pattern, source, re.M) - if not match: - raise RuntimeError(f"missing production block {pattern}") - start = match.start() - brace = source.index("{", match.end()) - depth = 0 - for pos in range(brace, len(source)): - if source[pos] == "{": - depth += 1 - elif source[pos] == "}": - depth -= 1 - if not depth: - return source[start:pos + 1] - raise RuntimeError("unterminated production block") diff --git a/tools/testing/selftests/aurora-apif/dart_retained.py b/tools/testing/selftests/aurora-apif/dart_retained.py deleted file mode 100755 index a7c6f3c4d560b..0000000000000 --- a/tools/testing/selftests/aurora-apif/dart_retained.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only -"""Execute the production DART stream-state operation builder with retained SIDs.""" -from pathlib import Path -import subprocess,tempfile,sys -root=Path(__file__).resolve().parents[4] -p=Path(sys.argv[1]) if len(sys.argv)>1 else root/'drivers/iommu/apple-dart-apif.c' -s=p.read_text();start=s.index('\t\tcase APIF_DART_ENABLE:');end=s.index('\t\tcase 7:',start);fragment=s[start:end] -code=r''' -#include -#include -#include -#include -typedef uint64_t u64; -#define APIF_DART_ENABLE 5 -#define APIF_DART_DISABLE 6 -#define AURORA_APIF_SELECTOR(t,e) (((u64)(t)<<32)|(e)) -#define EINVAL 22 -struct apple_dart_apif { unsigned dart_id,num_streams;void *retained[256]; }; -struct apif_op { unsigned op;u64 arg[6]; }; -struct aurora_apif_op { u64 selector,arg[6],ret; }; -static int build(struct apple_dart_apif *dart,unsigned opcode,u64 sid,unsigned *count,struct aurora_apif_op *native) { - struct apif_op request={.op=opcode,.arg={sid}};struct apif_op *op=&request;u64 *a=op->arg; - struct aurora_apif_op call={.arg={dart->dart_id,sid}};unsigned n=0;int ret=0; - switch(op->op) { -''' + fragment + r''' - default:assert(false); - } -out:*count=n;return ret; -} -int main(void) { - const unsigned sizes[]={1,16,256};unsigned checked=0; - for(unsigned k=0;k<3;k++) { - struct apple_dart_apif d={.dart_id=16,.num_streams=sizes[k]}; - for(unsigned sid=0;sid Date: Mon, 9 Nov 2026 12:16:43 -0500 Subject: [PATCH 346/392] dt-bindings: Drop the APIF bindings Remove the APIF MMIO, DART and SART bindings and the APIF ASCWrap v6 compatible, and the APIF host wording in the N1 Control binding. Signed-off-by: Diljit Singh --- .../bindings/firmware/aurora,apif-mmio.yaml | 36 -------- .../bindings/iommu/apple,dart-apif.yaml | 84 ------------------- .../bindings/mailbox/apple,ascwrap-v6.yaml | 1 - .../bindings/soc/apple/apple,sart-apif.yaml | 35 -------- 4 files changed, 156 deletions(-) delete mode 100644 Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml delete mode 100644 Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml delete mode 100644 Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml diff --git a/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml b/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml deleted file mode 100644 index c154786a0593c..0000000000000 --- a/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/firmware/aurora,apif-mmio.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Aurora Platform Interface over Memory-Mapped I/O - -maintainers: - - Dj - -description: | - Synthetic MMIO transport supplied by Aura's EL2-resident monitor. Version 2 - forwards native SPTM selectors and arguments and provides memory-ownership - services. Peripheral policy belongs to consumers, which reference this node - through the aurora,apif phandle. The register page must be absent from the - guest stage-2 mappings so accesses reach Aura. - -properties: - compatible: - const: aurora,apif-mmio - reg: - maxItems: 1 - -required: - - compatible - - reg - -additionalProperties: false - -examples: - - | - apif: firmware@61ff0000 { - compatible = "aurora,apif-mmio"; - reg = <0x61ff0000 0x4000>; - }; diff --git a/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml b/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml deleted file mode 100644 index dbf1169dae26b..0000000000000 --- a/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/iommu/apple,dart-apif.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Apple DART mediated through Aura APIF - -maintainers: - - Dj - -description: | - A DART instance accessed through native SPTM calls rather than guest MMIO. - Linux owns stream grouping, table setup, mapping rollback and DMA apertures. - Firmware-owned streams are adopted without replacing their roots or replaying - retained mappings. Unit identifiers are firmware metadata, not chip IDs. - -properties: - compatible: - const: apple,dart-apif - '#iommu-cells': - const: 1 - description: Stream identifier. - aurora,apif: - $ref: /schemas/types.yaml#/definitions/phandle - description: APIF version-2 transport provider. - power-domains: - maxItems: 1 - description: | - Native register-clock dependency needed before the first APIF DART call. - The provider must be powered independently of consumer probe ordering. - For T6050 ATC2 USB DART this is the ATC2_USB domain shared with DWC3. - apple,apif-dart-id: - $ref: /schemas/types.yaml#/definitions/uint32 - maximum: 255 - description: Native SPTM DART unit identifier. - apple,apif-num-streams: - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 1 - maximum: 256 - default: 256 - description: Number of streams supported by this unit. - apple,apif-retained-sids: - $ref: /schemas/types.yaml#/definitions/uint32-array - minItems: 1 - maxItems: 256 - uniqueItems: true - items: - maximum: 255 - description: Firmware-owned stream IDs which Aura adopted at bootstrap. - apple,apif-iova-force: - $ref: /schemas/types.yaml#/definitions/uint64 - enum: [0, 1099511627776] - description: | - Optional native bit-40 force mask for a low-address device DMA view. - Must not overlap apple,dma-range. Omit for a native DVA aperture. - apple,apif-reserve-last-page: - type: boolean - description: Native aperture forbids allocating its final 16 KiB granule. - apple,dma-range: - $ref: /schemas/types.yaml#/definitions/uint64-array - minItems: 2 - maxItems: 2 - description: | - Device-visible DMA aperture base and size, aligned to 16 KiB and within - 42 bits. Defaults to the low 4 GiB. This is not a CPU mapping. - -required: - - compatible - - '#iommu-cells' - - aurora,apif - - apple,apif-dart-id - -additionalProperties: false - -examples: - - | - iommu { - compatible = "apple,dart-apif"; - #iommu-cells = <1>; - aurora,apif = <&apif>; - apple,apif-dart-id = <5>; - apple,apif-num-streams = <16>; - }; diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index a08c3224e7b1e..349d6930d07e0 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -24,7 +24,6 @@ properties: - apple,t6040-ascwrap-v6 - apple,t8140-ascwrap-v6 - apple,t8142-ascwrap-v6 - - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 - apple,t6050-j775d-ascwrap-v6 diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml deleted file mode 100644 index 4e5ece8bebaf5..0000000000000 --- a/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/soc/apple/apple,sart-apif.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Apple SART mediated through Aura APIF - -maintainers: - - Dj - -description: | - SART provider using native SPTM table-5 calls. Existing consumers reference - this provider through their apple,sart phandle. Linux tracks only its own - regions; unknown firmware grants are not reported as inherited regions. - -properties: - compatible: - const: apple,sart-apif - aurora,apif: - $ref: /schemas/types.yaml#/definitions/phandle - description: APIF version-2 transport provider. - -required: - - compatible - - aurora,apif - -additionalProperties: false - -examples: - - | - sart { - compatible = "apple,sart-apif"; - aurora,apif = <&apif>; - }; From 9310f7631c1867f10c5b253fa77c8f7357b95367 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 12:16:43 -0500 Subject: [PATCH 347/392] Documentation: Drop the APIF documentation Remove the APIF platform, driver and N1 transport pages, their index entries and the APIF MAINTAINERS entries. Signed-off-by: Diljit Singh --- .../arch/arm64/aurora-apif-centauri.rst | 78 ------------- .../arch/arm64/aurora-apif-platform.rst | 109 ------------------ Documentation/arch/arm64/index.rst | 7 -- .../firmware/aurora-apif-display.rst | 99 ---------------- .../driver-api/firmware/aurora-apif-nvme.rst | 39 ------- .../driver-api/firmware/aurora-apif-pcie.rst | 30 ----- .../driver-api/firmware/aurora-apif-pmp.rst | 35 ------ .../driver-api/firmware/aurora-apif.rst | 100 ---------------- Documentation/driver-api/firmware/index.rst | 4 - MAINTAINERS | 23 ---- 10 files changed, 524 deletions(-) delete mode 100644 Documentation/arch/arm64/aurora-apif-centauri.rst delete mode 100644 Documentation/arch/arm64/aurora-apif-platform.rst delete mode 100644 Documentation/driver-api/firmware/aurora-apif-display.rst delete mode 100644 Documentation/driver-api/firmware/aurora-apif-nvme.rst delete mode 100644 Documentation/driver-api/firmware/aurora-apif-pcie.rst delete mode 100644 Documentation/driver-api/firmware/aurora-apif-pmp.rst delete mode 100644 Documentation/driver-api/firmware/aurora-apif.rst diff --git a/Documentation/arch/arm64/aurora-apif-centauri.rst b/Documentation/arch/arm64/aurora-apif-centauri.rst deleted file mode 100644 index c24f4d6fb7cb9..0000000000000 --- a/Documentation/arch/arm64/aurora-apif-centauri.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -N1 firmware and PCI transport -============================= - -The Control function owns the firmware bootstrap and its DMA allocations. -Alpha and Beta use their own PCI-function DMA mappings. The driver requires -an APIF host with the supported isolated Gen3 x1 reset layout. All endpoint -DMA/MSI must be quiesced and the endpoint left in the stopped ROM personality; -live sessions cannot be adopted by this interface. - -The Control PCI DT node supplies ``firmware-name``, -``apple,centauri-platform-id`` and ``apple,centauri-protocol = <27>``. The -platform identifier and personalized FTAB must agree with the actual device. -These are firmware inputs supplied by the boot producer, rather than a -machine-name lookup in the driver. No firmware or calibration data is shipped -in this series. The driver loads the firmware directly and starts asynchronously; -a missing file or failed phase requires correcting the input and reprobe. - -The host's native DART reset requires exclusive Linux ownership and no retained -firmware streams on that unit. The boot producer must resolve ownership before -handoff. This driver cannot discard retained streams to make a reset succeed. -A failed link transition prevents further endpoint accesses. Published DMA and -vectors remain pinned when shutdown cannot be proved; a platform reset is then -required. This is an error path, not a supported live-session handoff. - -Bootstrap command rings are bounded and used once. Radio data-plane ring sizes -and batching are separate from bootstrap. The transport builds without the -Wi-Fi consumer; it still starts the shared firmware for the later Alpha/Beta -consumers. Wi-Fi and Bluetooth are separate follow-up changes. Hardware -qualification of this cleaned series, including suspend/resume, remains pending. - -Wi-Fi station consumer ----------------------- - -``CONFIG_APPLE_CENTAURI_WIFI`` adds the Alpha cfg80211/netdev consumer within -this transport module. It supports the protocol-27 command profile, active -2.4/5 GHz scanning and WPA2-PSK/CCMP station connections with firmware handshake -offload. The matched profile uses a raw 32-byte PMK supplied by cfg80211; -there is no separate passphrase interface. Unknown profiles are rejected by -the Control driver. Passive scan encoding is not established, so disabled and -NO_IR channels are excluded from active scans. WPA3, 6 GHz, roaming and wider -security/radio qualification remain outside the established driver scope. - -Data uses the original 2,048-entry rings and coalesced completions. IRQs and -queued packets schedule the consumer; no periodic data poller or diagnostic -packet flow is added. Malformed frames and completion spans cannot index beyond -the coherent window. A lost command completion poisons further commands and -stops new transmission. Its key payload is cleared only after the transfer-ring -tail proves consumption; uncertain ownership keeps backing memory pinned. -Safe IPC teardown scrubs buffers after verified DMA shutdown. The cleaned -consumer still needs hardware traffic and suspend/resume qualification. - -Bluetooth consumer ------------------- - -``CONFIG_BT_HCICENTAURI`` adds the Beta HCI/ACL PCI driver. A managed device -link requires the Control supplier to have established shared firmware and -orders removal and suspend against it. The Beta DT node supplies the factory -``local-mac-address`` in network byte order; no board-name or temporary-address -fallback selects initialization behavior. - -Protocol-27 initialization provisions the address, completes firmware setup -and supplies a fresh random seed. The connection-command adaptation is enabled -only after the firmware's capability bit confirms support. Status and connection -completion events are normalized to the standard HCI command being tracked. -The firmware duplicate filter is reset in the HCI command-sync path before -filtered scanning starts; other drivers without that callback keep their -existing behavior. Standard report lengths are checked before applying the -matched legacy scan-response fixup. - -The driver uses IRQ-triggered work and the original ring geometry. Diagnostic -capture, unrelated auxiliary rings, manual mode switches and success logging -are excluded. SCO and ISO delivery are not implemented. Coordinated firmware -teardown remains unqualified: a published Beta arena stays pinned on detach, -and rebinding requires a fresh shared-firmware boot. This is not a reset of the -Wi-Fi function. The cleaned driver still needs hardware HCI/ACL, repeated-scan -and suspend/resume qualification. diff --git a/Documentation/arch/arm64/aurora-apif-platform.rst b/Documentation/arch/arm64/aurora-apif-platform.rst deleted file mode 100644 index af062d4c09a57..0000000000000 --- a/Documentation/arch/arm64/aurora-apif-platform.rst +++ /dev/null @@ -1,109 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -M5 Pro non-GPU platform series -============================= - -The APIF platform series separates transport/provider support from device -consumers. The shared ``t6050/base`` PR targets ``integration/aurora`` on -Linux 7.3; every other PR targets ``t6050/base``. Consumer dependencies are -recorded separately, so the review branches do not form a linear chain. -The source snapshot is ``t6050-gpu`` at ``ee68abb105200``. This review map -covers its non-GPU platform changes; hardware qualification of the rebased -combination remains separate. - -.. list-table:: Review branches - :header-rows: 1 - - * - Area - - Branch - * - APIF transport, DART and SART - - `t6050/base `_ - * - Native ANS/NVMe queues - - `t6050/nvme `_ - * - Preinitialized PCIe and MSI - - `t6050/pcie `_ - * - PMP firmware registry - - `t6050/pmp `_ - * - Guest DT conversion - - `t6050/guest-wiring `_ - * - Generation-4 SPMI - - `t6050/spmi `_ - * - SPMI PMU RTC - - `t6050/pmu-rtc `_ - * - SMC consumers and pcIO GPIO - - `t6050/smc `_ - * - DP/USB combo PHY and crossbar - - `t6050/display-phy `_ - * - CPU frequency profile - - `t6050/cpufreq `_ - * - N1 firmware and PCI lifetime - - `t6050/centauri-core `_ - * - N1 Wi-Fi - - `t6050/centauri-wifi `_ - * - N1 Bluetooth - - `t6050/centauri-bluetooth `_ - * - Firmware-27 AFK, DCP and DPTX - - `t6050/dcp `_ - * - Host GPIO interrupt groups - - `t6050/gpio-irqs `_ - * - Initial SPMI Type-C events - - `t6050/typec-events `_ - * - DWC3 controller - - `t6050/usb `_ - * - Power-domain handoff - - `t6050/pmgr `_ - * - MTP keyboard reports - - `t6050/mtp `_ - * - Composed platform configuration - - `t6050/platform-wiring `_ - -CPU identification, idle register preservation and delay handling are -already represented by the base. Existing public interrupt-controller, -USB, I2C/SPI, DockChannel HID, MTP helper and standard platform drivers are -preserved rather than replaced with older copies. The fragment enables -their dependencies alongside the new providers and consumers. - -AGX acceleration, its dedicated mailbox transports and diagnostic system -register probes are outside this series. Firmware binaries, board -calibration data and factory addresses are not included. - -Configuration -------------- - -Apply aurora-apif.config after defconfig and asahi.config, then run -olddefconfig. The fragment makes the selected platform consumers built-in; -firmware and loader metadata are still required. - -The existing HID interface uses explicit apple,power-method and -apple,firmware-managed-reset properties. The latter is valid only for a -power-method-2 multi-touch interface with firmware-owned reset. The loader -must describe the actual ownership; no board-name or firmware-filename -guess selects it. - -Loader and qualification requirements -------------------------------------- - -* NVMe needs a stopped RTKit session and four distinct, reserved queue - pages. The new profile does not adopt an unqualified live queue session. -* N1 cold bootstrap needs its packed platform identifier, protocol 27, - factory Bluetooth address and an exclusively owned PCIe/DART unit. - Unqualified live-session adoption is not enabled. -* CPU frequency control requires an explicit, qualified firmware thermal - ownership claim covering every exposed OPP, including after PMP setup. - The DT conversion tool does not invent that claim. -* DCP needs a running firmware-27 session, inherited framebuffer region, - ordered provider clock table and the coprocessor physical register view - when guest resource addresses differ. -* GPIO interrupt groups and HID reset ownership must match the loader's - retained firmware ownership. - -The conversion tool validates supplied metadata; it cannot establish that -firmware has stopped using a queue, stream, reset line or clock policy. - -Validation ----------- - -The series has native driver builds and host protocol/bounds tests. -Hardware qualification must separately cover boot, storage I/O, network -traffic, physical Type-C attach/detach, display, brightness and suspend. -Retained firmware/DMA state is not reset as a fallback after a timeout. diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index cd82b573e7bfd..72e1fa881a94a 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -9,13 +9,6 @@ ARM64 Architecture acpi_object_usage amu - ../../driver-api/firmware/aurora-apif - ../../driver-api/firmware/aurora-apif-nvme - ../../driver-api/firmware/aurora-apif-pcie - ../../driver-api/firmware/aurora-apif-pmp - ../../driver-api/firmware/aurora-apif-display - aurora-apif-centauri - aurora-apif-platform apple-j813 apple-m6-cpu arm-acpi diff --git a/Documentation/driver-api/firmware/aurora-apif-display.rst b/Documentation/driver-api/firmware/aurora-apif-display.rst deleted file mode 100644 index 8da1054cc8c78..0000000000000 --- a/Documentation/driver-api/firmware/aurora-apif-display.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -DCP display in an Aura guest -=========================== - -The firmware-27 display profile uses the normal IOMFB shared-memory -transport and compact AFK framing. It supplies a separate method/callback -table and present serializer. The existing queued present, framebuffer -retention, backlight policy, Type-C routing and suspend drain are shared -with the public driver. - -This is display-controller support. It does not add AGX acceleration. - -Loader contract ---------------- - -The loader leaves each coprocessor running with its firmware translations -intact. The driver negotiates RTKit without writing its CPU run state or -restarting the coprocessor. A missing running session or a firmware ABI -other than the explicit apple,firmware-compat = <27 0 0> is rejected. - -The internal and external providers use apple,t6050-dcp and -apple,t6050-dcpext respectively. The profile uses a 41-bit DMA -aperture for the coprocessor CPU; scanout and PIODMA retain their independent apertures. - -Each provider describes its inherited framebuffer region using -memory-region and the framebuffer entry in memory-region-names. -Default-surface allocation uses firmware-reported geometry, format and -compression. Unsupported layouts or allocations larger than that region -are rejected. - -apple,clock-ids describes the ordered firmware provider clock table. -Its entry count must match clocks. An explicitly empty property is -accepted for a provider with no clocks; a missing property is rejected. -A callback clock index selects the corresponding Linux clock. - -Display register callbacks normally use resource physical addresses. -If Linux resources are translated guest addresses, the loader supplies -apple,firmware-physical-regs in complete register-table order, -including the bandwidth scratch and optional doorbell apertures. These -addresses are returned to the coprocessor, not dereferenced by Linux. - -The loader must also grant the reserved firmware buffers needed by AFK -through the corresponding IOMMU provider. Remote requests remain bounded -and fail if their device addresses cannot be translated. - -Direct DisplayPort routes -------------------------- - -Protocol-27 DPTX uses endpoint 0x2c and its own drive-settings records. -The explicit service unit takes precedence over the transport interface -identifier. Drive requests validate lane count and voltage/pre-emphasis -before changing the PHY. - -The PHY reports its live clock and coding mode to the crossbar consumer. -A missing required provider or conflicting clock configuration fails the -link operation. Other PHY types retain their existing routing behavior. - -Session lifetime ----------------- - -An adopted session has no confirmed DMA stop boundary. Failed probes and -unbind therefore retain published buffers, mappings and device/domain -references until a fresh firmware boot. Rebinding a retained session is -unsupported. No timeout falls back to resetting a live coprocessor. - -Validation ----------- - -Native arm64 display-driver builds and host tests cover the present wire -layout and framebuffer sizing/bounds. These checks do not establish boot, -physical hotplug, brightness or suspend behavior on hardware. - -Paired firmware-27 scanout -------------------------- - -The T6050 peer allocator consumes reported endpoint capacity and explicit -firmware tiling metadata. It may reserve only an idle, otherwise unrouted -peer on the same DRM device and display die. Physical Type-C routes remain -owned by their normal allocator. The current qualification gates are -``appledrm.peer_allocation_probe=1`` and ``appledrm.peer_scanout=1``; they -match the private baseline's explicit configuration and remain off until -the cleaned combination is qualified. ``appledrm.show_notch=1`` is independent. - -Both wire records are serialized from the primary's logical atomic input -without mutating it. Cross-endpoint start replies, errors and combined -completion watermarks are copied to a bounded callback queue on the owning -IOMFB worker. They never take another endpoint's IOMFB lock from a receive -callback. Normal command/transaction ordering and retained-framebuffer -ownership remain in the common IOMFB implementation. One physical engine's -completion cannot retire the pair. A failed peer submission may follow -acceptance by the primary, so it retains the active transaction and scanout -references as a crashed session rather than treating the error as retirement. - -The background-only paired probe also uses one tracked transaction per -endpoint and cannot run while the paired production completion owner is active. -Host checks cover copied callback dispatch, stop/cancellation, paired completion -coverage and split-wire serialization. Build/host evidence does not replace -physical display, hotplug, DPMS, sleep or paired-frame qualification. diff --git a/Documentation/driver-api/firmware/aurora-apif-nvme.rst b/Documentation/driver-api/firmware/aurora-apif-nvme.rst deleted file mode 100644 index ab4e9b7088bad..0000000000000 --- a/Documentation/driver-api/firmware/aurora-apif-nvme.rst +++ /dev/null @@ -1,39 +0,0 @@ -Resident ANS over APIF -====================== - -``apple,nvme-apif`` selects the native table-6 queue ABI. It keeps the -existing Apple NVMe queue depth, batching, crypto fields, recovery ownership -and synchronous post-M4 flush policy. Linux builds and translates the TCB -and physical page list; Aura forwards selectors and arguments unchanged. -No payload copying or monitor-side NVMe policy is required. - -The boot producer must quiesce RTKit and stop controller DMA before Linux -starts. It supplies ``asahi,rtkit-quiesced`` and four distinct reserved-memory -pages named ``admin-sq``, ``admin-cq``, ``io-sq`` and ``io-cq``. Each is coherent, -16 KiB aligned, 16 KiB sized and ``no-map``. Their guest addresses must map to -the native physical pages already registered with the firmware. Linux clears -and reuses those pages; it does not substitute new queue addresses. They stay -reserved after unbind because the native queue registrations survive it. - -This profile requires an identity DMA aperture. It checks each request's -physical translation against the DMA address in its command and PRP chain. -An offset aperture is rejected before native request registration. Native -metadata addresses are independently translated, including the scratch TCB -and page list. Queue-control and MAP endpoints return void; UNMAP must return -one before a request's backing can be reused. - -A lost MAP response retains that request and prevents new registration until -recovery has stopped the controller and explicitly invalidated every retained -tag. Failed invalidation quarantines the controller and retains its DMA, -requests and APIF client until platform reset. SART unmap failures also retain -Linux-owned backing. These are ownership failures, not successful I/O. - -Live RTKit adoption is deliberately excluded from this new profile: controller -CC.EN/RDY alone cannot prove a live mailbox session or authorize inherited -buffers outside guest RAM. Existing raw-controller profiles keep their existing -validated adoption path. The guest producer must provide the stopped handoff -above; a version-1 live-session DT is not a drop-in replacement. - -Build and host fault tests do not qualify this path on hardware. Boot, -read/write/flush durability, controller reset and teardown must be tested with -the matching Aura version-2 monitor and generated handoff description. diff --git a/Documentation/driver-api/firmware/aurora-apif-pcie.rst b/Documentation/driver-api/firmware/aurora-apif-pcie.rst deleted file mode 100644 index fd366c73ef77f..0000000000000 --- a/Documentation/driver-api/firmware/aurora-apif-pcie.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -Preinitialized PCIe guests -========================= - -``apple,apif-pcie`` admits an ECAM host whose producer has already configured -its clocks, PHY, link, address windows and RID-to-SID routes. The producer -must stop endpoint DMA and MSI before transferring ownership. Linux allocates -MSI vectors from the single native four-cell AIC edge range in ``msi-ranges``; -the doorbell is supplied by ``apple,msi-address``. Vector counts and addresses -are checked before the routing tables are touched. No polling interrupt path -or fixed IRQ assignment per function is introduced. - -A second compatible, ``apple,t6050-pcie-apif``, selects the known port-0 -register layout and supplies port, PHY, PERST and interrupt-to-AXI resources. -It restores explicit ``apple,rid-to-sid`` tuples and the same declared MSI -range after a controlled preboot cycle. Tunables come from the boot producer. -Unknown layouts use only ECAM/MSI; they cannot enter the T6050 reset sequence. - -``apple_apif_pcie_cycle()`` is for the isolated N1 ROM/preboot function before -radio functions are enumerated or have live DMA. The caller must quiesce all -activity on that unit. PCI topology locking excludes sibling functions, and -DART transitions refuse other Linux groups or retained firmware streams. -Timeouts are bounded. A partial transition leaves bus mastering and bridge -decoding stopped; it does not attempt a speculative DMA restart. This is not -a general PCI bus reset API and does not add N1 Wi-Fi/Bluetooth drivers. - -The generic host and its matched port sequence require hardware qualification -with the matching Aura monitor and boot description. Host sequence tests and -native builds do not establish link recovery or radio firmware behavior. diff --git a/Documentation/driver-api/firmware/aurora-apif-pmp.rst b/Documentation/driver-api/firmware/aurora-apif-pmp.rst deleted file mode 100644 index e15e69bf07ee8..0000000000000 --- a/Documentation/driver-api/firmware/aurora-apif-pmp.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -Registry-v2 PMP guests -====================== - -The matched ``apple,t6050-pmp-v2`` firmware ABI uses 88-byte registry -descriptors, firmware-assigned 16-bit IDs, negotiated value buffers, and -update opcode ``0x36``. Linux validates descriptor sizes and names, rejects -duplicate live IDs, and bounds every shared-buffer copy. Boot arguments are -validated completely before any patch is written back. Board and calibration -values come from the boot description; no machine-name policy is embedded. - -The existing T6000 protocol and T8140 42-bit DMA mask remain unchanged. -Registry-v2 configures a 48-bit mailbox DMA address mask and requires an -attached IOMMU domain before starting the processor. Its APIF DART description -must retain the firmware streams, aperture and final-page guard reported by -the producer. A missing provider must not permit direct-DMA fallback. - -Probe accepts only the known cold context marker with the CPU stopped and no -retained DMA context. Unbind and reboot request RTKit idle, verify the matched -context-save marker/version and save counter, then clear CPU_RUN and check -its readback. Firmware allocations cannot be freed merely because RTKit -acknowledged a power state. A failed stop drains callbacks but retains both -shared RTKit backing and the PMP allocation owner until platform reset; it -also leaves the cold-reprobe check unable to admit the retained context. - -The driver owns registry-v2 RTKit outside the callback context, avoiding an -Arc ownership cycle. The notifier is removed and drained before the RTKit -handle or context is released. Probe failure after firmware start follows -the same stop-or-retain ownership path. Endpoint-7 auxiliary buffers are -session-owned; identical requests reuse them rather than replacing live DMA. - -This is support for the inspected firmware ABI, not a generic promise that -later PMP layouts use the same context registers. Hardware cold boot, -registry exchange, idle/save, reboot and unbind qualification remain pending. diff --git a/Documentation/driver-api/firmware/aurora-apif.rst b/Documentation/driver-api/firmware/aurora-apif.rst deleted file mode 100644 index 1622f49dd426e..0000000000000 --- a/Documentation/driver-api/firmware/aurora-apif.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-only - -=============================================== -Aurora Platform Interface over Memory-Mapped I/O -=============================================== - -Aura is Aurora's EL2-resident monitor for Apple Silicon, providing SPTM -mediation and nested virtualization. APIF is a synchronous, trusted-kernel -transport. Linux supplies native selectors and endpoint arguments; Aura -forwards them unchanged. Invalid native calls can stop the machine. This -interface does not isolate an untrusted kernel. - -Wire contract -============= - -The version-2 synthetic register page occupies 16 KiB at guest IPA 0x61ff0000. -Registers are little-endian 64-bit fields: magic at 0x000 is -0x4f494d4d46495041 (APIFMMIO), version at 0x008 is 2, maximum operations at -0x010, and feature flags at 0x018 (bit 0 native, bit 1 memory, bit 2 I/O). -A 64-bit write at 0x100 supplies a 16-byte-aligned guest IPA for the batch. -The entire buffer must fit within one guest RAM bank. The register page is -unmapped at guest stage 2; Aura handles its data aborts. - -A batch has a 16-byte header (u32 count, u32 flags, s32 status, u32 done), -then count 64-byte operations (u64 selector, u64 arg[6], u64 ret). Flags are -zero. Selector upper/lower halves give native table/endpoint (x16); arguments -are native x0-x5 and ret is native x0, unchanged. Transport status does not -interpret native results. Void and boolean endpoints need different caller -handling. A failed Aura service stops the batch; done is the completed prefix. - -Each client owns a physically contiguous buffer and serializes its submissions -with IRQs masked. The transport orders memory around the trapping store and -preserves the Linux stack in x9 across the synchronous redispatch contract. -Device links order supplier lifetime; client revocation drains submissions and -subsequent calls return ENODEV. It is safe to call from atomic context. Provider -and consumers must be described together in the guest DT; this series does not -rewrite existing board DTs or enable the backend on an existing boot. - -Ownership services -================== - -Table 0xffffffff supplies Aura services (other tables are native selectors): - -* 1 TRANSLATE(guest IPA, size): native PA for a contiguous guest range. -* 2 ALLOC_FRAME(owner, address): zeroed 16 KiB native frame, CPU alias removed. -* 3 FREE_FRAME(PA): restore Aura's alias after detachment and retype to type 11. -* 4 MAP_IO(guest IPA, PA, size, readonly): map bounded page-aligned device I/O. -* 5 GUEST_ROOT(): guest stage-2 root PA. -* 6 BOOT_IOMMU(kind, unit, SID, index, output IPA, size): bootstrap snapshot. -* 7 BOOT_TABLE(kind, unit, SID, address, level): existing bootstrap table PA. -* 8 FIND_FRAME(owner, address): registered lent frame PA or zero. -* 9 TAG_FRAME(PA, owner, address): register a unique opaque key after attachment. - -Unused arguments are zero. Frame keys are bookkeeping, not an ownership -transition. The pool is capped at 1024 frames. Never free attached or uncertain -frames. MAP_IO does not roll back a partially successful mapping; a failed -mapping requires recovery before its address range can be reused. Initialize -batch status to EINPROGRESS so a missing completion cannot be mistaken for -success. Lost table-operation completion stops further DART transactions. - -BOOT_IOMMU kind 0 means DART. Index U64_MAX returns root level, run count and -DVA aperture; another index returns a retained mapping. The 64-byte output is -u64 level, count, base, size, pa, ipa, attrs, reserved. An unavailable IPA is -U64_MAX. BOOT_TABLE levels 0-2 return existing bootstrap tables, while U64_MAX -returns the bootstrap initialized flag. Bootstrap queries never replay maps. - -DART and SART providers -======================= - -APPLE_DART_APIF supplies an IOMMU provider for apple,dart-apif. It preserves -stream groups, shared IOVA allocation, retained mappings, bounded batches and -partial-prefix rollback. Linux builds native physical page lists, complements -IOMMU allow bits into native deny bits, and sets up typed tables. Both DMA -backing and list-buffer pointers are translated before native calls. Retained -streams remain reserved and are excluded from ordinary power-down. - -APPLE_SART_APIF supplies apple,sart-apif through the existing apple_sart API. -The raw MMIO provider continues using its current operations. The APIF provider -tracks Linux-owned regions and respects endpoint-specific return semantics; -unknown firmware grants are not advertised as inherited. Neither provider -prints routine successful calls or runs probe-time mapping self-tests. - -The existing arm64 TSO task interface discovers guest ACTLR support through -its toggle/readback probe; APIF does not add a second TSO protocol. The Apple -WFI preservation quirk also covers captured T6050 parts 0x064/0x065. These -parts also avoid unsafe timed waits in arbitrary delay callers. The current -boot-time idle quirk and existing CPU handling are preserved. This does not claim future Ultra support. - -Validation -========== - -Build the providers and run the production-function host tests with:: - - make ARCH=arm64 LLVM=1 Image - make -C tools/testing/selftests/aurora-apif run_tests - -Host tests exercise framing, prefix completion, ownership ordering, list -translation and endpoint return handling. Hardware validation of this cleaned -version-2 integration is pending. GPU backend changes are outside this series; -legacy version-1 Aura aliases remain available to existing clients. diff --git a/Documentation/driver-api/firmware/index.rst b/Documentation/driver-api/firmware/index.rst index ada189185094c..86a3dd4bc3f8a 100644 --- a/Documentation/driver-api/firmware/index.rst +++ b/Documentation/driver-api/firmware/index.rst @@ -4,10 +4,6 @@ Linux Firmware API .. toctree:: - aurora-apif - aurora-apif-nvme - aurora-apif-pcie - aurora-apif-pmp introduction core efi/index diff --git a/MAINTAINERS b/MAINTAINERS index fc20e250d5476..b13c1507cf391 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4566,35 +4566,12 @@ AURORA N1 CENTAURI TRANSPORT M: Dj L: linux-wireless@vger.kernel.org S: Maintained -F: Documentation/arch/arm64/aurora-apif-centauri.rst F: Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml F: Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml F: drivers/bluetooth/btcentauri.c F: include/linux/apple-centauri.h F: drivers/net/wireless/apple/ -AURORA PLATFORM MMIO INTERFACE -M: Dj -S: Maintained -F: Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml -F: Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml -F: Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml -F: Documentation/driver-api/firmware/aurora-apif*.rst -F: Documentation/devicetree/bindings/nvme/apple,nvme-apif.yaml -F: Documentation/devicetree/bindings/pci/apple,apif-pcie.yaml -F: arch/arm64/configs/aurora-apif.config -F: drivers/nvme/host/apple-apif.h -F: drivers/pci/controller/apple-apif/ -F: include/linux/pci-apple-apif.h -F: tools/aurora-apif/ -F: drivers/firmware/aurora-apif-mmio.c -F: drivers/iommu/apple-dart-apif.c -F: drivers/soc/apple/apple-sart-apif.c -F: drivers/soc/apple/sart-backend.h -F: include/linux/apple-dart-apif.h -F: include/linux/soc/aurora/apif-mmio.h -F: tools/testing/selftests/aurora-apif/ - AUXILIARY DISPLAY DRIVERS M: Andy Shevchenko R: Geert Uytterhoeven From 7b7c2377a78424f7f63fb06b652ab0434f5b662e Mon Sep 17 00:00:00 2001 From: AxxzyWasTaken Date: Mon, 9 Nov 2026 18:11:48 +0000 Subject: [PATCH 348/392] drm/asahi: add the T6041 25G83 profile captured on the J616C The 32-core M4 Max in the J616C boots through a macOS 26.6.2 stub (iBoot mBoot-18000.161.10) and loads the T6041 25G83 image that "drm/asahi: run the 25G83 build on the 40-core M4 Max" admits. That build runs the 40-core configuration, whose core check refuses this GPU. Admit the image a second time, with a 32-core configuration (hw/t6041_25g83.rs). When builds share an image, the selector now takes the one whose configuration matches the probed clusters and enabled cores, else the first one, so a 40-core part runs as before and another layout is still refused by the core check. The GPU ID is read before the build is selected for this. T6040 and T8140 are unchanged. The configuration: - 4 clusters with 32 enabled cores. Operand-state banks for all 40 core slots: the enabled slots run to 39 (core mask 0xf6_fdbf6fdb), and with banks for the 32 enabled cores only, a compute load beside rendering hit reason-0 recoveries in 5 of 5 runs, and in 0 of 4 with 40 banks. Four-cluster deflake and TA mode 1, as the 40-core 25G83 build. - A compute pool of 96 blocks, which the J616C has run with since 23 Oct 2026; 21 blocks hang its firmware on grids of more than ten workgroups. - The 40-core 25G83 register map with six memory controllers in slot 3 (ADT active-mcc-bit-vector 0x3f), laid out by the same rule. A compile-time check holds it to that rule and to the 40-core windows. What varies between units comes from the device tree, as on the 40-core part: the boot loader fills the operating points with one voltage per cluster (from the ADT perf-states tables) and the CS and AFR tables (from cs-perf-states and afr-perf-states), and the 25G83 builders turn them into the performance, CS and AFR tables of the hardware data. The previous version of this change applied contents captured on this unit, binned operating points included, because its device tree had no GPU tables at the time. The rest is static, because the driver has no source for it on T6041: - The selector ladders: the highest and lowest of this unit's operating points at each of its 11 core voltages (state 5 of ladder B is OPP 5, where the 40-core profile has OPP 6), and the M4 Pro's relative ladders. - The power limit, 110586 mW (the M4 Pro's 69116 mW for 20 cores, scaled to 32), and the M4 Pro's leakage coefficients for four clusters. Neither is in the ADT, and neither is a measurement of this unit. - The AON rows of the four GPU clusters and the SoC, from the ADT's aon-ptd ptd_data_table, which the loader does not publish. - Five power configuration words: -1000 / power limit at +0x9ec, the second temperature-sensor mask word at +0x3b5c, the temperature sources of clusters 2 and 3 at +0x6328 and +0x632c, and 5 at +0x6360. Built on the host from the J616C's ADT operating points, the hardware data, power configuration and region C hash (FNV-1a 64 without the runtime words) to those of the contents this unit has run with since October 2026, dumped on 25 Oct 2026. Those contents, and so the static values above, were built by the M4 GPU encoders of Gravity Linux (g16_fw.rs, g16_platform.rs, g16_power.rs), with Jonathan Goyvaerts' T6040 values. Based on work by Cody Ho, who wrote the encoders (GravityLinux bd25d194b0a2, cherry-picked from 778088823509, which carries no Signed-off-by), and Niklas Sheth, who added the bootloader power limit (GravityLinux 3a2ad760d731). Not run on hardware, and not built in a kernel tree: the host tests compile the build selector, the configuration's constants with the 25G83 builders, and its compile-time checks. Link: https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 Link: https://github.com/GravityLinux/linux/commit/3a2ad760d7316813f37c0ddc653ef783827f855b Link: https://github.com/aurora-silicon/linux/pull/1050 Co-developed-by: Niklas Sheth Signed-off-by: Niklas Sheth Signed-off-by: AxxzyWasTaken --- drivers/gpu/drm/asahi/fwid.rs | 13 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 11 +- drivers/gpu/drm/asahi/g17/mod.rs | 34 +++- drivers/gpu/drm/asahi/hw/mod.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 26 ++- drivers/gpu/drm/asahi/hw/t6041_25g83.rs | 210 ++++++++++++++++++++++++ 6 files changed, 275 insertions(+), 20 deletions(-) create mode 100644 drivers/gpu/drm/asahi/hw/t6041_25g83.rs diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 514f09539dc73..2aeca708359d8 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -254,8 +254,13 @@ mod tests { else { panic!("T6041 must identify its loaded firmware"); }; - assert_eq!(builds.len(), 2); - for (i, build) in builds.iter().enumerate() { + assert_eq!(builds.len(), 3); + // The last two admit the 25G83 images, with the 40- and the 32-core configuration. + assert_eq!(builds[1].images.len(), builds[2].images.len()); + for (a, b) in builds[1].images.iter().zip(builds[2].images) { + assert_eq!((a.entropy, a.text_sha256), (b.entropy, b.text_sha256)); + } + for (i, build) in builds[..2].iter().enumerate() { let other = &builds[1 - i]; for image in build.images { let (entropy, digest) = (image.entropy, image.text_sha256); @@ -277,6 +282,10 @@ mod tests { let cores = crate::hw::t6041::CONFIG.core_layout.cores; assert_eq!(g83.core_layout.cores, cores); assert_eq!(cores, Some((4, 40))); + let g83_32 = builds[2].config.unwrap(); + assert_eq!(g83_32.core_layout.cores, Some((4, 32))); + assert!(g83_32.reserved_slots.is_empty()); + assert_eq!(g83_32.windows.len(), map.len()); } #[cfg(CONFIG_DRM_ASAHI_G16)] diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index f63821c904fd2..88828bfefb25b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1629,13 +1629,18 @@ mod ta_mode_tests { } } - /// The variants of the builds T6041 admits: 25F84, then 25G83. + /// The variants of the builds T6041 admits: 25F84, then 25G83, which the 40- and the + /// 32-core configurations run with the same rules. fn t6041_variants() -> [&'static Variant; 2] { let Firmware::Identified(builds) = crate::hw::t6041::CONFIG.firmware else { panic!("T6041 must identify its loaded firmware"); }; - assert_eq!(builds.len(), 2); - [builds[0].variant, builds[1].variant] + assert_eq!(builds.len(), 3); + let (g83, g83_32) = (builds[1].variant, builds[2].variant); + assert_eq!(g83.ta_mode_1, g83_32.ta_mode_1); + assert_eq!(g83.compute, g83_32.compute); + assert_eq!(g83.programs, g83_32.programs); + [builds[0].variant, g83] } /// The TA mode of every admitted T6041 build. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index df5a31f621a3c..a13ff85529eb2 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1173,10 +1173,12 @@ impl Gpu { } else { None }; - let (variant, build_cfg) = Self::firmware_variant(dev, &cfg.firmware)?; + let id = regs.get_g17_id()?; + let active_cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let (variant, build_cfg) = + Self::firmware_variant(dev, &cfg.firmware, (id.num_clusters, active_cores))?; // A build with its own configuration of the SoC runs with that one. let cfg = build_cfg.unwrap_or(cfg); - let id = regs.get_g17_id()?; if let Some(calibration) = calibration { calibration.check_cores(&id.core_masks)?; calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; @@ -1216,7 +1218,6 @@ impl Gpu { let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; - let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { clusters == id.num_clusters && cores != active_cores @@ -1329,9 +1330,15 @@ impl Gpu { /// Returns the variant of the loaded firmware build and the configuration it runs with, if /// it has its own. An identified SoC refuses a build it does not know, before anything is /// written to the GPU. + /// + /// Builds can admit the same image with configurations measured on different core layouts. + /// The first identified build whose configuration admits `cores` (clusters and enabled + /// cores, as probed) is chosen, else the first identified one, whose core check then + /// refuses the GPU. fn firmware_variant( dev: &AsahiDrmDevice, firmware: &fw::variant::Firmware, + cores: (u32, u32), ) -> Result<(&'static fw::Variant, Option<&'static Config>)> { let builds = match firmware { fw::variant::Firmware::Fixed(variant) => return Ok((variant, None)), @@ -1346,11 +1353,26 @@ impl Gpu { let others = builds .iter() .filter(|build| Some(build.os_version) != version); + let mut identified = None; for build in Iterator::chain(preferred, others) { - if Self::identify_image(dev, build).is_ok() { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok((build.variant, build.config)); + if Self::identify_image(dev, build).is_err() { + continue; } + let admits = build.config.is_none_or(|config| { + config + .core_layout + .cores + .is_none_or(|measured| measured == cores) + }); + if admits { + identified = Some(build); + break; + } + identified.get_or_insert(build); + } + if let Some(build) = identified { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok((build.variant, build.config)); } dev_err!( dev.as_ref(), diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 438d071de971c..bf1fd92822ced 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -15,6 +15,7 @@ pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t6040; pub(crate) mod t6041; +mod t6041_25g83; mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 956aa0cc7728b..04ddfacd53cec 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -9,9 +9,9 @@ use crate::hw::{GpuGen, GpuVariant}; use super::t604x::{instances, window}; use crate::mmu::UatQuirks; -/// What the configurations of both firmware builds share: the 40-core M4 Max measured with -/// 25F84. Only the register map is the build's own. -const BASE: Config = Config { +/// What the configurations of the firmware builds share: the 40-core M4 Max measured with +/// 25F84. The 40-core 25G83 configuration only has its own register map. +pub(super) const BASE: Config = Config { chip_id: CHIP_ID, identity: Identity { family: 0xa, @@ -58,18 +58,26 @@ const BASE: Config = Config { }, }; -/// The 40-core M4 Max. 25F84 runs this configuration; 25G83 runs [`CONFIG_25G83`]. +/// The 40-core M4 Max. 25F84 runs this configuration; 25G83 runs [`CONFIG_25G83`], or +/// [`super::t6041_25g83::CONFIG`] on a 32-core part. pub(crate) const CONFIG: Config = Config { firmware: fw::variant::Firmware::Identified(&BUILDS), ..BASE }; -const BUILDS: [fw::variant::Build; 2] = [ +/// The 25G83 image is admitted twice, with the configuration of each measured core layout; +/// the one that matches the probed cores is used. +const BUILDS: [fw::variant::Build; 3] = [ BUILD_25F84, fw::variant::Build { config: Some(&CONFIG_25G83), ..BUILD_25G83 }, + fw::variant::Build { + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10), 32 cores", + config: Some(&super::t6041_25g83::CONFIG), + ..BUILD_25G83 + }, ]; const BUILD_25F84: fw::variant::Build = fw::variant::Build { @@ -96,9 +104,9 @@ const BUILD_25F84: fw::variant::Build = fw::variant::Build { /// The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 build the M4 Pro /// runs. iBoot fills part of the loaded TEXT for each Mac, so each admitted image is a -/// measurement. The one listed is the only T6041 image measured so far, on a 32-core J616C -/// (linux#1050); there the core count check of [`CONFIG_25G83`] refuses the GPU. -const BUILD_25G83: fw::variant::Build = fw::variant::Build { +/// measurement. The one listed is the only T6041 image measured so far, on a 32-core J616C, +/// which runs it with [`super::t6041_25g83::CONFIG`]. +pub(super) const BUILD_25G83: fw::variant::Build = fw::variant::Build { os_version: [26, 6, 2], name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", images: &[crate::fwid::FirmwareBuild { @@ -252,7 +260,7 @@ const _: () = { const MAX_POWER_MW: u32 = 129_702; -const REGION_C: RegionCConfig = RegionCConfig { +pub(super) const REGION_C: RegionCConfig = RegionCConfig { unk_54: 0x0fa0, unk_78: [1, 0, 0], max_power_mw: MAX_POWER_MW, diff --git a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs new file mode 100644 index 0000000000000..370561d906bf2 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// +// The power configuration words, leakage coefficients, power limit and AON rows of this +// profile are what the M4 GPU encoders of Gravity Linux (g16_fw.rs, g16_platform.rs and +// g16_power.rs, written by Cody Ho, with the bootloader power limit added by Niklas Sheth) +// built on a 32-core J616C: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// https://github.com/GravityLinux/linux/commit/3a2ad760d7316813f37c0ddc653ef783827f855b + +//! T6041 (M4 Max) with 32 cores on the 25G83 firmware build, as the J616C runs it. +//! +//! The J616C boots through a macOS 26.6.2 stub (iBoot mBoot-18000.161.10), which loads the +//! 25G83 image [`t6041::BUILD_25G83`] admits. Its GPU has 32 of 40 cores in four clusters, so +//! the 40-core configuration of that build refuses it; this one is selected instead. +//! +//! What differs between units comes from the device tree, which the boot loader fills from the +//! ADT: the operating points with one voltage per cluster (`operating-points-v2`, from the +//! ADT `perf-states` tables) and the CS and AFR tables (`apple,cs-opp`, `apple,afr-opp`, from +//! `cs-perf-states` and `afr-perf-states`). The builders turn them into the performance, CS and +//! AFR tables of the hardware data. The enabled cores are probed from the GPU. +//! +//! Everything else is static, because the driver has no source for it on T6041: +//! - [`PERF`]: the selector ladders, see there. +//! - [`MAX_POWER_MW`] and the leakage coefficients of [`CHIP`]: not in the ADT. They are the +//! M4 Pro's values (its per-core power budget for 32 cores, its per-cluster coefficients for +//! four clusters), not values of this unit. +//! - [`AON_TABLE`]: the GPU cluster and SoC rows of the ADT's `/arm-io/aon-ptd` +//! `ptd_data_table`, which the loader does not publish. +//! - [`REGISTER_WINDOWS`]: slot 3 holds this part's six memory controllers (ADT +//! `/arm-io/sgx` `active-mcc-bit-vector` 0x3f). +//! +//! Built from the device tree of the J616C, the hardware data and region C equal what this +//! unit ran with since October 2026, and the power configuration does with [`CHIP`]'s words. + +use super::t6041; +use super::t604x::{instances, window}; +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout}; + +pub(super) const CONFIG: Config = Config { + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + region_c: ®ION_C, + chip: Some(&CHIP), + firmware: fw::variant::Firmware::Identified(&[t6041::BUILD_25G83]), + core_layout: CoreLayout { + cores: Some((4, 32)), + alternate_core_masks: None, + // Banks for all forty core slots: the firmware indexes them by physical slot, and the + // J616C's enabled slots run to 39 (core mask 0xf6_fdbf6fdb). With banks for the 32 + // enabled cores only, a compute load beside rendering hit reason-0 recoveries. + operand_state: fw::compute::OperandState::FOUR_BY_TEN, + deflake: fw::render::Deflake::FOUR_CLUSTERS, + // The compute pool the J616C has run with since 23 Oct 2026; the M4 Pro's 21 blocks + // hang this GPU's firmware on grids of more than ten workgroups. + compute_blocks: 96, + }, + ..t6041::BASE +}; + +/// The register map of [`t6041::REGISTER_WINDOWS_25G83`] with six memory controller +/// instances in slot 3 instead of eight, laid out by the same rule, so every window after it +/// lies 0x30000 lower. +const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances( + window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), + 6, + 0x200_0000, + ), + window(4, 0x5_0244_c000, 0x29_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x2a_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x2a_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x2b_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x2b_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x2c_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2d_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2e_0000, 0x2_1400, 0, 2), + instances( + window(20, 0x2_8b00_0000, 0x30_8000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), + window(22, 0x2_8900_0000, 0x40_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x41_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x42_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x43_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x43_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x44_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x46_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x46_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x47_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x47_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x48_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x49_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x49_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x4a_4000, 0x58, 0xe6_0000, 2), +]; + +// The windows of the 40-core 25G83 map, with six controllers, laid out by its rule. +const _: () = { + assert!(super::t604x::laid_out_25g83(®ISTER_WINDOWS)); + let mut index = 0; + while index < REGISTER_WINDOWS.len() { + let (max, ours) = ( + &t6041::REGISTER_WINDOWS_25G83[index], + ®ISTER_WINDOWS[index], + ); + assert!( + max.slot == ours.slot + && max.phys == ours.phys + && max.size == ours.size + && max.unk_18 == ours.unk_18 + && max.unk_20 == ours.unk_20 + && max.phys_stride == ours.phys_stride + && (max.instances == ours.instances || (max.slot == 3 && ours.instances == 6)) + ); + index += 1; + } +}; + +/// The selector ladders. `states_a` and `states_b` are the highest and lowest of the J616C's +/// operating points at each of its 11 core voltages, in ADT order; the 40-core 25F84 profile +/// selects OPP 6 for state 5 of `states_b`, this one OPP 5. The relative ladders are the M4 +/// Pro's. The driver has no source for them on T6041. +const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < t6041::OPPS); + assert!((PERF.states_b[index] as usize) < t6041::OPPS); + index += 1; + } +}; + +/// The M4 Pro's power budget per core (69116 mW for 20) for 32 cores. +const MAX_POWER_MW: u32 = 110_586; + +const REGION_C: RegionCConfig = RegionCConfig { + max_power_mw: MAX_POWER_MW, + ..t6041::REGION_C +}; + +/// The AON temperature-source rows (12 bytes each) the power configuration names: GPU clusters +/// 0 to 3 and the SoC, as in the ADT's `ptd_data_table`. +const AON_TABLE: [u32; 15] = [ + u32::from_be_bytes(*b"g0TR"), + 0x00c2_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x0097_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g2TR"), + 0x0092_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g3TR"), + 0x00c7_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x012c_0001, + 0x0000_1401, +]; + +const CHIP: v25g83::Chip = v25g83::Chip { + max_power_mw: MAX_POWER_MW, + // The M4 Pro's per-cluster core coefficients for four clusters, its SRAM, CS and AFR ones. + leakage: { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; + words[1] = 0x44f3_0000; + words[2] = 0x44f0_3000; + words[3] = 0x44f3_0000; + words[32] = 0x4373_0000; + words[33] = 0x4373_0000; + words[34] = 0x4373_0000; + words[35] = 0x4373_0000; + words[52] = 0x437b_0000; + words[54] = 0x43bd_c000; + words + }, + aon_table: &AON_TABLE, + hw_data_words: &[], + power_config_words: POWER_CONFIG_WORDS, + perf_a_replicate_lanes: false, + cs_perf_table: true, +}; + +/// What differs from the M4 Pro's power configuration besides the power limit and leakage. +const POWER_CONFIG_WORDS: &[(usize, u32, usize)] = &[ + // -1000 / MAX_POWER_MW, as the M4 Pro's word is for its 69116 mW. + (0x9ec, 0xbc14_27fc, 1), + // The second temperature-sensor mask word. + (0x3b5c, 0x2a22_2aa2, 1), + // The temperature sources of GPU clusters 2 and 3. + (0x6328, u32::from_be_bytes(*b"g2TR"), 1), + (0x632c, u32::from_be_bytes(*b"g3TR"), 1), + (0x6360, 0x5, 1), +]; From 226a65c7dc5249927f03abe65d80382776dc08bb Mon Sep 17 00:00:00 2001 From: AxxzyWasTaken Date: Mon, 9 Nov 2026 18:11:48 +0000 Subject: [PATCH 349/392] tools/asahi: test the 32-core T6041 25G83 configuration on the host The build selector now takes the probed core layout. Model it, and check that of two builds admitting one image, the configuration of the probed layout is chosen, that another layout gets the first one, and that the layout never changes which image is admitted. Build the 32-core T6041 25G83 objects from the operating points the loader publishes from the J616C's ADT (perf-states for four clusters, cs-perf-states and afr-perf-states), and check that the hardware data, power configuration and region C hash (FNV-1a 64, without the runtime words) to those of the contents dumped on that unit on 25 Oct 2026, and that the word at +0x9ec is -1000 / the power limit. Signed-off-by: AxxzyWasTaken --- tools/asahi/g17-firmware-selection-tests.py | 49 +++++++++++++++---- tools/asahi/g17-t6041-initdata-tests.py | 53 ++++++++++++++++++++- 2 files changed, 91 insertions(+), 11 deletions(-) diff --git a/tools/asahi/g17-firmware-selection-tests.py b/tools/asahi/g17-firmware-selection-tests.py index 2a3cae91fdb08..6b9cc52d70ea7 100644 --- a/tools/asahi/g17-firmware-selection-tests.py +++ b/tools/asahi/g17-firmware-selection-tests.py @@ -34,7 +34,13 @@ def method(source, signature): image: u32, checked: std::cell::RefCell>, } -struct Config(u32); +struct CoreLayout { + cores: Option<(u32, u32)>, +} +struct Config { + id: u32, + core_layout: CoreLayout, +} mod fw { pub struct Variant(pub u32); pub mod variant { @@ -62,12 +68,23 @@ def method(source, signature): } static FIRST: fw::Variant = fw::Variant(1); static SECOND: fw::Variant = fw::Variant(2); -static OWN: Config = Config(7); +static OWN: Config = Config { id: 7, core_layout: CoreLayout { cores: None } }; static BUILDS: [fw::variant::Build; 2] = [ fw::variant::Build { os_version: [26,6,2], name: "first", image: 1, variant: &FIRST, config: None }, fw::variant::Build { os_version: [27,0,0], name: "second", image: 2, variant: &SECOND, config: Some(&OWN) }, ]; +// One image admitted with the configurations of two core layouts, as T6041 25G83 is. +static FORTY: Config = Config { id: 40, core_layout: CoreLayout { cores: Some((4, 40)) } }; +static THIRTY_TWO: Config = Config { id: 32, core_layout: CoreLayout { cores: Some((4, 32)) } }; +static SHARED: [fw::variant::Build; 3] = [ + fw::variant::Build { os_version: [26,5,2], name: "other", image: 1, variant: &FIRST, config: None }, + fw::variant::Build { os_version: [26,6,2], name: "40", image: 2, variant: &SECOND, + config: Some(&FORTY) }, + fw::variant::Build { os_version: [26,6,2], name: "32", image: 2, variant: &SECOND, + config: Some(&THIRTY_TWO) }, +]; +const CORES: (u32, u32) = (4, 40); fn device(version: Option<[u32; 3]>, image: u32) -> AsahiDrmDevice { AsahiDrmDevice { version, image, checked: std::cell::RefCell::new(Vec::new()) } } @@ -75,26 +92,40 @@ def method(source, signature): use fw::variant::Firmware; for version in [None, Some([26,6,2]), Some([26,7,0]), Some([99,0,0])] { let dev = device(version, 1); - let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap(); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS), CORES).unwrap(); assert_eq!(variant.0, 1); assert!(config.is_none(), "a build without its own configuration keeps the probed one"); } let dev = device(Some([27,0,0]), 1); - assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap().0.0, 1); + assert_eq!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS), CORES).unwrap().0.0, 1); assert_eq!(*dev.checked.borrow(), [2,1], "the OS version only orders the candidates"); let dev = device(None, 2); - let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).unwrap(); - assert_eq!((variant.0, config.map(|c| c.0)), (2, Some(7)), "the identified build's configuration"); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS), CORES).unwrap(); + assert_eq!((variant.0, config.map(|c| c.id)), (2, Some(7)), "the identified build's configuration"); let dev = device(Some([26,6,2]), 9); - assert!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS)).is_err()); + assert!(Gpu::firmware_variant(&dev, &Firmware::Identified(&BUILDS), CORES).is_err()); assert_eq!(*dev.checked.borrow(), [1,2]); let dev = device(None, 9); - let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Fixed(&SECOND)).unwrap(); + let (variant, config) = Gpu::firmware_variant(&dev, &Firmware::Fixed(&SECOND), CORES).unwrap(); assert_eq!(variant.0, 2); assert!(config.is_none()); assert!(dev.checked.borrow().is_empty(), "fixed Neo admission is unchanged"); + for (cores, expected) in [((4, 40), 40), ((4, 32), 32), ((4, 36), 40)] { + for version in [None, Some([26,6,2])] { + let dev = device(version, 2); + let (variant, config) = + Gpu::firmware_variant(&dev, &Firmware::Identified(&SHARED), cores).unwrap(); + assert_eq!((variant.0, config.map(|c| c.id)), (2, Some(expected)), + "the configuration of the probed core layout, else the first one"); + } + } + let dev = device(None, 1); + let (variant, config) = + Gpu::firmware_variant(&dev, &Firmware::Identified(&SHARED), (4, 32)).unwrap(); + assert_eq!(variant.0, 1); + assert!(config.is_none(), "the core layout does not change which image is admitted"); println!("PASS G17 selection: exact image required, OS hint optional, build configuration \ - follows the image, fixed Neo unchanged"); + follows the image and the probed core layout, fixed Neo unchanged"); } ''' diff --git a/tools/asahi/g17-t6041-initdata-tests.py b/tools/asahi/g17-t6041-initdata-tests.py index a48174df48547..f243c34f4b294 100644 --- a/tools/asahi/g17-t6041-initdata-tests.py +++ b/tools/asahi/g17-t6041-initdata-tests.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Build T6041 initialization data and compare it with captures when given.""" +"""Build T6041 initialization data and compare it with captures when given. + +The 32-core 25G83 configuration is also built from the operating points the loader publishes +from the J616C's ADT, and must give the contents that unit ran with.""" import argparse import re @@ -110,6 +113,16 @@ def item(source, start): "const POWER_CONFIG_WORDS", ] +J616C_ITEMS = [ + "const REGISTER_WINDOWS: [", + "const PERF: PerfConfig", + "const MAX_POWER_MW", + "const REGION_C", + "const AON_TABLE", + "const CHIP: ", + "const POWER_CONFIG_WORDS", +] + T6041_OPP_HZ = [0, 338, 618, 796, 924, 952, 1056, 1062, 1182, 1182, 1312, 1242, 1380, 1326, 1470, 1578] T6041_OPP_MV = [ @@ -121,6 +134,19 @@ def item(source, start): T6041_AUX = [(24, 585000), (338, 585000), (618, 635000), (796, 685000), (992, 720000), (1188, 770000), (1380, 855000), (1470, 885000)] +# The 32-core J616C's ADT /arm-io/sgx: perf-states (one table per cluster) and cs-perf-states +# (afr-perf-states is the same), as the loader publishes them in the device tree. +J616C_OPP_HZ = [0, 338, 618, 796, 924, 952, 1056, 1062, 1182, 1182, 1312, 1242, 1380, 1326, + 1470, 1578] +J616C_OPP_MV = [ + [125, 600, 650, 695, 720, 770, 770, 815, 815, 870, 870, 910, 910, 950, 950, 1030], + [125, 595, 645, 690, 720, 760, 760, 805, 805, 850, 850, 890, 890, 930, 930, 1025], + [125, 615, 660, 705, 725, 775, 775, 805, 805, 855, 855, 895, 895, 935, 935, 1030], + [125, 600, 660, 700, 715, 770, 770, 805, 805, 860, 860, 900, 900, 940, 940, 1020], +] +J616C_AUX = [(24, 585000), (338, 585000), (618, 630000), (796, 680000), (992, 720000), + (1188, 770000), (1380, 865000), (1470, 900000)] + def opps(name, hz, mv): rows = [] @@ -205,6 +231,26 @@ def aux(name, rows): .filter(|(i, (x, y))| x != y && !map.contains(i)).count(); assert_eq!(outside(&hw_g83, &hw), 0, "the 25G83 hardware data differs only in its register map"); std::fs::write(format!("{}/t6041-25g83-hwdata.bin", out), &hw_g83).unwrap(); + // The 32-core J616C on 25G83, from its operating points. What it ran with since October + // 2026, hashed without the runtime words (FNV-1a 64), from a dump of 25 Oct 2026. + let (hw32, power32, region32) = build(t6041::CHIP_ID, &t6041_25g83::REGISTER_WINDOWS, &[], + &t6041_25g83::PERF, &J616C_OPPS, &J616C_AUX, &t6041_25g83::CHIP, + &t6041_25g83::REGION_C, 32); + println!("T6041 25G83 32-core initialization objects built"); + let fnv = |bytes: &[u8], skip: &[(usize, usize)]| { + bytes.iter().enumerate().filter(|(i, _)| !skip.iter().any(|&(at, n)| (at..at + n).contains(i))) + .fold(0xcbf2_9ce4_8422_2325u64, |h, (_, &b)| (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3)) + }; + assert_eq!(fnv(&hw32, &[(0x2620, 0x80), (0x26b8, 8)]), 0xd781_973f_9983_1e35, + "the J616C hardware data"); + assert_eq!(fnv(&power32, &[(0x3b50, 8), (0x6318, 8), (0x63d4, 4)]), 0xc36d_7dba_a597_aac4, + "the J616C power configuration"); + assert_eq!(fnv(®ion32, &[]), 0x91a4_c670_17e3_0a24, "the J616C region C"); + let word = |at: usize| u32::from_le_bytes(power32[at..at + 4].try_into().unwrap()); + assert_eq!(word(0x9ec), (-1000.0 / t6041_25g83::MAX_POWER_MW as f32).to_bits()); + println!("T6041 25G83 32-core objects match the J616C's"); + std::fs::write(format!("{}/t6041-25g83-32-hwdata.bin", out), &hw32).unwrap(); + std::fs::write(format!("{}/t6041-25g83-32-power-config.bin", out), &power32).unwrap(); } """ @@ -222,12 +268,15 @@ def program(): common = (DRIVER / "hw/t604x.rs").read_text().replace("use crate::g17::fw::initdata::RegisterWindow;", "use super::RegisterWindow;") code.append("mod t604x {\n%s\n}\n" % common) code.append("pub mod v25g83 {\n%s\n}\n" % v25g83) - for name, source, items in (("t6041", t6041, T6041_ITEMS),): + j616c = (DRIVER / "hw/t6041_25g83.rs").read_text() + for name, source, items in (("t6041", t6041, T6041_ITEMS), ("t6041_25g83", j616c, J616C_ITEMS)): body = "\n".join(item(source, i) for i in items) body = re.sub(r"(?m)^const ", "pub(crate) const ", body) code.append("pub mod %s {\nuse super::*;\nuse super::t604x::{instances, window};\n%s\n}\n" % (name, body)) code.append(opps("T6041_OPPS", T6041_OPP_HZ, T6041_OPP_MV)) code.append(aux("T6041_AUX", T6041_AUX)) + code.append(opps("J616C_OPPS", J616C_OPP_HZ, J616C_OPP_MV)) + code.append(aux("J616C_AUX", J616C_AUX)) code.append(MAIN) return "\n".join(code) From 0bcf5e810ab60d0868ee3e96fdce013b47587736 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:59:04 +1100 Subject: [PATCH 350/392] soc: apple: add probe-only S8000 AES diagnostics (#1328) * dt-bindings: crypto: describe Apple S8000-family AES engines Describe the two- and three-aperture layouts, per-SoC compatibles, AES power domain and SIO DART streams. Keep key-disable register layout selection tied to the SoC compatible. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> * soc: apple: add probe-only SIO AES diagnostics Read the readiness and hardware-key disable words once under runtime PM, then expose the cached snapshot in a root-readable debugfs file. Report the revision from compatible data and leave unverified bitfields undecoded. Do not enable the engine, request interrupts, access the command FIFO or DKEY aperture, allocate DMA buffers, or register a crypto or userspace device interface. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> * arm64: dts: apple: describe disabled SIO AES engines Add AES nodes for T6000, T8112, T6041 and T8140 with named apertures, power domains and SIO DART stream 1. Describe the missing T8140 SIO DART provider as disabled as well. All new nodes remain opt-in. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> * soc: apple: use existing AES register resource order Map engine and key-disable resources by index as ordered by apple,aes.yaml so reg-names are not required. Continue to leave the optional DKEY bank and IRQ untouched, and retain the SoC-specific matches and probe-only read path. Update the module author address. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --------- Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/soc/apple/Kconfig | 14 +++ drivers/soc/apple/Makefile | 2 + drivers/soc/apple/apple-sio-aes.c | 194 ++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 drivers/soc/apple/apple-sio-aes.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 113e09aea5c47..9001a709fb6e7 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,20 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_SIO_AES + tristate "Apple SIO AES probe-time diagnostics" + depends on OF && HAS_IOMEM && PM && IOMMU_API + help + Capture the S8000-family AES readiness and hardware-key disable + state once at probe on T6000, T8112, T6041 and T8140. The snapshot + is logged and available in a root-readable debugfs state file. + Revision is reported from the SoC compatible; register words are + raw diagnostics, not a claim that a hardware key is usable. + + This does not enable the engine, submit commands, handle interrupts, + register crypto algorithms or provide a device node. Keep the DT + node disabled except during an explicitly enabled diagnostic boot. + config APPLE_DBGVIEW tristate "Apple SoC read-only debug register views" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 5ef85752f160d..34ede633d2165 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_SIO_AES) += apple-sio-aes.o + obj-$(CONFIG_APPLE_DBGVIEW) += apple-dbgview.o apple-dbgview-y = dbgview.o diff --git a/drivers/soc/apple/apple-sio-aes.c b/drivers/soc/apple/apple-sio-aes.c new file mode 100644 index 0000000000000..4432f2976ee77 --- /dev/null +++ b/drivers/soc/apple/apple-sio-aes.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Probe-time diagnostics for the Apple S8000-family SIO AES engine. + * Only the readiness word and key-disable state are read. In particular, + * do not initialise the engine, acknowledge interrupts or touch the FIFO. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_SIO_AES_READINESS 0x0c + +struct apple_sio_aes_data { + u32 revision; + /* Revision 3 has a shared word; revision 5 has per-slot words. */ + bool shared_disable; + u32 uid_offset; + u32 gid0_offset; + u32 gid1_offset; +}; + +struct apple_sio_aes { + const struct apple_sio_aes_data *data; + u32 readiness; + u32 disable; + u32 uid_disable; + u32 gid0_disable; + u32 gid1_disable; +}; + +static int apple_sio_aes_state_show(struct seq_file *s, void *unused) +{ + const struct apple_sio_aes *aes = s->private; + + seq_puts(s, "snapshot: probe\n"); + seq_printf(s, "revision_from_compatible: %u\n", aes->data->revision); + seq_printf(s, "readiness_raw: %#010x\n", aes->readiness); + seq_printf(s, "disable_raw: %#010x\n", aes->disable); + if (aes->data->shared_disable) { + seq_puts(s, "uid_gid_disable: shared word, slot bits undecoded\n"); + } else { + seq_printf(s, "uid_disable_raw: %#010x\n", aes->uid_disable); + seq_printf(s, "gid0_disable_raw: %#010x\n", aes->gid0_disable); + seq_printf(s, "gid1_disable_raw: %#010x\n", aes->gid1_disable); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_sio_aes_state); + +static void apple_sio_aes_debugfs_remove(void *dir) +{ + debugfs_remove_recursive(dir); +} + +static void __iomem *apple_sio_aes_map(struct platform_device *pdev, + unsigned int index, resource_size_t min_size) +{ + struct resource *res; + + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res || resource_size(res) < min_size) + return IOMEM_ERR_PTR(-EINVAL); + + return devm_ioremap_resource(&pdev->dev, res); +} + +static int apple_sio_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct apple_sio_aes_data *data; + void __iomem *engine, *disable; + struct apple_sio_aes *aes; + struct dentry *dir; + char *name; + int ret; + + data = of_device_get_match_data(dev); + if (!data) + return -ENODEV; + + /* The platform bus attaches the single power domain and SIO DART. */ + if (!dev->pm_domain || !of_property_present(dev->of_node, "power-domains")) + return dev_err_probe(dev, -EINVAL, "Missing AES power domain\n"); + if (!of_property_present(dev->of_node, "iommus") || + !iommu_get_domain_for_dev(dev)) + return dev_err_probe(dev, -ENODEV, "Missing SIO DART domain\n"); + + aes = devm_kzalloc(dev, sizeof(*aes), GFP_KERNEL); + if (!aes) + return -ENOMEM; + aes->data = data; + + /* apple,aes orders the banks as engine, key-disable, then optional DKEY. */ + engine = apple_sio_aes_map(pdev, 0, APPLE_SIO_AES_READINESS + 4); + if (IS_ERR(engine)) + return dev_err_probe(dev, PTR_ERR(engine), "Invalid engine aperture\n"); + disable = apple_sio_aes_map(pdev, 1, data->gid1_offset + 4); + if (IS_ERR(disable)) + return dev_err_probe(dev, PTR_ERR(disable), "Invalid key-disable aperture\n"); + + /* The DKEY aperture and IRQ are described by DT but never accessed. */ + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to power AES for snapshot\n"); + + /* No polling or decoding of unverified readiness/disable bitfields. */ + aes->readiness = readl(engine + APPLE_SIO_AES_READINESS); + aes->disable = readl(disable); + if (!data->shared_disable) { + aes->uid_disable = readl(disable + data->uid_offset); + aes->gid0_disable = readl(disable + data->gid0_offset); + aes->gid1_disable = readl(disable + data->gid1_offset); + } + + ret = pm_runtime_put_sync_suspend(dev); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to release AES power\n"); + + if (data->shared_disable) + dev_info(dev, + "probe snapshot: revision %u (compatible), readiness %#010x, shared UID/GID disable %#010x (undecoded)\n", + data->revision, aes->readiness, aes->disable); + else + dev_info(dev, + "probe snapshot: revision %u (compatible), readiness %#010x, disable %#010x, UID %#010x, GID0 %#010x, GID1 %#010x (raw)\n", + data->revision, aes->readiness, aes->disable, + aes->uid_disable, aes->gid0_disable, aes->gid1_disable); + + name = devm_kasprintf(dev, GFP_KERNEL, "apple-sio-aes-%s", dev_name(dev)); + if (!name) + return -ENOMEM; + dir = debugfs_create_dir(name, NULL); + if (IS_ERR(dir)) + return 0; + ret = devm_add_action_or_reset(dev, apple_sio_aes_debugfs_remove, dir); + if (ret) + return ret; + debugfs_create_file("state", 0400, dir, aes, &apple_sio_aes_state_fops); + + return 0; +} + +static const struct apple_sio_aes_data apple_sio_aes_v3 = { + .revision = 3, + .shared_disable = true, +}; + +static const struct apple_sio_aes_data apple_sio_aes_t6041 = { + .revision = 5, + .uid_offset = 0x04, + .gid0_offset = 0x08, + .gid1_offset = 0x0c, +}; + +static const struct apple_sio_aes_data apple_sio_aes_t8140 = { + .revision = 5, + .uid_offset = 0x40, + .gid0_offset = 0x44, + .gid1_offset = 0x48, +}; + +static const struct of_device_id apple_sio_aes_of_match[] = { + { .compatible = "apple,t6000-aes", .data = &apple_sio_aes_v3 }, + { .compatible = "apple,t8112-aes", .data = &apple_sio_aes_v3 }, + { .compatible = "apple,t6041-aes", .data = &apple_sio_aes_t6041 }, + { .compatible = "apple,t8140-aes", .data = &apple_sio_aes_t8140 }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_sio_aes_of_match); + +static struct platform_driver apple_sio_aes_driver = { + .probe = apple_sio_aes_probe, + .driver = { + .name = "apple-sio-aes", + .of_match_table = apple_sio_aes_of_match, + }, +}; +module_platform_driver(apple_sio_aes_driver); + +MODULE_DESCRIPTION("Apple SIO AES probe-time read-only diagnostics"); +MODULE_AUTHOR("Ryan Murray <81541646+RyanTheTide@users.noreply.github.com>"); +MODULE_LICENSE("GPL"); From 26d61797c7aec2f82758f24512b92461d4a1be67 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:59:23 +1100 Subject: [PATCH 351/392] Revert "arm64: dts: apple: attach the Neo internal PCIe host to its full power chain" This reverts commit 5c96ab0a5f97a. The Neo change has not been tested on hardware yet; it comes back as its own change once it has. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 4242e655a3af7..b781dd756a2d2 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1206,7 +1206,7 @@ #size-cells = <2>; ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; - power-domains = <&ps_apcie_phy_sw>; + power-domains = <&ps_apcie_gp>; status = "disabled"; pcie0_port0: pci@0,0 { From 20e67a19beb58dfa5d983192d68a5b1799f8fcef Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 9 Nov 2026 11:57:58 -0300 Subject: [PATCH 352/392] media: apple: isp: offer only the board's camera views isp_ch_cache_sensor_configs() appends every firmware sensor config that passes the geometry check as a preset, cropped to its output size at offset 0. On the J616s IMX958 the only one that passes is config 12, 1536x3024: a sensor window over the right half of the 3072-wide sensor, dark lens edge included. The firmware reports output == input for all 13 configs, so its framing cannot be told from the reply. That preset is off-centre on its own, and as the largest crop it was also the source isp_add_call_sizes() used for 640x480, 1280x960, 960x1280, 1080x1920, 1328x1760 and the square. Browsers open 640x480 by default, so a web camera test showed a zoomed-in corner. Keep reading the configs for P010 qualification, but stop appending them. The sizes then all come from the board presets: 1920x1080, 1280x720, and 640x480, 1280x960 and 1080x1080 as the centre of the 1080p view. The sensor-config KUnit case now expects only the two board presets. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM --- .../media/platform/apple/isp/isp-cam-test.c | 7 ++---- drivers/media/platform/apple/isp/isp-cam.c | 24 +------------------ 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 191d8ada0d265..72bcefdae4373 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -438,14 +438,11 @@ static void isp_test_sensor_config_list(struct kunit *test) KUNIT_EXPECT_EQ(test, isp->sensor_configs[5].reply[ISP_SENSOR_CONFIG_SIZE - 1], 0x5a); KUNIT_EXPECT_EQ(test, count, 6); KUNIT_EXPECT_EQ(test, isp->num_sensor_configs, 6); - KUNIT_EXPECT_EQ(test, isp->num_presets, 3); + /* Firmware configs only qualify P010: the uncropped 4096x3072 one is not offered. */ + KUNIT_ASSERT_EQ(test, isp->num_presets, 2); KUNIT_EXPECT_EQ(test, isp->presets[0].index, 5); KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.y, 1080); KUNIT_EXPECT_TRUE(test, isp->presets[1].p010); - KUNIT_EXPECT_TRUE(test, isp->presets[2].p010); - KUNIT_EXPECT_EQ(test, isp->presets[2].index, 1); - KUNIT_EXPECT_EQ(test, isp->presets[2].output_dim.x, 4096); - KUNIT_EXPECT_EQ(test, isp->presets[2].crop_size.y, 3072); KUNIT_EXPECT_FALSE(test, isp->sensor_configs[2].usable); KUNIT_EXPECT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 0), -EINVAL); KUNIT_EXPECT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 65), -EINVAL); diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 4b096aa29dddf..0c618146c8716 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -279,7 +279,7 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) if (!count || count > ISP_MAX_SENSOR_CONFIGS) return -EINVAL; - presets = devm_kcalloc(isp->dev, num + count, sizeof(*presets), GFP_KERNEL); + presets = devm_kcalloc(isp->dev, num, sizeof(*presets), GFP_KERNEL); if (!presets) return -ENOMEM; memcpy(presets, isp->presets, num * sizeof(*presets)); @@ -290,7 +290,6 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) for (u32 i = 0; i < count; i++) { struct cmd_ch_camera_config *config = &args.config; struct isp_sensor_config *cached = &configs[i]; - bool duplicate = false; err = isp_cmd_ch_camera_config_get(isp, ch, i, &args); if (err) @@ -335,27 +334,6 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) preset->crop_size.y * preset->output_dim.x) preset->p010 = preview || reported; } - if (!cached->usable) - continue; - - /* Keep the board's config-5 presets, including the default, first. */ - for (unsigned int j = 0; j < num; j++) { - if (presets[j].output_dim.x == config->out_width && - presets[j].output_dim.y == config->out_height) { - duplicate = true; - break; - } - } - if (duplicate) - continue; - - presets[num++] = (struct isp_preset){ - .index = i, - .input_dim = cached->input_dim, - .output_dim = cached->output_dim, - .crop_size = cached->output_dim, - .p010 = true, - }; } isp->sensor_configs = configs; isp->num_sensor_configs = count; From 0cc9479d28ae508a5bea6e849434ec2b12701f5a Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:01:56 +1100 Subject: [PATCH 353/392] tools/asahi: tbnet-refill: Model the login retry and login answer worker tbnet_connect_failed_locked() now starts a new login after TBNET_LOGIN_DELAY and tbnet_stop() cancels the login answer worker. Give the fixture the new struct tbnet members and constants, and count a login retry apart from the Rx refill retry, whose 100 ms delay the fixture still checks. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- tools/asahi/tbnet-refill-fixture.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/asahi/tbnet-refill-fixture.h b/tools/asahi/tbnet-refill-fixture.h index d9dee1baab9b2..aa2218f8af8c7 100644 --- a/tools/asahi/tbnet-refill-fixture.h +++ b/tools/asahi/tbnet-refill-fixture.h @@ -16,6 +16,8 @@ #define TBNET_RX_PAGE_SIZE (PAGE_SIZE<work.fn=(f)) static _Thread_local int heldmutex; @@ -92,7 +96,7 @@ static bool fake_softirq; static bool in_softirq(void){return fake_softirq;} static int smp_processor_id(void){return 0;} static void schedule_work(struct work_struct *w){} -static bool queue_delayed_work(int wq,struct delayed_work *w,unsigned long delay){assert(delay==100);pthread_mutex_lock(&gate);bool new=!w->pending;if(new){w->pending=true;retry_queues++;retry_delay=delay;}pthread_mutex_unlock(&gate);return new;} +static bool queue_delayed_work(int wq,struct delayed_work *w,unsigned long delay){if(delay==TBNET_LOGIN_DELAY){assert(w==&container_of(w,struct tbnet,login_work)->login_work);login_queues++;return true;}assert(delay==100);pthread_mutex_lock(&gate);bool new=!w->pending;if(new){w->pending=true;retry_queues++;retry_delay=delay;}pthread_mutex_unlock(&gate);return new;} static void cancel_delayed_work_sync(struct delayed_work *w) { pthread_mutex_lock(&gate); From 960e1634f86683c3a413dd1a308086d06c595e75 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:26:47 +1100 Subject: [PATCH 354/392] selftests: apple-dcp: Expect CD321x to watch a router present at probe CD321x now watches for the USB4/Thunderbolt router to register a new switch also when the router is already there at probe, so that it takes the new switch after an unbind and bind. Update the typec-probe case: a port probed with its router holds the watch until it is unregistered, and a watch that cannot start leaves the port probed with the router it has. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- tools/testing/selftests/drivers/gpu/apple-dcp/pm.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 8cae88d722d88..92ccae9ed052f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -747,8 +747,14 @@ def function(text, name): assert(cd321x_register_port(&c.tps,&connector)==0&&c.state_valid&&!c.tbt_switch); assert(!watches&&!c.tbt_watch_registered&&!c.tbt_fwnode); cd321x_unregister_port(&c.tps);balanced(); - /* A router present at probe needs no watch. */ - reset(&c);assert(cd321x_register_port(&c.tps,&connector)==0&&c.tbt_switch&&!watches); + /* A router present at probe is watched too: it can unbind and bind again with a new switch. */ + reset(&c);assert(cd321x_register_port(&c.tps,&connector)==0&&c.tbt_switch); + assert(watches==1&&c.tbt_watch_registered&&c.tbt_fwnode==&connector); + cd321x_unregister_port(&c.tps);balanced(); + /* Without that watch the port still probes with the router it has. */ + reset(&c);watch_error=-ENOMEM; + assert(cd321x_register_port(&c.tps,&connector)==0&&c.tbt_switch); + assert(!watches&&!c.tbt_watch_registered&&!c.tbt_fwnode); cd321x_unregister_port(&c.tps);balanced(); ''', 1) From e39c9000c3395ea83139c5c313103e3b86c4b837 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 15:53:50 -0500 Subject: [PATCH 355/392] dt-bindings: soc: apple: pmp: Type the firmware's segment names as a string The ADT's iop-pmp nub carries segment-names as a NUL-terminated string ("__TEXT;__DATA" on J813, "__TEXT;__DATA;__ETEXT" on J714s). A DTB has no types, so dt-validate decodes those bytes as a string and the uint8-array pattern rejects them on every board that carries the value (t8142-j813 today). Give the property its own string type and keep it out of the opaque-bytes pattern; the bytes in the DTB do not change. Signed-off-by: Diljit Singh --- .../devicetree/bindings/soc/apple/apple,pmp.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index 4ae4380167632..7eeca76b274ed 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -114,8 +114,14 @@ properties: minItems: 2 description: Host physical address and size pairs mapped into firmware PIO space. + apple,tunable-segment-names: + $ref: /schemas/types.yaml#/definitions/string + description: + The firmware's segment names from the Apple device tree, separated by + semicolons. The firmware reads them as the raw string. + patternProperties: - '^apple,tunable-': + '^apple,tunable-(?!segment-names$)': $ref: /schemas/types.yaml#/definitions/uint8-array description: Opaque firmware calibration copied from the Apple device tree. From ce19002def96ef7a4f3f0546dcf1a3b1d030c79f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 15:53:50 -0500 Subject: [PATCH 356/392] arm64: dts: apple: t6050: Name the PMP DART's regions The binding requires reg-names when a DART has a second region. The ADT dart-pmp0 instance list names them DART and DAPF (it also carries dapf-instance-0), so call them "dart" and "dapf". No driver looks them up by name. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-peripherals.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index d3524080347f6..8063b37f816db 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -2286,8 +2286,10 @@ pmp0_dart: iommu@2841a0000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; + /* ADT dart-pmp0 instance: DART, then DAPF */ reg = <0x2 0x841a0000 0x0 0xc000>, <0x2 0x841b0000 0x0 0x4000>; + reg-names = "dart", "dapf"; interrupts = ; #iommu-cells = <1>; /* ADT dart-pmp0 vm-base, vm-size */ From 9b2acdbbb2122fe9276a530f379005519aab7955 Mon Sep 17 00:00:00 2001 From: Friedrich <261542475+aeries0x@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:21:51 +0100 Subject: [PATCH 357/392] drm/apple: m4: accept the J604 firmware's page-aligned D451 buffers On the J604 (MacBook Pro 14-inch, M4) the internal DCP's 26.6.2 firmware asks D451 allocate_buffer for buffers with an alignment of 4096 as well as 4. The internal path accepts only 4, so the first such request fails the callback, the retained session is quarantined and appledrm does not bind: the panel stays on simpledrm. Let a panel profile accept an alignment that is a power of two up to a page on its internal DCP, which every buffer has, as the external DCPs' rule already does, and set it for the J604 only. Every other check of the qualified request stays as it is; nothing changes for the J713 or the J715. Tested on a J604 (26.6.2) with apple_dart.j713_dcp=1 and apple_dart.j713_display=1: the DCP starts, appledrm binds the panel at 3024x1964 and the desktop runs on it. Assisted-by: LLM Signed-off-by: Friedrich <261542475+aeries0x@users.noreply.github.com> --- drivers/gpu/drm/apple/dcp-m4-memory.h | 14 ++++++++++---- drivers/gpu/drm/apple/dcp-m4-profile.h | 6 ++++++ drivers/gpu/drm/apple/iomfb_m4.c | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-m4-memory.h b/drivers/gpu/drm/apple/dcp-m4-memory.h index 8969aee8c2447..24d685e2755f0 100644 --- a/drivers/gpu/drm/apple/dcp-m4-memory.h +++ b/drivers/gpu/drm/apple/dcp-m4-memory.h @@ -3,6 +3,7 @@ #define APPLE_DCP_M4_MEMORY_H #include +#include #include #include #include @@ -32,6 +33,8 @@ struct dcp_m4_memory { * internal firmware's requests are only qualified as they are. */ bool relaxed; + /* Internal DCP: also an alignment up to a page (the profile's alloc_page_align). */ + bool page_align; }; struct dcp_m4_memory_ops { @@ -129,9 +132,10 @@ static inline int dcp_m4_memory_callback(struct dcp_m4_memory *memory, size = get_unaligned_le64(in + 4); align = get_unaligned_le32(in + 12); /* - * The qualified internal firmware requests 4. An external one's - * alignment, in bytes, must be a power of two within a page, which - * every buffer has. + * The qualified internal firmware requests 4, the J604's also a + * power of two within a page. An external one's alignment, in + * bytes, must be a power of two within a page, which every buffer + * has. */ if (memory->relaxed && (!size || size > DCP_M4_MEMORY_MAX || (align && (align > DCP_M4_MEMORY_PAGE || @@ -140,7 +144,9 @@ static inline int dcp_m4_memory_callback(struct dcp_m4_memory *memory, goto failed; if (!memory->relaxed && (get_unaligned_le32(in) != 0x703 || !size || size > DCP_M4_MEMORY_MAX || - align != 4 || in[16] || in[17] || in[18])) + (align != 4 && (!memory->page_align || !is_power_of_2(align) || + align > DCP_M4_MEMORY_PAGE)) || + in[16] || in[17] || in[18])) goto failed; /* Byte 19 is padding, observed nonzero in otherwise valid requests. */ size = (size + DCP_M4_MEMORY_PAGE - 1) & ~(u64)(DCP_M4_MEMORY_PAGE - 1); diff --git a/drivers/gpu/drm/apple/dcp-m4-profile.h b/drivers/gpu/drm/apple/dcp-m4-profile.h index 88900add94724..438f734f87d06 100644 --- a/drivers/gpu/drm/apple/dcp-m4-profile.h +++ b/drivers/gpu/drm/apple/dcp-m4-profile.h @@ -32,6 +32,11 @@ struct dcp_m4_profile { */ const u32 (*timing)[8]; u32 timing_elements; /* most timing elements reported */ + /* + * D451 may also ask for an alignment that is a power of two up to a + * page, besides the 4 every qualified panel firmware asks for. + */ + bool alloc_page_align; }; static const u32 dcp_m4_timing_j713[2][8] = { @@ -96,6 +101,7 @@ static const struct dcp_m4_profile dcp_m4_profile_j604 = { .brightness_max = 1000, .brightness_default = 120, .width_mm = 0, .height_mm = 0, .timing = dcp_m4_timing_j604, .timing_elements = 6, + .alloc_page_align = true, }; static const struct dcp_m4_profile *const dcp_m4_profiles[] = { diff --git a/drivers/gpu/drm/apple/iomfb_m4.c b/drivers/gpu/drm/apple/iomfb_m4.c index b6b301a20ab20..2c9194aa5f582 100644 --- a/drivers/gpu/drm/apple/iomfb_m4.c +++ b/drivers/gpu/drm/apple/iomfb_m4.c @@ -2367,6 +2367,7 @@ static int dcp_m4_startup(struct iomfb_m4 *dcp) } /* An external firmware's D451 requests are not qualified as such. */ dcp->memory->relaxed = dcp->external; + dcp->memory->page_align = !dcp->external && dcp_m4_pf->alloc_page_align; /* An external display DART counts as off until a D572(1). */ WRITE_ONCE(dcp->ext_dart_down, dcp->external); r->properties = kzalloc(sizeof(*r->properties), GFP_KERNEL); From 3e440ed4d1f712d51499a5e919bce8b73d6b230e Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:23:14 +1100 Subject: [PATCH 358/392] arm64: dts: apple: t6000: add the SMC rail voltages The M1 Pro MacBook Pros expose no CPU, GPU or SoC rail voltage, though their SMC reports every output of the cluster, GPU and SoC voltage regulators on the CLVR SPMI buses, and macOS reads the same keys. Add them as hwmon voltage channels: VC10-VC13 (CPU P clusters), VC20-VC23 (GPU) and VC30-VC33 (SoC), plus the two main PMU bucks that follow the E cluster (VP0b, VP2b). The keys were mapped on a J316s by holding every cluster at its lowest step and loading one at a time: VC10/VC11 follow the second P cluster (571-906 mV and 770-1060 mV from the lowest to the highest step), VC12/VC13 the first (566-906 mV, 768-1058 mV), VP0b/VP2b the E cluster (547-834 mV, 768-924 mV). VC20-VC23 read 0 mV while the GPU is power-gated and 700-800 mV under load; VC32/VC33 rise with GPU load, VC30/VC31 stay flat. Linux sends nothing on the CLVR buses; the channels are SMC key reads. Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Assisted-by: LLM --- .../boot/dts/apple/t6000-hwmon-rails.dtsi | 88 +++++++++++++++++++ arch/arm64/boot/dts/apple/t6000-j314s.dts | 1 + arch/arm64/boot/dts/apple/t6000-j316s.dts | 1 + 3 files changed, 90 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t6000-hwmon-rails.dtsi diff --git a/arch/arm64/boot/dts/apple/t6000-hwmon-rails.dtsi b/arch/arm64/boot/dts/apple/t6000-hwmon-rails.dtsi new file mode 100644 index 0000000000000..7d1070efb8ccc --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6000-hwmon-rails.dtsi @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T6000 rail voltages published by the SMC. + * + * The three CLVR SPMI buses (clvr-spmi0-2) connect the SMC to the cluster, + * GPU and SoC voltage regulators. Linux leaves those buses to the SMC and + * reads the SMC's VCxy keys instead (x = bus + 1, y = regulator output). + * With the other clusters held at their lowest step, one cluster under load + * moves only its own keys: + * P0 cluster (cpu2-5): VC12 566-906 mV, VC13 768-1058 mV + * P1 cluster (cpu6-9): VC10 571-906 mV, VC11 770-1060 mV + * GPU: VC20-VC23, 0 mV while power-gated, 700-800 mV under load + * SoC (clvr-spmi2): VC30/VC31 stay flat, VC32/VC33 rise with GPU load + * The E cluster (cpu0-1) is fed by the main PMU: VP0b 547-834 mV and + * VP2b 768-924 mV. + */ + +&smc_hwmon { + voltage-VC10 { + apple,key-id = "VC10"; + label = "P1-cluster VC10"; + }; + + voltage-VC11 { + apple,key-id = "VC11"; + label = "P1-cluster VC11"; + }; + + voltage-VC12 { + apple,key-id = "VC12"; + label = "P0-cluster VC12"; + }; + + voltage-VC13 { + apple,key-id = "VC13"; + label = "P0-cluster VC13"; + }; + + voltage-VC20 { + apple,key-id = "VC20"; + label = "GPU VC20"; + }; + + voltage-VC21 { + apple,key-id = "VC21"; + label = "GPU VC21"; + }; + + voltage-VC22 { + apple,key-id = "VC22"; + label = "GPU VC22"; + }; + + voltage-VC23 { + apple,key-id = "VC23"; + label = "GPU VC23"; + }; + + voltage-VC30 { + apple,key-id = "VC30"; + label = "SoC VC30"; + }; + + voltage-VC31 { + apple,key-id = "VC31"; + label = "SoC VC31"; + }; + + voltage-VC32 { + apple,key-id = "VC32"; + label = "SoC VC32"; + }; + + voltage-VC33 { + apple,key-id = "VC33"; + label = "SoC VC33"; + }; + + voltage-VP0b { + apple,key-id = "VP0b"; + label = "E-cluster VP0b"; + }; + + voltage-VP2b { + apple,key-id = "VP2b"; + label = "E-cluster VP2b"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 2286e31c8f59a..01108aa644dca 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -18,6 +18,7 @@ #include "t6000.dtsi" #include "t600x-j314-j316.dtsi" +#include "t6000-hwmon-rails.dtsi" / { compatible = "apple,j314s", "apple,t6000", "apple,arm-platform"; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 51e615c5ccf0d..13a6826ae5374 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -17,6 +17,7 @@ #include "t6000.dtsi" #include "t600x-j314-j316.dtsi" +#include "t6000-hwmon-rails.dtsi" / { compatible = "apple,j316s", "apple,t6000", "apple,arm-platform"; From 9554b061cb252e98393b5b43700c9c1aaca7f2af Mon Sep 17 00:00:00 2001 From: Dylan Ironside Date: Mon, 9 Nov 2026 22:38:01 +0000 Subject: [PATCH 359/392] rust: drm: gpuvm: lock a shared reservation once in exec_lock() On a MacBook Pro 14" M4 (J604, T8132, G16), mpv with zero-copy VA-API into its Vulkan output (what hwdec=auto-safe picks once the VA driver loads) shows a blank window for 1080p H.264, HEVC and VP9 video and reports VK_ERROR_DEVICE_LOST. For each failure the kernel logs: aurora-g16 ...gpu: [File N Queue 1]: IOCTL: submit failed! (submission ID: N err: EFAULT) The submit is refused during validation: 1. Mesa drops a BO by closing its handle. The close callback (AsahiObject::close() -> Vm::drop_mappings()) defers the unmap while the VM has accepted jobs in flight, so the closed GEM object keeps its GPUVAs and its vm_bo stays on the VM's external-object list. 2. mpv re-imports the same decoder dma-buf for every frame. With the old handle gone from the prime cache, drm_gem_prime_import_dev() creates a second GEM object for it, and both objects share dma_buf->resv. 3. Every G16 SUBMIT reads its control streams through Vm::read_user() (g16/gate.rs), which takes GpuVm::exec_lock(), i.e. drm_gpuvm_exec_lock() with flags 0. drm_gpuvm_prepare_objects() then locks the shared reservation a second time with the same ticket, ww_mutex returns -EALREADY, and without DRM_EXEC_IGNORE_DUPLICATES drm_exec returns that error. 4. The gate maps any read_user() error to EFAULT, so the submit fails and Mesa reports the device as lost for that client. Whether it triggers depends on the VA layout: when the re-import is bound at the old address, that bind removes the stale mapping first (seen at 4K); at 1080p the re-import landed at a new address and the next submit failed every time after a seek. C users of drm_gpuvm_exec and drm_exec that can see shared reservations set DRM_EXEC_IGNORE_DUPLICATES for this case, for example drivers/gpu/drm/nouveau/nouveau_sched.c (the struct drm_gpuvm_exec in nouveau_job_submit()) and drivers/gpu/drm/panthor/panthor_drv.c (panthor_submit_ctx_init()); xe, msm and imagination do the same. Set it in exec_lock_impl(): a reservation shared by two objects is then locked and unlocked once, and nothing else changes. exec_lock_vm() shares the initializer and locks only the VM reservation, so its behaviour does not change. The idle-VM restore path (mmu/residency.rs), which also takes the full exec lock, is covered as well. Before the Fixes: commit, drop_mappings() unmapped at handle close, so two live objects on one reservation could not both be on the external-object list. Tested only on the J604 (T8132, G16), on the Aurora 7.3 base 6ff31cbfa2 plus the J604 board patches, with this change unmodified: stock mpv with the VA-API alias, generated and real 1080p H.264/HEVC/VP9 and 4K H.264 clips, 3 seek+pause and 3 play-from-start runs each, hardware decoding in every run, pictures equal to software decoding, no Vulkan errors and no "submit failed" lines (before: blank picture and one EFAULT per run for every 1080p clip). Not tested on other GPU generations, and the idle-VM restore path was not exercised. Fixes: 52ef95b9e763 ("drm/asahi: add the T8140 firmware interface") Assisted-by: LLM Signed-off-by: Dylan Ironside --- rust/kernel/drm/gpuvm/mod.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 8b40308767619..66adaf99e0859 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -335,11 +335,14 @@ impl GpuVm { let mut vm_exec = KBox::init( init!(bindings::drm_gpuvm_exec { vm: self.as_raw(), - flags: if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }, + // Two GEM objects can share one reservation (a dma-buf imported again while + // the closed object still has mappings); lock it once, as C drm_exec users do. + flags: bindings::DRM_EXEC_IGNORE_DUPLICATES + | if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, exec: Default::default(), extra: match (is_ext, obj) { (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { From e7f996f17c2c4b33f3c0903aa72b249e776f7417 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:42:38 -0600 Subject: [PATCH 360/392] arm64: dts: apple: t6050-j714c: Run the PMP and enable the right USB-C port (#1391) * arm64: dts: apple: t6050-j714c: Run the PMP and report the GPU to it Enable the PMP on the M5 Max 14-inch the way the J714s does since #1376: the PMP with its boot arguments, mailbox, DART and power states, the PMP report, the PMC's agent 0 and the GPU's request in the report. They stay enabled together, so the GPU never waits on a disabled report. The CLPC keeps its vote; the PMC takes the higher of the two. The values come from ADTs of the same silicon, not from a J714c unit: - apple,board-id is 10, the J714c's board ID (0x0a; the N1 platform ID above has it too). - apple,dram-vendor-id is 255 and pmc/pmc-pmgr are 1/63, as a J716c reports them. They are only fallbacks: the boot loader copies /chosen and the PMGR's values over them. - t6050-max-pmp-tunables.dtsi is the 173 iop-pmp0-nub properties of a J716cAP 27.0 ADT, raw bytes, in the format of the J714s file. The J714cAP and J716cAP firmware ADTs have identical PMP nodes (props and all 29 variant sub-nodes), and the h17c-family variant tables of that firmware ADT reproduce 166 of the 173 values; the other seven (the AGX DVFS voltages, the segment ranges and names, pre-loaded and three variant picks) are runtime values iBoot fills in. Not run on a J714c. * arm64: dts: apple: t6050-j716c: Move the right USB-C port into t6050-max-atc2.dtsi The J714cAP and J716cAP firmware ADTs carry identical nodes for the right USB-C port: the HPM on nub-spmi-a1, ATC2's PHY, USB controller and DARTs, the USB4/Thunderbolt router and the PCIe tunnel. Move the J716c's description of them into t6050-max-atc2.dtsi so the J714c can include it instead of a second copy. The host DROM (apple,thunderbolt-drom) stays in the board file: this J716c unit's DROM carries a router UUID from an earlier boot. No change to the J716c: its DTB is byte-identical before and after, and so are the other ten T6050 board DTBs. * arm64: dts: apple: t6050-j714c: Enable the right USB-C port Include t6050-max-atc2.dtsi on the J714c: the right USB-C port's PD controller (nub-spmi-a1, hpm2), ATC2's PHY, USB controller and DARTs, the USB4/Thunderbolt router and the PCIe tunnel. The J714cAP firmware ADT describes them with the same values as the J716cAP one. The ADT also has the PD controllers hpm0 and hpm1 on nub-spmi-a0 and an hpm5; this commit does not describe them or the ports behind them (ATC0, ATC1, ATC3). The nodes the J714c board did not reach are now in t6050-max.dtsi's J714c-and-J716c blocks (ATC2, ACIO2, APCIEC2 and nub-spmi-a1), and t6050-peripherals.dtsi stops adding its own staged nub-spmi-a1 and apciec2_dart for the J714c. apple,thunderbolt-drom is the template value of the J714cAP ADT, not a unit's UUID; the boot loader replaces it. Not run on a J714c. Checked offline: all 11 T6050 board DTBs build with dtc, and the ten other than the J714c are byte-identical to the base tree. dt-validate of the J714c DTB has one warning, the PCIe "port0" in the apciec2 node, which the J716c DTB has already; the J714c had none before. --- arch/arm64/boot/dts/apple/t6050-j714c.dts | 68 ++- arch/arm64/boot/dts/apple/t6050-j716c.dts | 390 +---------------- arch/arm64/boot/dts/apple/t6050-max-atc2.dtsi | 398 ++++++++++++++++++ .../dts/apple/t6050-max-pmp-tunables.dtsi | 184 ++++++++ arch/arm64/boot/dts/apple/t6050-max.dtsi | 4 +- .../boot/dts/apple/t6050-peripherals.dtsi | 4 +- 6 files changed, 655 insertions(+), 393 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t6050-max-atc2.dtsi create mode 100644 arch/arm64/boot/dts/apple/t6050-max-pmp-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t6050-j714c.dts b/arch/arm64/boot/dts/apple/t6050-j714c.dts index bfc2e21e909e6..bd4ea3fc0d288 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714c.dts @@ -14,6 +14,7 @@ /dts-v1/; +#define T6050_J714C #include "t6050-max.dtsi" / { @@ -220,5 +221,70 @@ }; }; -#define T6050_J714C #include "t6050-peripherals.dtsi" + +/* + * The PMP (ADT pmp0, /chosen, /arm-io/pmgr), as on the J714s. iBoot loads its firmware and the + * driver starts it with these boot arguments and the firmware properties below. Its report + * tells it which devices are on; it then votes DCS and the fabric for GPU load itself, and the + * PMC honours that next to the CLPC. The report, the PMP's vote and the GPU's request are + * enabled together, so the GPU never waits on a disabled report. + */ +&ps_pms { + status = "okay"; +}; + +&ps_pmp { + status = "okay"; +}; + +&ps_pms_sram { + status = "okay"; +}; + +&pmp0_dart { + status = "okay"; +}; + +&pmp0_mbox { + status = "okay"; +}; + +&pmp0 { + /* The J714c's board ID (0x0a, as in the N1 platform ID above) and the dram-vendor-id a J716c reports */ + apple,board-id = <10>; + apple,dram-vendor-id = <255>; + /* ADT /arm-io/pmgr pmc, pmc-pmgr, as read on a J716c */ + apple,pmc = <1>; + apple,pmc-pmgr = <63>; + status = "okay"; +}; + +&pmp0_report { + status = "okay"; +}; + +&pmc { + /* PMP, CLPC and AP agents, once the PMP runs */ + apple,static-voters = <0 1 3>; + apple,pmp = <&pmp0>; +}; + +&sgx { + power-domains = <&pmp0_gfx>; +}; + +#include "t6050-max-pmp-tunables.dtsi" + +#include "t6050-max-atc2.dtsi" + +/* The host DROM from the J714cAP ADT (/arm-io/acio2 thunderbolt-drom); the boot loader replaces it */ +&acio2_nhi { + apple,thunderbolt-drom = [fd 02 00 00 00 00 00 00 00 90 19 c3 8c 00 5c 00 + 00 00 00 00 00 00 08 81 82 02 82 00 00 00 08 82 + 92 01 82 00 00 00 03 83 20 08 84 00 00 00 00 00 + 00 08 85 00 00 00 00 00 00 05 86 09 01 00 08 87 + 00 00 00 00 00 00 05 88 09 01 00 05 89 09 01 00 + 05 8a 09 01 00 0e 01 41 70 70 6c 65 2c 20 49 6e + 63 2e 00 06 02 69 4f 53 00 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j716c.dts b/arch/arm64/boot/dts/apple/t6050-j716c.dts index 25ba638862ca0..1ff75c278ce84 100644 --- a/arch/arm64/boot/dts/apple/t6050-j716c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j716c.dts @@ -223,338 +223,10 @@ #include "t6050-peripherals.dtsi" -/* - * ==== The right USB-C port (ATC2) ==== - * - * ADT /arm-io/nub-spmi-a1/hpm2 (SN201202x, SPMI address 0xc, port-number 3, port-location - * "right", acio-parent acio2), atc-phy2, usb-drd2 and dart-usb2. The HPM is on its own bus, - * nub-spmi-a1, and nothing hands the port off, so the chain is enabled outright. Aura's - * kboot_setup_atc() fills the PHY's tunables through the atcphy2 alias. - */ -/ { - aliases { - atcphy2 = &atc2_phy; - }; -}; - -&ps_atc2_common { - status = "okay"; -}; - -&ps_atc2_usb_aon { - status = "okay"; -}; - -&ps_atc2_phymxwrap { - status = "okay"; -}; - -&ps_atc2_usb { - status = "okay"; -}; - -&atc2_phy_mbox { - status = "okay"; -}; - -&atc2_phy { - status = "okay"; - - port { - atc2_phy_ep: endpoint { - remote-endpoint = <&hpm2_ss>; - }; - }; -}; - -&usb_drd2 { - status = "okay"; - - port { - usb_drd2_hs: endpoint { - remote-endpoint = <&hpm2_hs>; - }; - }; -}; - -&usb_drd2_dart_0 { - status = "okay"; -}; - -&usb_drd2_dart_1 { - status = "okay"; -}; - -/* - * dcpext0's Type-C route stays off with dcpext0: a mode-switch endpoint whose mux is never - * registered would defer the PD controller forever. - */ -&dcpext0 { - typec-routes { - route@0 { - status = "disabled"; - - port { - dcpext0_typec: endpoint { - remote-endpoint = <&hpm2_dp>; - }; - }; - }; - }; -}; - -&nub_spmi_a1 { - hpm2: usb-pd@c { - compatible = "apple,sn201202x"; - reg = <0xc SPMI_USID>; - - connector { - compatible = "usb-c-connector"; - label = "USB-C Right"; - power-role = "dual"; - data-role = "dual"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - hpm2_hs: endpoint { - remote-endpoint = <&usb_drd2_hs>; - }; - }; - - port@1 { - reg = <1>; - hpm2_ss: endpoint { - remote-endpoint = <&atc2_phy_ep>; - }; - }; - - port@2 { - reg = <2>; - hpm2_dp: endpoint { - remote-endpoint = <&dcpext0_typec>; - }; - }; - }; - }; - }; -}; - -/* - * ==== The right USB-C port's USB4/Thunderbolt router (ATC2) ==== - * - * Until the boot loader fills them, the router, NHI, PHY CIO and DART tunables and the host - * DROM are this board's, from its ADT (the DROM's router UUID from an earlier boot). - */ -/* ATC2_CIO_AUSS's other parent; kept as the boot loader leaves it */ -&ps_auss { - status = "okay"; -}; - -/* The CIO reconfig request, port 2 (clock gate CIO2_RECONFIG_V) */ -&cio_reconfig0 { - status = "okay"; -}; - -/* The PHY's CIO80 RX_TOP tunables (ADT tunable_LN0/LN1_RX_TOP_CIO_DFLT); the other three are empty */ -&atc2_phy { - apple,tunable-lane0-rx-top-cio = <0x35c 0xe00 0x0>; - apple,tunable-lane1-rx-top-cio = <0x35c 0xe00 0x0>; -}; - -/* Only the router's LSTX pin (81) of the AOP GPIO controller is exposed. */ -&pinctrl_aop { - gpio-reserved-ranges = <0 81>, <82 3>; - status = "okay"; -}; - -&hpm2 { - connector { - ports { - port@3 { - reg = <3>; - hpm2_usb4: endpoint { - remote-endpoint = <&acio2_tbt>; - }; - }; - }; - }; -}; - -&acio2 { - apple,tunable-rc = <0x0 0xfff0 0x4020>, - <0x2c 0x4 0x0>, - <0x4004 0x1ff01ff 0x1000100>, - <0x4008 0x801f 0x4>, - <0x4010 0x7f 0x40>, - <0x4014 0x801f 0x4>, - <0x401c 0x7f 0x40>, - <0x4024 0x801f 0x4>, - <0x402c 0x7f 0x40>, - <0x4030 0x801f 0x4>, - <0x4038 0xffffffff 0x1010101>, - <0x403c 0xffff 0x101>, - <0x4040 0x1 0x1>, - <0x4044 0xffffffff 0x1010101>, - <0x4048 0x1 0x1>, - <0x404c 0x801f 0x4>, - <0x8004 0x70007 0x40004>, - <0x8008 0x801f 0x4>, - <0x8010 0x70007 0x40004>, - <0x8018 0x801f 0x4>, - <0x8020 0x70007 0x40004>, - <0x8028 0x801f 0x4>, - <0x8030 0x801f 0x4>, - <0x8038 0x801f 0x4>, - <0x8040 0x801f 0x4>, - <0x8048 0x801f 0x4>, - <0x8050 0x801f 0x4>, - <0x8058 0x801f 0x4>, - <0x8060 0x801f 0x4>; - apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; - apple,tunable-router-ap-fw = <0x8 0xff2 0x482>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@1 { - reg = <1>; - acio2_tbt: endpoint { - remote-endpoint = <&hpm2_usb4>; - }; - }; - }; -}; - -&acio2_dart { - apple,tunable = <0x20c 0x80000121 0x80000121>, - <0x220 0xf0f 0xf0f>, - <0x224 0xffff 0x909>, - <0x300 0x1f31 0x1>, - <0x308 0x3ffffffc 0x10000000>, - <0x310 0x3ffffffc 0x3ffffffc>, - <0x800 0xf003f 0x10000>, - <0x804 0xf003f 0x10002>, - <0x808 0xf003f 0x10004>, - <0x80c 0xf003f 0x20006>, - <0x810 0xf003f 0x2000a>, - <0x814 0xf003f 0x2000e>, - <0x818 0xf003f 0x20012>, - <0x81c 0xf003f 0x20016>, - <0x820 0xf003f 0x2001a>, - <0x824 0xf003f 0x2001e>, - <0x828 0xf003f 0x20022>, - <0x82c 0xf003f 0x20026>, - <0x830 0xf003f 0x2002a>, - <0x834 0xf003f 0x2002e>, - <0x838 0xf003f 0x20032>, - <0x83c 0xf003f 0x10036>; -}; +#include "t6050-max-atc2.dtsi" +/* This unit's host DROM (router UUID from an earlier boot) */ &acio2_nhi { - apple,tunable-nhi = <0xf8004 0xff 0x80>, - <0xf8008 0x801f 0x4>, - <0xf8010 0x3f 0x20>, - <0xf8014 0x801f 0x4>, - <0xf801c 0x1f 0x10>, - <0xf8020 0x801f 0x4>, - <0xf8028 0xff 0x80>, - <0xf802c 0x801f 0x4>, - <0xf8034 0xff 0x80>, - <0xf8038 0x801f 0x4>, - <0xf8040 0x3f 0x20>, - <0xf8044 0x801f 0x4>, - <0xf804c 0x8007 0x1>, - <0xfc004 0x3 0x2>, - <0xfc008 0x801f 0x4>, - <0xfc010 0x7 0x4>, - <0xfc014 0x801f 0x4>, - <0xfc01c 0x7 0x4>, - <0xfc020 0x801f 0x4>, - <0xfc028 0x7 0x4>, - <0xfc02c 0x801f 0x4>, - <0xfc034 0x7 0x4>, - <0xfc038 0x801f 0x4>, - <0xfc040 0x3 0x2>, - <0xfc044 0x801f 0x4>, - <0xfc04c 0x8007 0x1>, - <0x4000 0x1000000f 0x10000002>, - <0x4004 0x1000000f 0x10000002>, - <0x4008 0x1000000f 0x10000002>, - <0x400c 0x1000000f 0x10000002>, - <0x4010 0x1000000f 0x10000002>, - <0x4014 0x1000000f 0x10000002>, - <0x4018 0x1000000f 0x10000002>, - <0x401c 0x1000000f 0x10000002>, - <0x4020 0x1000000f 0x10000002>, - <0x4024 0x1000000f 0x10000002>, - <0x4028 0x1000000f 0x10000002>, - <0x402c 0x1000000f 0x10000002>, - <0x4040 0x1000000f 0x10000002>, - <0x4044 0x1000000f 0x10000002>, - <0x4048 0x1000000f 0x10000002>, - <0x404c 0x1000000f 0x10000002>, - <0x4050 0x1000000f 0x10000002>, - <0x4054 0x1000000f 0x10000002>, - <0x4058 0x1000000f 0x10000002>, - <0x405c 0x1000000f 0x10000002>, - <0x4060 0x1000000f 0x10000002>, - <0x4064 0x1000000f 0x10000002>, - <0x4068 0x1000000f 0x10000002>, - <0x406c 0x1000000f 0x10000002>, - <0x4080 0x1000000f 0x10000002>, - <0x4084 0x1000000f 0x10000002>, - <0x4088 0x1000000f 0x10000002>, - <0x408c 0x1000000f 0x10000002>, - <0x4090 0x1000000f 0x10000002>, - <0x4094 0x1000000f 0x10000002>, - <0x4098 0x1000000f 0x10000002>, - <0x409c 0x1000000f 0x10000002>, - <0x40a0 0x1000000f 0x10000002>, - <0x40a4 0x1000000f 0x10000002>, - <0x40a8 0x1000000f 0x10000002>, - <0x40ac 0x1000000f 0x10000002>, - <0x40c0 0x1000000f 0x10000002>, - <0x40c4 0x1000000f 0x10000002>, - <0x40c8 0x1000000f 0x10000002>, - <0x40cc 0x1000000f 0x10000002>, - <0x40d0 0x1000000f 0x10000002>, - <0x40d4 0x1000000f 0x10000002>, - <0x40d8 0x1000000f 0x10000002>, - <0x40dc 0x1000000f 0x10000002>, - <0x40e0 0x1000000f 0x10000002>, - <0x40e4 0x1000000f 0x10000002>, - <0x40e8 0x1000000f 0x10000002>, - <0x40ec 0x1000000f 0x10000002>, - <0x4100 0x1000000f 0x10000002>, - <0x4104 0x1000000f 0x10000002>, - <0x4108 0x1000000f 0x10000002>, - <0x410c 0x1000000f 0x10000002>, - <0x4110 0x1000000f 0x10000002>, - <0x4114 0x1000000f 0x10000002>, - <0x4118 0x1000000f 0x10000002>, - <0x411c 0x1000000f 0x10000002>, - <0x4120 0x1000000f 0x10000002>, - <0x4124 0x1000000f 0x10000002>, - <0x4128 0x1000000f 0x10000002>, - <0x412c 0x1000000f 0x10000002>, - <0x4140 0x1000000f 0x10000002>, - <0x4144 0x1000000f 0x10000002>, - <0x4148 0x1000000f 0x10000002>, - <0x414c 0x1000000f 0x10000002>, - <0x4150 0x1000000f 0x10000002>, - <0x4154 0x1000000f 0x10000002>, - <0x4158 0x1000000f 0x10000002>, - <0x415c 0x1000000f 0x10000002>, - <0x4160 0x1000000f 0x10000002>, - <0x4164 0x1000000f 0x10000002>, - <0x4168 0x1000000f 0x10000002>, - <0x416c 0x1000000f 0x10000002>, - <0x43e4 0x7 0x7>; apple,thunderbolt-drom = [ff 92 e6 6e ee 18 7c ac 05 90 19 c3 8c 00 5c 00 00 00 00 00 00 00 08 81 82 02 82 00 00 00 08 82 92 01 82 00 00 00 03 83 20 08 84 00 00 00 00 00 @@ -564,62 +236,4 @@ 63 2e 00 06 02 69 4f 53 00 00 00 00]; }; -/* ==== The right USB-C port's PCIe tunnel (ATC2): this board's tunables ==== */ -&apciec2_dart { - apple,tunable = <0x20c 0x80000121 0x80000121>, - <0x220 0xf0f 0xf0f>, - <0x224 0xffff 0x909>, - <0x300 0x1f31 0x1>, - <0x308 0x3ffffffc 0x10000000>, - <0x310 0x3ffffffc 0x3ffffffc>; -}; - -&apciec2 { - apple,tunable-debug = <0x1000 0x1 0x0>; - apple,tunable-rc = <0x78 0x7000 0x0>, - <0x718 0x7c000 0x0>, - <0x814 0xffffffff 0x2020f060>, - <0x8bc 0x1 0x0>; - apple,tunable-fabric = <0x4 0x3f003f 0x240024>, - <0x8 0x1 0x1>, - <0xc 0x1 0x1>, - <0x10 0x801f 0x4>, - <0x18 0x1ff 0x1e0>, - <0x1c 0x1 0x1>, - <0x20 0x801f 0x4>, - <0x28 0x1f 0x10>, - <0x2c 0x801f 0x4>, - <0x34 0x3f 0x20>, - <0x38 0x801f 0x4>, - <0x40 0x3f 0x20>, - <0x44 0x801f 0x4>, - <0x4c 0x10001 0x10001>, - <0x50 0x1 0x1>, - <0x54 0x1 0x1>, - <0x58 0x801f 0x4>, - <0x60 0x7f007f 0x450045>, - <0x64 0x801f 0x4>, - <0x6c 0x1f001f 0x100010>, - <0x70 0xf1 0x1>, - <0x74 0x801f 0x4>, - <0x7c 0x801f 0x4>, - <0x84 0x1 0x1>, - <0x88 0x1 0x1>, - <0x8c 0x801f 0x4>, - <0x94 0x1 0x1>, - <0x98 0x801f 0x4>, - <0xa0 0x1 0x1>, - <0xa4 0x801f 0x4>, - <0xac 0x1 0x1>, - <0xb0 0x1 0x1>, - <0xb4 0x801f 0x4>, - <0xbc 0x801f 0x4>, - <0xc4 0x801f 0x4>; -}; - -&apciec2_port { - apple,tunable = <0x140 0x1 0x1>, - <0x2a0 0x7e 0xa>; -}; - #include "t6050-j716c-isp.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6050-max-atc2.dtsi b/arch/arm64/boot/dts/apple/t6050-max-atc2.dtsi new file mode 100644 index 0000000000000..40bcd58cd843f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-max-atc2.dtsi @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * The M5 Max MacBook Pros' right USB-C port (ATC2): USB 2/3, the PD controller and the + * USB4/Thunderbolt router. A board file includes this after t6050-peripherals.dtsi and sets its + * own host DROM (apple,thunderbolt-drom) on &acio2_nhi; the boot loader replaces it. + */ + +/* + * ==== The right USB-C port (ATC2) ==== + * + * ADT /arm-io/nub-spmi-a1/hpm2 (SN201202x, SPMI address 0xc, port-number 3, port-location + * "right", acio-parent acio2), atc-phy2, usb-drd2 and dart-usb2. The HPM is on its own bus, + * nub-spmi-a1, and nothing hands the port off, so the chain is enabled outright. Aura's + * kboot_setup_atc() fills the PHY's tunables through the atcphy2 alias. + */ +/ { + aliases { + atcphy2 = &atc2_phy; + }; +}; + +&ps_atc2_common { + status = "okay"; +}; + +&ps_atc2_usb_aon { + status = "okay"; +}; + +&ps_atc2_phymxwrap { + status = "okay"; +}; + +&ps_atc2_usb { + status = "okay"; +}; + +&atc2_phy_mbox { + status = "okay"; +}; + +&atc2_phy { + status = "okay"; + + port { + atc2_phy_ep: endpoint { + remote-endpoint = <&hpm2_ss>; + }; + }; +}; + +&usb_drd2 { + status = "okay"; + + port { + usb_drd2_hs: endpoint { + remote-endpoint = <&hpm2_hs>; + }; + }; +}; + +&usb_drd2_dart_0 { + status = "okay"; +}; + +&usb_drd2_dart_1 { + status = "okay"; +}; + +/* + * dcpext0's Type-C route stays off with dcpext0: a mode-switch endpoint whose mux is never + * registered would defer the PD controller forever. + */ +&dcpext0 { + typec-routes { + route@0 { + status = "disabled"; + + port { + dcpext0_typec: endpoint { + remote-endpoint = <&hpm2_dp>; + }; + }; + }; + }; +}; + +&nub_spmi_a1 { + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + hpm2_hs: endpoint { + remote-endpoint = <&usb_drd2_hs>; + }; + }; + + port@1 { + reg = <1>; + hpm2_ss: endpoint { + remote-endpoint = <&atc2_phy_ep>; + }; + }; + + port@2 { + reg = <2>; + hpm2_dp: endpoint { + remote-endpoint = <&dcpext0_typec>; + }; + }; + }; + }; + }; +}; + +/* + * ==== The right USB-C port's USB4/Thunderbolt router (ATC2) ==== + * + * Until the boot loader fills them, the router, NHI, PHY CIO and DART tunables and the host + * DROM are this board's, from its ADT (the DROM's router UUID from an earlier boot). + */ +/* ATC2_CIO_AUSS's other parent; kept as the boot loader leaves it */ +&ps_auss { + status = "okay"; +}; + +/* The CIO reconfig request, port 2 (clock gate CIO2_RECONFIG_V) */ +&cio_reconfig0 { + status = "okay"; +}; + +/* The PHY's CIO80 RX_TOP tunables (ADT tunable_LN0/LN1_RX_TOP_CIO_DFLT); the other three are empty */ +&atc2_phy { + apple,tunable-lane0-rx-top-cio = <0x35c 0xe00 0x0>; + apple,tunable-lane1-rx-top-cio = <0x35c 0xe00 0x0>; +}; + +/* Only the router's LSTX pin (81) of the AOP GPIO controller is exposed. */ +&pinctrl_aop { + gpio-reserved-ranges = <0 81>, <82 3>; + status = "okay"; +}; + +&hpm2 { + connector { + ports { + port@3 { + reg = <3>; + hpm2_usb4: endpoint { + remote-endpoint = <&acio2_tbt>; + }; + }; + }; + }; +}; + +&acio2 { + apple,tunable-rc = <0x0 0xfff0 0x4020>, + <0x2c 0x4 0x0>, + <0x4004 0x1ff01ff 0x1000100>, + <0x4008 0x801f 0x4>, + <0x4010 0x7f 0x40>, + <0x4014 0x801f 0x4>, + <0x401c 0x7f 0x40>, + <0x4024 0x801f 0x4>, + <0x402c 0x7f 0x40>, + <0x4030 0x801f 0x4>, + <0x4038 0xffffffff 0x1010101>, + <0x403c 0xffff 0x101>, + <0x4040 0x1 0x1>, + <0x4044 0xffffffff 0x1010101>, + <0x4048 0x1 0x1>, + <0x404c 0x801f 0x4>, + <0x8004 0x70007 0x40004>, + <0x8008 0x801f 0x4>, + <0x8010 0x70007 0x40004>, + <0x8018 0x801f 0x4>, + <0x8020 0x70007 0x40004>, + <0x8028 0x801f 0x4>, + <0x8030 0x801f 0x4>, + <0x8038 0x801f 0x4>, + <0x8040 0x801f 0x4>, + <0x8048 0x801f 0x4>, + <0x8050 0x801f 0x4>, + <0x8058 0x801f 0x4>, + <0x8060 0x801f 0x4>; + apple,tunable-pcie-adapter = <0x1008 0x6 0x6>; + apple,tunable-router-ap-fw = <0x8 0xff2 0x482>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + acio2_tbt: endpoint { + remote-endpoint = <&hpm2_usb4>; + }; + }; + }; +}; + +&acio2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>, + <0x800 0xf003f 0x10000>, + <0x804 0xf003f 0x10002>, + <0x808 0xf003f 0x10004>, + <0x80c 0xf003f 0x20006>, + <0x810 0xf003f 0x2000a>, + <0x814 0xf003f 0x2000e>, + <0x818 0xf003f 0x20012>, + <0x81c 0xf003f 0x20016>, + <0x820 0xf003f 0x2001a>, + <0x824 0xf003f 0x2001e>, + <0x828 0xf003f 0x20022>, + <0x82c 0xf003f 0x20026>, + <0x830 0xf003f 0x2002a>, + <0x834 0xf003f 0x2002e>, + <0x838 0xf003f 0x20032>, + <0x83c 0xf003f 0x10036>; +}; + +&acio2_nhi { + apple,tunable-nhi = <0xf8004 0xff 0x80>, + <0xf8008 0x801f 0x4>, + <0xf8010 0x3f 0x20>, + <0xf8014 0x801f 0x4>, + <0xf801c 0x1f 0x10>, + <0xf8020 0x801f 0x4>, + <0xf8028 0xff 0x80>, + <0xf802c 0x801f 0x4>, + <0xf8034 0xff 0x80>, + <0xf8038 0x801f 0x4>, + <0xf8040 0x3f 0x20>, + <0xf8044 0x801f 0x4>, + <0xf804c 0x8007 0x1>, + <0xfc004 0x3 0x2>, + <0xfc008 0x801f 0x4>, + <0xfc010 0x7 0x4>, + <0xfc014 0x801f 0x4>, + <0xfc01c 0x7 0x4>, + <0xfc020 0x801f 0x4>, + <0xfc028 0x7 0x4>, + <0xfc02c 0x801f 0x4>, + <0xfc034 0x7 0x4>, + <0xfc038 0x801f 0x4>, + <0xfc040 0x3 0x2>, + <0xfc044 0x801f 0x4>, + <0xfc04c 0x8007 0x1>, + <0x4000 0x1000000f 0x10000002>, + <0x4004 0x1000000f 0x10000002>, + <0x4008 0x1000000f 0x10000002>, + <0x400c 0x1000000f 0x10000002>, + <0x4010 0x1000000f 0x10000002>, + <0x4014 0x1000000f 0x10000002>, + <0x4018 0x1000000f 0x10000002>, + <0x401c 0x1000000f 0x10000002>, + <0x4020 0x1000000f 0x10000002>, + <0x4024 0x1000000f 0x10000002>, + <0x4028 0x1000000f 0x10000002>, + <0x402c 0x1000000f 0x10000002>, + <0x4040 0x1000000f 0x10000002>, + <0x4044 0x1000000f 0x10000002>, + <0x4048 0x1000000f 0x10000002>, + <0x404c 0x1000000f 0x10000002>, + <0x4050 0x1000000f 0x10000002>, + <0x4054 0x1000000f 0x10000002>, + <0x4058 0x1000000f 0x10000002>, + <0x405c 0x1000000f 0x10000002>, + <0x4060 0x1000000f 0x10000002>, + <0x4064 0x1000000f 0x10000002>, + <0x4068 0x1000000f 0x10000002>, + <0x406c 0x1000000f 0x10000002>, + <0x4080 0x1000000f 0x10000002>, + <0x4084 0x1000000f 0x10000002>, + <0x4088 0x1000000f 0x10000002>, + <0x408c 0x1000000f 0x10000002>, + <0x4090 0x1000000f 0x10000002>, + <0x4094 0x1000000f 0x10000002>, + <0x4098 0x1000000f 0x10000002>, + <0x409c 0x1000000f 0x10000002>, + <0x40a0 0x1000000f 0x10000002>, + <0x40a4 0x1000000f 0x10000002>, + <0x40a8 0x1000000f 0x10000002>, + <0x40ac 0x1000000f 0x10000002>, + <0x40c0 0x1000000f 0x10000002>, + <0x40c4 0x1000000f 0x10000002>, + <0x40c8 0x1000000f 0x10000002>, + <0x40cc 0x1000000f 0x10000002>, + <0x40d0 0x1000000f 0x10000002>, + <0x40d4 0x1000000f 0x10000002>, + <0x40d8 0x1000000f 0x10000002>, + <0x40dc 0x1000000f 0x10000002>, + <0x40e0 0x1000000f 0x10000002>, + <0x40e4 0x1000000f 0x10000002>, + <0x40e8 0x1000000f 0x10000002>, + <0x40ec 0x1000000f 0x10000002>, + <0x4100 0x1000000f 0x10000002>, + <0x4104 0x1000000f 0x10000002>, + <0x4108 0x1000000f 0x10000002>, + <0x410c 0x1000000f 0x10000002>, + <0x4110 0x1000000f 0x10000002>, + <0x4114 0x1000000f 0x10000002>, + <0x4118 0x1000000f 0x10000002>, + <0x411c 0x1000000f 0x10000002>, + <0x4120 0x1000000f 0x10000002>, + <0x4124 0x1000000f 0x10000002>, + <0x4128 0x1000000f 0x10000002>, + <0x412c 0x1000000f 0x10000002>, + <0x4140 0x1000000f 0x10000002>, + <0x4144 0x1000000f 0x10000002>, + <0x4148 0x1000000f 0x10000002>, + <0x414c 0x1000000f 0x10000002>, + <0x4150 0x1000000f 0x10000002>, + <0x4154 0x1000000f 0x10000002>, + <0x4158 0x1000000f 0x10000002>, + <0x415c 0x1000000f 0x10000002>, + <0x4160 0x1000000f 0x10000002>, + <0x4164 0x1000000f 0x10000002>, + <0x4168 0x1000000f 0x10000002>, + <0x416c 0x1000000f 0x10000002>, + <0x43e4 0x7 0x7>; +}; + +/* ==== The right USB-C port's PCIe tunnel (ATC2): this board's tunables ==== */ +&apciec2_dart { + apple,tunable = <0x20c 0x80000121 0x80000121>, + <0x220 0xf0f 0xf0f>, + <0x224 0xffff 0x909>, + <0x300 0x1f31 0x1>, + <0x308 0x3ffffffc 0x10000000>, + <0x310 0x3ffffffc 0x3ffffffc>; +}; + +&apciec2 { + apple,tunable-debug = <0x1000 0x1 0x0>; + apple,tunable-rc = <0x78 0x7000 0x0>, + <0x718 0x7c000 0x0>, + <0x814 0xffffffff 0x2020f060>, + <0x8bc 0x1 0x0>; + apple,tunable-fabric = <0x4 0x3f003f 0x240024>, + <0x8 0x1 0x1>, + <0xc 0x1 0x1>, + <0x10 0x801f 0x4>, + <0x18 0x1ff 0x1e0>, + <0x1c 0x1 0x1>, + <0x20 0x801f 0x4>, + <0x28 0x1f 0x10>, + <0x2c 0x801f 0x4>, + <0x34 0x3f 0x20>, + <0x38 0x801f 0x4>, + <0x40 0x3f 0x20>, + <0x44 0x801f 0x4>, + <0x4c 0x10001 0x10001>, + <0x50 0x1 0x1>, + <0x54 0x1 0x1>, + <0x58 0x801f 0x4>, + <0x60 0x7f007f 0x450045>, + <0x64 0x801f 0x4>, + <0x6c 0x1f001f 0x100010>, + <0x70 0xf1 0x1>, + <0x74 0x801f 0x4>, + <0x7c 0x801f 0x4>, + <0x84 0x1 0x1>, + <0x88 0x1 0x1>, + <0x8c 0x801f 0x4>, + <0x94 0x1 0x1>, + <0x98 0x801f 0x4>, + <0xa0 0x1 0x1>, + <0xa4 0x801f 0x4>, + <0xac 0x1 0x1>, + <0xb0 0x1 0x1>, + <0xb4 0x801f 0x4>, + <0xbc 0x801f 0x4>, + <0xc4 0x801f 0x4>; +}; + +&apciec2_port { + apple,tunable = <0x140 0x1 0x1>, + <0x2a0 0x7e 0xa>; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t6050-max-pmp-tunables.dtsi new file mode 100644 index 0000000000000..3fd6ff7c7b7ad --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-max-pmp-tunables.dtsi @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 Max PMP firmware properties: the ADT /arm-io/pmp0/iop-pmp0-nub values the + * firmware queries by name at boot (pmp.rs serves apple,tunable-). + * Raw bytes from a J716cAP 27.0 ADT, where iBoot has merged the h17c-family + * variant tables into the nub. The J714cAP and J716cAP firmware ADTs carry + * identical PMP nodes, so the J714c uses the same set. + */ + +&pmp0 { + apple,tunable-afc-turbo-enable = [01 00 00 00]; + apple,tunable-afi-bw-a-enable = [01 00 00 00]; + apple,tunable-afi-bw-enable = [01 00 00 00]; + apple,tunable-afi-bw-threshold = [47 21 58 00 00 00 00 00 00 00 00 00 03 00 00 00 cc 4c ae 00 00 00 00 00 47 21 57 00 03 00 00 00 66 66 e8 00 00 00 00 00 cc 4c ad 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 e7 00 03 00 00 00]; + apple,tunable-afi-bw-threshold-a = [47 21 58 00 00 00 00 00 00 00 00 00 03 00 00 00 cc 4c ae 00 00 00 00 00 47 21 57 00 03 00 00 00 66 66 e8 00 00 00 00 00 cc 4c ad 00 03 00 00 00 66 66 e8 00 00 00 00 00 66 66 e7 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 e7 00 03 00 00 00]; + apple,tunable-agx-af-bw-a-enable = [00 00 00 00]; + apple,tunable-agx-af-bw-constraint = [00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00]; + apple,tunable-agx-af-bw-constraint-a = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00]; + apple,tunable-agx-af-bw-constraint-a-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-af-bw-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 d6 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 02 00 00 00 00 00 00 d5 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 80 5b 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 80 5a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 e1 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 00 e0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 bc 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 bb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5e 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-agx-af-bw-threshold-a = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 d6 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 02 00 00 00 00 00 00 d5 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 80 5b 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 80 5a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 e1 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 00 e0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 bc 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 bb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5e 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-agx-af-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-af-lat-enable = [00 00 00 00]; + apple,tunable-agx-afr-bw-constraint = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 01 00 00 00 01 00 00 00 0c 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 03 00 00 00 03 00 00 00 0c 00 00 00 04 00 00 00 04 00 00 00 0c 00 00 00 05 00 00 00 05 00 00 00 0c 00 00 00 06 00 00 00 06 00 00 00 0c 00 00 00 07 00 00 00 07 00 00 00 0c 00 00 00 08 00 00 00 08 00 00 00 0c 00 00 00 09 00 00 00 09 00 00 00 0c 00 00 00 0a 00 00 00 0a 00 00 00 0c 00 00 00 0b 00 00 00 0b 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00]; + apple,tunable-agx-afr-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-afr-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-afr-bw-enable = [01 00 00 00]; + apple,tunable-agx-afr-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 96 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 e9 00 00 00 00 00 5c 8f 95 00 00 00 00 00 d7 e3 48 01 00 00 00 00 66 e6 e8 00 00 00 00 00 f5 28 b8 01 00 00 00 00 d7 e3 47 01 00 00 00 00 99 d9 0a 02 00 00 00 00 f5 28 b7 01 00 00 00 00 d7 23 69 02 00 00 00 00 99 d9 09 02 00 00 00 00 c2 f5 cb 02 00 00 00 00 d7 23 68 02 00 00 00 00 cc 0c 37 03 00 00 00 00 c2 f5 ca 02 00 00 00 00 33 b3 a7 03 00 00 00 00 cc 0c 36 03 00 00 00 00 e1 3a 1f 04 00 00 00 00 33 b3 a6 03 00 00 00 00 d7 a3 9d 04 00 00 00 00 e1 3a 1e 04 00 00 00 00 1e 05 2d 05 00 00 00 00 d7 a3 9c 04 00 00 00 00 00 00 ff ff 00 00 00 00 1e 05 2c 05 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 d1 00 00 00 00 00 00 00 00 00 06 00 00 00 7a 54 28 01 00 00 00 00 1e 05 d0 00 06 00 00 00 66 66 8f 01 00 00 00 00 7a 54 27 01 06 00 00 00 d7 63 dd 01 00 00 00 00 66 66 8e 01 06 00 00 00 5c 8f 36 02 00 00 00 00 d7 63 dc 01 06 00 00 00 c2 75 94 02 00 00 00 00 5c 8f 35 02 06 00 00 00 ae 87 fa 02 00 00 00 00 c2 75 93 02 06 00 00 00 1e 45 66 03 00 00 00 00 ae 87 f9 02 06 00 00 00 47 e1 d8 03 00 00 00 00 1e 45 65 03 06 00 00 00 47 61 52 04 00 00 00 00 47 e1 d7 03 06 00 00 00 51 38 dc 04 00 00 00 00 47 61 51 04 06 00 00 00 00 00 ff ff 00 00 00 00 51 38 db 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 3d ca 14 01 00 00 00 00 00 00 00 00 06 00 00 00 33 f3 76 01 00 00 00 00 3d ca 13 01 06 00 00 00 28 1c c2 01 00 00 00 00 33 f3 75 01 06 00 00 00 c2 35 18 02 00 00 00 00 28 1c c1 01 06 00 00 00 f5 28 73 02 00 00 00 00 c2 35 17 02 06 00 00 00 51 38 d6 02 00 00 00 00 f5 28 72 02 06 00 00 00 8f 02 3f 03 00 00 00 00 51 38 d5 02 06 00 00 00 85 ab ae 03 00 00 00 00 8f 02 3e 03 06 00 00 00 51 38 25 04 00 00 00 00 85 ab ad 03 06 00 00 00 00 c0 ab 04 00 00 00 00 51 38 24 04 06 00 00 00 00 00 ff ff 00 00 00 00 00 c0 aa 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 66 a6 66 01 00 00 00 00 00 00 00 00 06 00 00 00 85 eb af 01 00 00 00 00 66 a6 65 01 06 00 00 00 51 f8 03 02 00 00 00 00 85 eb ae 01 06 00 00 00 c2 f5 5c 02 00 00 00 00 51 f8 02 02 06 00 00 00 8f 02 be 02 00 00 00 00 c2 f5 5b 02 06 00 00 00 7a d4 24 03 00 00 00 00 8f 02 bd 02 06 00 00 00 ae 87 92 03 00 00 00 00 7a d4 23 03 06 00 00 00 b8 1e 07 04 00 00 00 00 ae 87 91 03 06 00 00 00 14 6e 8b 04 00 00 00 00 b8 1e 06 04 06 00 00 00 00 00 ff ff 00 00 00 00 14 6e 8a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 85 eb af 01 00 00 00 00 00 00 00 00 06 00 00 00 51 f8 03 02 00 00 00 00 85 eb ae 01 06 00 00 00 c2 f5 5c 02 00 00 00 00 51 f8 02 02 06 00 00 00 8f 02 be 02 00 00 00 00 c2 f5 5b 02 06 00 00 00 7a d4 24 03 00 00 00 00 8f 02 bd 02 06 00 00 00 ae 87 92 03 00 00 00 00 7a d4 23 03 06 00 00 00 b8 1e 07 04 00 00 00 00 ae 87 91 03 06 00 00 00 14 6e 8b 04 00 00 00 00 b8 1e 06 04 06 00 00 00 00 00 ff ff 00 00 00 00 14 6e 8a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 70 bd ef 01 00 00 00 00 00 00 00 00 06 00 00 00 8f c2 46 02 00 00 00 00 70 bd ee 01 06 00 00 00 5c cf a5 02 00 00 00 00 8f c2 45 02 06 00 00 00 f5 a8 0a 03 00 00 00 00 5c cf a4 02 06 00 00 00 d7 63 76 03 00 00 00 00 f5 a8 09 03 06 00 00 00 8f 02 e9 03 00 00 00 00 d7 63 75 03 06 00 00 00 28 1c 6b 04 00 00 00 00 8f 02 e8 03 06 00 00 00 00 00 ff ff 00 00 00 00 28 1c 6a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 5c 8f 30 02 00 00 00 00 00 00 00 00 06 00 00 00 99 99 8d 02 00 00 00 00 5c 8f 2f 02 06 00 00 00 70 7d f0 02 00 00 00 00 99 99 8c 02 06 00 00 00 8f 42 5a 03 00 00 00 00 70 7d ef 02 06 00 00 00 f5 e8 ca 03 00 00 00 00 8f 42 59 03 06 00 00 00 3d ca 4a 04 00 00 00 00 f5 e8 c9 03 06 00 00 00 00 00 ff ff 00 00 00 00 3d ca 49 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 d7 63 75 02 00 00 00 00 00 00 00 00 06 00 00 00 5c 4f d6 02 00 00 00 00 d7 63 74 02 06 00 00 00 b8 1e 3e 03 00 00 00 00 5c 4f d5 02 06 00 00 00 cc cc ac 03 00 00 00 00 b8 1e 3d 03 06 00 00 00 e1 7a 2a 04 00 00 00 00 cc cc ab 03 06 00 00 00 00 00 ff ff 00 00 00 00 e1 7a 29 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 d7 23 bc 02 00 00 00 00 00 00 00 00 06 00 00 00 e1 fa 21 03 00 00 00 00 d7 23 bb 02 06 00 00 00 a3 b0 8e 03 00 00 00 00 e1 fa 20 03 06 00 00 00 f5 28 0a 04 00 00 00 00 a3 b0 8d 03 06 00 00 00 00 00 ff ff 00 00 00 00 f5 28 09 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a d7 05 03 00 00 00 00 00 00 00 00 06 00 00 00 0a 97 70 03 00 00 00 00 0a d7 04 03 06 00 00 00 0a d7 e9 03 00 00 00 00 0a 97 6f 03 06 00 00 00 00 00 ff ff 00 00 00 00 0a d7 e8 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 eb 51 25 03 00 00 00 00 00 00 00 00 06 00 00 00 cc 0c 99 03 00 00 00 00 eb 51 24 03 06 00 00 00 00 00 ff ff 00 00 00 00 cc 0c 98 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 40 48 03 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 40 47 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00]; + apple,tunable-agx-afr-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-afr-lat-enable = [00 00 00 00]; + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 02 00 00 00 06 00 00 00 03 00 00 00 03 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00]; + apple,tunable-agx-dcs-bw-constraint-enable = [01 00 00 00]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-dcs-bw-enable = [01 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 44 00 00 00 00 00 00 00 d2 00 00 00 00 00 00 00 8f 00 00 00 00 00 cc cc 47 01 00 00 00 00 00 00 d1 00 00 00 00 00 00 00 ff ff 00 00 00 00 cc cc 46 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 2e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 2d 00 06 00 00 00 00 00 d2 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 d1 00 06 00 00 00 00 00 ff ff 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 be 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 bd 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 be 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 bd 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 86 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a0 00 00 00 00 00 00 00 85 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 9f 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 51 f8 18 01 00 00 00 00 00 00 13 00 06 00 00 00 00 00 82 01 00 00 00 00 51 f8 17 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 81 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 51 f8 18 01 00 00 00 00 00 00 13 00 06 00 00 00 00 00 82 01 00 00 00 00 51 f8 17 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 81 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 69 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 8e 00 00 00 00 00 00 00 68 00 06 00 00 00 00 00 b9 00 00 00 00 00 00 00 8d 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 b8 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 a3 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 8b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 8b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 06 00 00 00]; + apple,tunable-agx-dcs-lat-constraint-enable = [01 00 00 00]; + apple,tunable-agx-dcs-lat-enable = [00 00 00 00]; + apple,tunable-agx-dvfs-state-voltages = [7d 00 00 00 53 02 00 00 8a 02 00 00 b2 02 00 00 e4 02 00 00 ee 02 00 00 07 03 00 00 20 03 00 00 34 03 00 00 57 03 00 00 7f 03 00 00 a7 03 00 00 f7 03 00 00 1a 04 00 00 7d 00 00 00 7d 00 00 00]; + apple,tunable-ane-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ane-af-bw-enable = [00 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-af-bw-threshold-a = [00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-ane-dcs-bw-enable = [01 00 00 00]; + apple,tunable-ane-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 46 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-ans-dcs-bw-enable = [01 00 00 00]; + apple,tunable-ans-dcs-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 5c 0f 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-dma-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ans-dma-af-bw-enable = [01 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-nand-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ans-nand-af-bw-enable = [01 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-a-enable = [01 00 00 00]; + apple,tunable-atc0-af-bw-enable = [01 00 00 00]; + apple,tunable-atc0-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc1-af-bw-a-enable = [01 00 00 00]; + apple,tunable-atc1-af-bw-enable = [01 00 00 00]; + apple,tunable-atc1-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc1-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc2-af-bw-a-enable = [01 00 00 00]; + apple,tunable-atc2-af-bw-enable = [01 00 00 00]; + apple,tunable-atc2-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc2-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc3-af-bw-a-enable = [01 00 00 00]; + apple,tunable-atc3-af-bw-enable = [01 00 00 00]; + apple,tunable-atc3-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc3-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-auss-af-bw-a-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-auss-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-a-enable = [00 00 00 00]; + apple,tunable-ave0-af-bw-enable = [00 00 00 00]; + apple,tunable-ave0-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave1-af-bw-a-enable = [00 00 00 00]; + apple,tunable-ave1-af-bw-enable = [00 00 00 00]; + apple,tunable-ave1-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave1-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-controller = [01 00 00 00 10 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 0f 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 4e 41 4e 44 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 4d 41 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 17 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 21 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 56 4c 50 4d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 20 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-dcs-bw-enable = [01 00 00 00]; + apple,tunable-dcs-bw-threshold = [00 c0 4f 00 01 00 00 00 00 00 00 00 00 00 00 00 5c 8f 9f 00 02 00 00 00 00 c0 4e 00 00 00 00 00 eb 11 e8 00 03 00 00 00 5c 8f 9e 00 00 00 00 00 f5 28 5c 01 04 00 00 00 eb 11 e7 00 00 00 00 00 66 26 96 01 05 00 00 00 f5 28 5b 01 00 00 00 00 33 33 d0 01 06 00 00 00 66 26 95 01 00 00 00 00 00 00 ff ff 07 00 00 00 33 33 cf 01 00 00 00 00]; + apple,tunable-disable-vlpm-feature = [00 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 07 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 0d 00 00 00 21 00 00 00 41 46 52 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 49 4e 54 00 00 00 00 00 00 00 00 05 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 45 58 54 00 00 00 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-firmware-name = [74 36 30 35 30 70 6d 70 00]; + apple,tunable-jpeg-af-bw-a-enable = [00 00 00 00]; + apple,tunable-jpeg-af-bw-enable = [00 00 00 00]; + apple,tunable-jpeg-af-bw-threshold = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-jpeg-af-bw-threshold-a = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-lts-acsk-tddb-config = [01 00 00 00 69 00 00 00 10 14 24 00 00 00 00 00 95 c2 0f 00 1a 04 00 00 56 04 00 00 00 00 00 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 d2 03 00 00 00 00 66 00 33 33 66 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-lts-config = [03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 01 00 00 00 69 00 00 00 bf d9 01 00 00 00 00 00 da fc 0f 00 6e 00 00 00 66 00 00 00 00 00 11 00 33 b3 00 00 f9 fe c3 00 02 00 00 00 3b 01 00 00 a6 79 0a 00 00 00 00 00 2b ee 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 97 6e 39 00]; + apple,tunable-lts-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 0d 00 00 00 09 00 00 00 4d 41 43 43 30 00 00 00 02 00 00 00 03 00 00 00 0e 00 00 00 0a 00 00 00 4d 41 43 43 31 00 00 00 03 00 00 00 03 00 00 00 0f 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 04 00 00 00 01 00 00 00 11 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 05 00 00 00 03 00 00 00 10 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-mcache-stream = [1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 28 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00]; + apple,tunable-pre-loaded = [01 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 4e 55 4c 4c 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 10 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 0c 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 10 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 18 00 00 00 20 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 38 00 00 00 48 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 08 00 00 00 80 00 00 00 10 00 00 00 00 00 00 00 50 4d 53 2d 50 57 52 53 2d 50 4b 54 00 00 00 00 09 00 00 00 90 00 00 00 50 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 e0 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 e8 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 f0 01 00 00 30 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 20 02 00 00 1a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 3a 02 00 00 02 00 00 00 00 00 00 00 44 56 46 53 2d 44 49 45 2d 54 4f 2d 44 49 45 00 0f 00 00 00 3c 02 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 10 00 00 00 58 02 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 11 00 00 00 5a 02 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 12 00 00 00 61 02 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 13 00 00 00 6a 02 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 18 00 00 00 70 02 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 19 00 00 00 75 02 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 1a 00 00 00 7a 02 00 00 20 00 00 00 00 00 00 00 47 46 58 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 1b 00 00 00 9a 02 00 00 09 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 1c 00 00 00 a3 02 00 00 01 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 53 4d 43 00 00 1d 00 00 00 a4 02 00 00 08 00 00 00 00 00 00 00 41 4e 45 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 20 00 00 00 b4 02 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 21 00 00 00 b5 02 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 22 00 00 00 b6 02 00 00 08 00 00 00 00 00 00 00 50 4d 55 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 28 00 00 00 20 03 00 00 14 00 00 00 00 00 00 00 50 4d 53 2d 56 49 52 54 2d 43 4e 54 52 00 00 00]; + apple,tunable-region-base = [00 00 50 84 02 00 00 00]; + apple,tunable-region-size = [00 00 10 00 00 00 00 00]; + apple,tunable-segment-names = [5f 5f 54 45 58 54 3b 5f 5f 44 41 54 41 3b 5f 5f 45 54 45 58 54 00]; + apple,tunable-segment-ranges = [00 00 50 84 02 00 00 00 00 00 00 01 00 00 00 00 00 00 50 84 02 00 00 00 00 80 05 00 03 00 00 00 00 80 55 84 02 00 00 00 00 80 05 01 00 00 00 00 00 80 55 84 02 00 00 00 00 b0 09 00 06 00 00 00 00 c0 e5 01 00 01 00 00 00 00 10 01 00 00 00 00 00 00 00 00 00 01 00 00 00 00 01 00 01 00 00 00]; + apple,tunable-soc-chip-variant = [01 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 03 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 04 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 05 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 06 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 08 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 09 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 00 0a 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 00 0b 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 00 0d 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 0e 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 0f 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 10 00 00 00 5f 58 46 47 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 11 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 12 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 13 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 49 4e 54 00 14 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 15 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 16 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 32 17 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 33 18 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 00 00 00 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 00 00 00 00 1a 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 1b 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 30 00 00 00 00 1c 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 31 00 00 00 00 1d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 1e 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1f 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 52 4f 52 45 53 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 22 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 23 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 24 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 00 00 00 00 25 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00]; + apple,tunable-soc-device-ps-group = [00 f0 ff ff ff 01 00 00]; + apple,tunable-soc-ni0-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni0-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni0-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni0-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni1-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni1-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni1-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni1-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni2-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni2-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni2-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni2-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni3-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni3-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni4-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni4-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni4-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni4-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni5-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni5-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni6-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni6-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni6-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-push-thr = [24 00 00 00]; + apple,tunable-soc-ni7-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni7-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni7-bw-threshold = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni7-bw-threshold-a = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni7-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni8-bw-a-enable = [00 00 00 00]; + apple,tunable-soc-ni8-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni8-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni8-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni8-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni9-bw-a-enable = [00 00 00 00]; + apple,tunable-soc-ni9-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni9-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-push-thr = [f5 06 00 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 19 00 00 00 1a 00 00 00 1b 00 00 00 1c 00 00 00 1d 00 00 00 1e 00 00 00 1f 00 00 00 20 00 00 00 21 00 00 00 22 00 00 00 23 00 00 00 24 00 00 00 25 00 00 00 26 00 00 00 27 00 00 00 28 00 00 00 29 00 00 00 2a 00 00 00 2b 00 00 00 2c 00 00 00 2d 00 00 00 2e 00 00 00 2f 00 00 00 30 00 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00 35 00 00 00 36 00 00 00 37 00 00 00 38 00 00 00 39 00 00 00 3a 00 00 00 3b 00 00 00 3c 00 00 00 3d 00 00 00 3e 00 00 00 3f 00 00 00 40 00 00 00 41 00 00 00 42 00 00 00 43 00 00 00 44 00 00 00 45 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 63 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 64 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 66 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 67 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 65 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 32 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 68 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 35 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 0a 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 73 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 62 69 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 38 69 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 33 69 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 34 69 00 00 00 00 00 00 00 00 00 00 20 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 30 69 00 00 00 00 00 00 00 00 00 00 21 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 31 69 00 00 00 00 00 00 00 00 00 00 22 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 36 69 00 00 00 00 00 00 00 00 00 00 23 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 37 69 00 00 00 00 00 00 00 00 00 00 24 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 35 69 00 00 00 00 00 00 00 00 00 00 25 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 32 69 00 00 00 00 00 00 00 00 00 00 26 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 38 69 00 00 00 00 00 00 00 00 00 00 27 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 28 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 29 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 39 69 00 00 00 00 00 00 00 00 00 00 2a 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 61 69 00 00 00 00 00 00 00 00 00 00 2b 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 63 69 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 64 69 00 00 00 00 00 00 00 00 00 00 2d 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 66 69 00 00 00 00 00 00 00 00 00 00 2e 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 67 69 00 00 00 00 00 00 00 00 00 00 2f 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 62 69 00 00 00 00 00 00 00 00 00 00 30 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 65 69 00 00 00 00 00 00 00 00 00 00 31 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 68 69 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 33 69 00 00 00 00 00 00 00 00 00 00 33 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 34 69 00 00 00 00 00 00 00 00 00 00 34 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 30 69 00 00 00 00 00 00 00 00 00 00 35 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 31 69 00 00 00 00 00 00 00 00 00 00 36 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 36 69 00 00 00 00 00 00 00 00 00 00 37 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 37 69 00 00 00 00 00 00 00 00 00 00 38 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 35 69 00 00 00 00 00 00 00 00 00 00 39 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 32 69 00 00 00 00 00 00 00 00 00 00 3a 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 38 69 00 00 00 00 00 00 00 00 00 00 3b 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6c 69 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6d 69 00 00 00 00 00 00 00 00 00 00 3d 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 39 69 00 00 00 00 00 00 00 00 00 00 3e 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 61 69 00 00 00 00 00 00 00 00 00 00 3f 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 63 69 00 00 00 00 00 00 00 00 00 00 40 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 64 69 00 00 00 00 00 00 00 00 00 00 41 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 66 69 00 00 00 00 00 00 00 00 00 00 42 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 67 69 00 00 00 00 00 00 00 00 00 00 43 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 62 69 00 00 00 00 00 00 00 00 00 00 44 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 65 69 00 00 00 00 00 00 00 00 00 00 45 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 68 69 00 00 00 00 00 00 00 00 00 00 46 00 00 00 01 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-vlpm-threshold = [55 01 00 00]; + apple,tunable-vlpm-timer-limit = [64 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index c04b66519953c..628004394bc33 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -116,7 +116,7 @@ #interrupt-cells = <2>; }; -#ifdef T6050_J716C +#if defined(T6050_J714C) || defined(T6050_J716C) /* * /arm-io/nub-spmi-a1: reg[0] 0x8a1a8000, generation 4, a bus of USB-C PD controllers (the * board's). They are polled, as on nub-spmi-a0, so the bus takes no interrupt. @@ -130,7 +130,7 @@ #endif }; -#ifdef T6050_J716C +#if defined(T6050_J714C) || defined(T6050_J716C) /* * ATC2's USB4/Thunderbolt router (ADT /arm-io/acio2, acio-cpu2 and dart-acio2), powered per * connection: the PHY enters CIO mode, then the router's domains come on (ATC2_CIO, _PCIE, _USB, diff --git a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi index 8063b37f816db..fea335f3693a6 100644 --- a/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-peripherals.dtsi @@ -1321,7 +1321,7 @@ }; #endif -#if defined(T6050_J714C) || defined(T6050_J714S) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) +#if defined(T6050_J714S) || defined(T6050_K114C) || defined(T6050_K114S) || defined(T6050_K116C) || defined(T6050_K116S) nub_spmi_a1: spmi@28a1a8000 { compatible = "apple,t6050-spmi", "apple,t8142-spmi"; reg = <0x2 0x8a1a8000 0x0 0x4000>, @@ -1471,7 +1471,7 @@ status = "disabled"; }; -#ifndef T6050_J716C +#if !defined(T6050_J714C) && !defined(T6050_J716C) apciec2_dart: iommu@384000000 { compatible = "apple,t6050-dart", "apple,t8110-dart"; reg = <0x3 0x84000000 0x0 0x20000>; From cbc8563f7bb946e1f487fd07ebf0c716a30957a5 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:13:49 +1100 Subject: [PATCH 361/392] arm64: dts: apple: t8140: expose the debug dock and firmware UARTs (#1396) Enable the debug dock connection status register (read through the apple,debug-dock debugfs view). On J700, enable uart4 and uart5, which carry the Wi-Fi and Bluetooth firmware debug output, with their receive pins muxed. Neither is a console; the ports stay closed until a user opens them. The PMGR clock provider is not enabled on T8140 yet, so both are clocked from the 24 MHz reference. dtc W=1 and CHECK_DTBS report nothing for the changed nodes. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140-j700.dts | 28 +++++++++++++++++++ .../boot/dts/apple/t8140-peripherals.dtsi | 3 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 4b21c9b2401af..ba91829924008 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -2047,6 +2047,34 @@ pcie0_clkreq: pcie0-clkreq-pins { pinmux = ; }; + + uart4_rx: uart4-rx-pins { + pinmux = ; + }; + + uart5_rx: uart5-rx-pins { + pinmux = ; + }; +}; + +/* + * Wi-Fi (uart4) and Bluetooth (uart5) firmware debug output. Only the + * receive line is routed; the ports stay closed unless a user opens them. + */ +&uart4 { + pinctrl-0 = <&uart4_rx>; + pinctrl-names = "default"; + /* The PMGR clock provider is not enabled on this SoC yet. */ + clocks = <&clkref>, <&clkref>; + status = "okay"; +}; + +&uart5 { + pinctrl-0 = <&uart5_rx>; + pinctrl-names = "default"; + /* The PMGR clock provider is not enabled on this SoC yet. */ + clocks = <&clkref>, <&clkref>; + status = "okay"; }; /* SID17 bootstrap; the arena stays owned until a full reset. */ diff --git a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi index f73a86c0f4326..f1d24c8d495f3 100644 --- a/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-peripherals.dtsi @@ -505,14 +505,13 @@ status = "disabled"; }; - /* ADT /dock-connect; no qualified host transport/ownership contract. */ + /* ADT /dock-connect: debug dock connection status, read only. */ debug_dock: debug-dock@3082d8008 { compatible = "apple,t8140-debug-dock", "apple,debug-dock"; reg = <0x3 0x082d8008 0x0 0x4>; interrupt-parent = <&aic>; interrupts = , ; - status = "disabled"; }; /* ADT /arm-io/eic named MMIO apertures are absolute addresses. */ From c880586efacb808b1cf5406d6079b76bf07d8722 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:20:36 +1100 Subject: [PATCH 362/392] arm64: dts: apple: attach the Neo internal PCIe host to its full power chain (#1397) The T8140 ADT gates APCIE_GP, APCIE_SYS_GP, APCIE_ST, APCIE_SYS_ST, ANS and APCIE_PHY_SW for apcie0. APCIE_PHY_SW already has all of the other domains as ancestors, so attach the host to that leaf instead of APCIE_GP alone. This removes its dependency on NVMe keeping the storage and PHY chain on. The host retains this chain during suspend-to-idle while the radio wake path leaves the internal link running. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index b781dd756a2d2..4242e655a3af7 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1206,7 +1206,7 @@ #size-cells = <2>; ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; - power-domains = <&ps_apcie_gp>; + power-domains = <&ps_apcie_phy_sw>; status = "disabled"; pcie0_port0: pci@0,0 { From 68c10dd872afe361fa38f98100059f19ce66714a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 19:33:37 -0500 Subject: [PATCH 363/392] arm64: dts: apple: t6050: Describe the CPU clusters The T6050 Max CPU nodes had no cpu-map, so Linux sees eighteen CPUs in one package with no clusters. parse_dt_topology() also reads each CPU's capacity-dmips-mhz only while it walks cpu-map, so without one no capacity reaches the scheduler. Describe the three clusters as the ADT does: MPIDR Aff1 0 and 1 are the M clusters, 2 the P cluster, each with its own L2. The boot loader prunes the entries of cores it did not start. The two-die description keeps its flat layout for now. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-max.dtsi | 69 ++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-max.dtsi b/arch/arm64/boot/dts/apple/t6050-max.dtsi index 628004394bc33..203155df9ceb9 100644 --- a/arch/arm64/boot/dts/apple/t6050-max.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-max.dtsi @@ -45,6 +45,75 @@ T6050_P_CPU(cpu15, 10203, cpufreq_pc); T6050_P_CPU(cpu16, 10204, cpufreq_pc); T6050_P_CPU(cpu17, 10205, cpufreq_pc); + + /* + * The clusters are MPIDR Aff1 (M0, M1, then P), each with its own L2. The boot + * loader prunes the entries of cores it did not start. + */ + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + core1 { + cpu = <&cpu1>; + }; + core2 { + cpu = <&cpu2>; + }; + core3 { + cpu = <&cpu3>; + }; + core4 { + cpu = <&cpu4>; + }; + core5 { + cpu = <&cpu5>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu6>; + }; + core1 { + cpu = <&cpu7>; + }; + core2 { + cpu = <&cpu8>; + }; + core3 { + cpu = <&cpu9>; + }; + core4 { + cpu = <&cpu10>; + }; + core5 { + cpu = <&cpu11>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu12>; + }; + core1 { + cpu = <&cpu13>; + }; + core2 { + cpu = <&cpu14>; + }; + core3 { + cpu = <&cpu15>; + }; + core4 { + cpu = <&cpu16>; + }; + core5 { + cpu = <&cpu17>; + }; + }; + }; }; }; From 28784bf850534c294a361a1c2ce90c36c4604f4f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 19:33:37 -0500 Subject: [PATCH 364/392] arm64: dts: apple: t6050: Give the M and P cores their capacities Without capacity-dmips-mhz every T6050 CPU reports a capacity of 1024, so the scheduler treats the six P cores (apple,everest, CPUs 12-17) like the twelve M cores and an unpinned single heavy thread lands on an M core most of the time (4 of 5 runs on J714sAP). Measured on J714sAP with every cluster pinned to 4284 MHz, which both OPP tables have, and each core's clock read with a dependent-add chain: per MHz an M core runs zstd -3 compression at 0.736, decompression at 0.871 and a Python dict loop at 0.728 of a P core (geometric mean 0.776). Small L1-resident integer, branch, sort, hash and FP loops run at 0.92-1.01, which shows the execution units match but not how general code runs. So M 794 and P 1024; with the 4380 and 4608 MHz tops the M cores' capacity is 754. The values take effect through the cpu-map the previous commit adds. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-base.dtsi | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-base.dtsi b/arch/arm64/boot/dts/apple/t6050-base.dtsi index 1be5e14cb12fe..f77f1d641051c 100644 --- a/arch/arm64/boot/dts/apple/t6050-base.dtsi +++ b/arch/arm64/boot/dts/apple/t6050-base.dtsi @@ -41,7 +41,7 @@ * number the cores 18 * d + 6 * c + n. The boot loader removes the nodes of * the cores it did not start. */ -#define T6050_CPU(label, mpidr, type, opp, cluster) \ +#define T6050_CPU(label, mpidr, type, opp, cluster, cpu_cap) \ label: cpu@mpidr { \ compatible = type; \ device_type = "cpu"; \ @@ -49,13 +49,20 @@ enable-method = "spin-table"; \ operating-points-v2 = <&opp>; \ performance-domains = <&cluster>; \ + capacity-dmips-mhz = ; \ cpu-release-addr = <0 0>; /* to be filled by the loader */ \ } +/* + * capacity-dmips-mhz from pinned single-thread throughput with every cluster at 4284 MHz (in + * both OPP tables), normalised by each core's measured clock (J714sAP): per MHz an M core runs + * zstd -3 compression at 0.736, decompression at 0.871 and a Python dict loop at 0.728 of a P + * core, geometric mean 0.776, so M 794 and P 1024. Small L1-resident loops run at 0.92-1.01. + */ #define T6050_M_CPU(label, mpidr, cluster) \ - T6050_CPU(label, mpidr, "apple,sawtooth", mcluster_opp, cluster) + T6050_CPU(label, mpidr, "apple,sawtooth", mcluster_opp, cluster, 794) #define T6050_P_CPU(label, mpidr, cluster) \ - T6050_CPU(label, mpidr, "apple,everest", pcluster_opp, cluster) + T6050_CPU(label, mpidr, "apple,everest", pcluster_opp, cluster, 1024) / { compatible = "apple,t6050", "apple,arm-platform"; From aa8e96db997f91b21f453d774f4bd84c7569deb1 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:50:55 +1100 Subject: [PATCH 365/392] drm/asahi: g17: back off between refused render capacity attempts (#1400) wait_render_capacity() retried at once whenever reclaim reported that it returned something. Under lower-window starvation reclaim keeps reporting returns (vacated pools the firmware has not released yet, or backing that another refused publisher takes first), so a publisher could make millions of placement attempts, each vacating idle pools and notifying the firmware, until its 2 s deadline. Each reclaim also notified the render domain, which ended the other refused publishers' waits at once. Allow one immediate retry after a returning reclaim per pause, and make every later refused attempt sleep for at least a backoff pause, starting at 50 us and doubling up to 2 ms, even when a notification ends the domain wait sooner. The 2 s bound is unchanged; a starved wait now makes about 3,000 attempts instead of one per reclaim. g17-capacity-backoff-tests.py runs the production wait against a fake clock: a reclaim storm and a quiet domain both end with EBUSY at 2.000 s (3,028 and 401 attempts), a release 5 ms in is placed at 5.0 ms, and the pre-change retry rule and an unpaused wake both fail it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../drm/asahi/g17/runtime/render/submit.rs | 89 +++++++++++- tools/asahi/g17-capacity-backoff-tests.py | 133 ++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 3 files changed, 219 insertions(+), 4 deletions(-) create mode 100755 tools/asahi/g17-capacity-backoff-tests.py diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 970035547dfee..c6c6b615ecd1a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,7 +8,11 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the @@ -275,6 +279,13 @@ impl Backend { /// Render capacity is short: return what idle owners hold, then block until /// a pair, lease or owner returns. No packet is firmware-visible here, and /// neither the device mutex nor a preparation lease is held while waiting. + /// + /// A reclaim that returned something retries at once, but only once per + /// pause: reclaim can keep reporting returns (vacated pools the firmware has + /// not released yet, or backing another refused publisher takes first) + /// while the refusal persists. Every later refusal sleeps for at least the + /// clock's backoff pause, even when a notification (often another refused + /// publisher's own reclaim) ends the domain wait sooner. fn wait_render_capacity( &self, observed: &mut Option, @@ -284,7 +295,7 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if self.reclaim_render_capacity()? { + if self.reclaim_render_capacity()? && clock.fast_retry() { *observed = None; return Ok(()); } @@ -294,15 +305,22 @@ impl Backend { }; self.shared.queue_events(); self.admission_open()?; + let started = super::super::now_ns(); let remaining = self .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? - .saturating_sub(super::super::now_ns()); + .saturating_sub(started); if remaining == 0 { return Err(EBUSY); } + let pause = clock.pause_ns(remaining); self.shared .renders - .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false)?; + let left = pause.saturating_sub(super::super::now_ns().saturating_sub(started)); + if left != 0 { + fsleep(Delta::from_nanos(left as i64)); + } + Ok(()) } /// Gives idle render backing back under capacity pressure. Returns whether @@ -742,11 +760,21 @@ fn recovery_wait_ns() -> u64 { u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 } +/// First pause of a render capacity wait (WaitClock::pause_ns). +const CAPACITY_PAUSE_MIN_NS: u64 = 50_000; +/// Longest pause of a render capacity wait: a 2 s wait makes at most about +/// a thousand attempts. +const CAPACITY_PAUSE_MAX_NS: u64 = 2_000_000; + /// One bounded wait (Backend::wait_deadline). #[derive(Default)] struct WaitClock { first: Option, calm: u64, + /// Pauses this wait has slept (pause_ns). + pauses: u32, + /// The immediate retry after a reclaim is used up until the next pause. + retried: bool, } impl WaitClock { @@ -760,6 +788,24 @@ impl WaitClock { .saturating_add(recovery_wait_ns()) .min(self.calm.saturating_add(bound_ns)) } + + /// Whether a refusal after a reclaim that returned something retries at + /// once: the first time, and again only after a pause. + fn fast_retry(&mut self) -> bool { + !core::mem::replace(&mut self.retried, true) + } + + /// The least time the next refused attempt sleeps: CAPACITY_PAUSE_MIN_NS, + /// doubling per pause up to CAPACITY_PAUSE_MAX_NS, never past `remaining`. + fn pause_ns(&mut self, remaining: u64) -> u64 { + let pause = CAPACITY_PAUSE_MIN_NS + .checked_shl(self.pauses) + .filter(|pause| *pause <= CAPACITY_PAUSE_MAX_NS) + .unwrap_or(CAPACITY_PAUSE_MAX_NS); + self.pauses = self.pauses.saturating_add(1); + self.retried = false; + pause.min(remaining) + } } #[kunit_tests(asahi_g17_wait_clock)] @@ -790,4 +836,39 @@ mod tests { let limit = 100 + recovery_wait_ns(); assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); } + + #[test] + fn a_reclaim_retries_at_once_only_once_per_pause() { + let mut clock = WaitClock::default(); + assert!(clock.fast_retry()); + assert!(!clock.fast_retry()); + clock.pause_ns(BOUND); + assert!(clock.fast_retry()); + assert!(!clock.fast_retry()); + } + + #[test] + fn pauses_double_up_to_the_cap_and_stop_at_the_deadline() { + let mut clock = WaitClock::default(); + assert_eq!(clock.pause_ns(BOUND), CAPACITY_PAUSE_MIN_NS); + assert_eq!(clock.pause_ns(BOUND), 2 * CAPACITY_PAUSE_MIN_NS); + for _ in 0..80 { + assert!(clock.pause_ns(BOUND) <= CAPACITY_PAUSE_MAX_NS); + } + assert_eq!(clock.pause_ns(BOUND), CAPACITY_PAUSE_MAX_NS); + assert_eq!(clock.pause_ns(1_000), 1_000); + } + + #[test] + fn a_wait_that_always_wakes_makes_bounded_attempts() { + let mut clock = WaitClock::default(); + let (mut now, mut attempts) = (0, 0); + while now < BOUND { + attempts += 1; + if !clock.fast_retry() { + now += clock.pause_ns(BOUND - now); + } + } + assert!(attempts <= 2 * (BOUND / CAPACITY_PAUSE_MAX_NS + 16)); + } } diff --git a/tools/asahi/g17-capacity-backoff-tests.py b/tools/asahi/g17-capacity-backoff-tests.py new file mode 100755 index 0000000000000..9510927abd674 --- /dev/null +++ b/tools/asahi/g17-capacity-backoff-tests.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production render capacity wait against a deterministic clock. + +Storm: every reclaim reports a return and notifies the render domain, and the +refusal never clears (the starvation case). Quiet: nothing is returned and +nothing notifies. Release: capacity appears after 5 ms. Each attempt costs +200 ns of fake CPU time. This tests the retry schedule, not firmware. +""" +from pathlib import Path +import os +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi/g17' + + +def item(source, signature): + start = source.index(signature) + begin = source.index('{', start) + depth, end = 1, begin + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +def const(source, name): + start = source.index('const ' + name) + return source[start:source.index(';', start) + 1] + + +def fixture(source): + parts = [const(source, 'CAPACITY_PAUSE_MIN_NS'), const(source, 'CAPACITY_PAUSE_MAX_NS'), + '#[derive(Default)]\n' + item(source, 'struct WaitClock'), item(source, 'impl WaitClock'), + 'impl Backend {\n' + item(source, 'fn wait_render_capacity(') + '\n}'] + code = '\n'.join(parts).replace('super::super::', 'rt::') + return r''' +#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +type Result = std::result::Result; +const EBUSY:i32=-16; const FALLBACK:i32=-115; +static NOW:AtomicU64=AtomicU64::new(0); +static GEN:AtomicU64=AtomicU64::new(0); +static ATTEMPTS:AtomicUsize=AtomicUsize::new(0); +static MODE:AtomicUsize=AtomicUsize::new(0); +const STORM:usize=0; const QUIET:usize=1; const RELEASE:usize=2; +const RELEASE_AT:u64=5_000_000; +fn recovery_wait_ns()->u64 {10_000_000_000} +mod rt {pub fn now_ns()->u64 {crate::NOW.load(crate::Ordering::SeqCst)} pub const WORK_RING_WAIT_NS:u64=2_000_000_000;} +struct Delta(i64); +impl Delta {fn from_nanos(n:i64)->Self {Delta(n)}} +fn fsleep(d:Delta) {assert!(d.0>0); NOW.fetch_add(d.0 as u64,Ordering::SeqCst);} +struct Renders; +impl Renders { + fn observe(&self)->u64 {GEN.load(Ordering::SeqCst)} + fn block(&self,sample:u64,ms:u32,_:bool)->Result { + assert!(ms>=1 && ms<=10); + if GEN.load(Ordering::SeqCst)!=sample {return Ok(());} + let now=NOW.load(Ordering::SeqCst); let mut until=now+u64::from(ms)*1_000_000; + if MODE.load(Ordering::SeqCst)==RELEASE && nowResult { + if MODE.load(Ordering::SeqCst)==STORM {GEN.fetch_add(1,Ordering::SeqCst); return Ok(true);} + Ok(false) + } + fn admission_open(&self)->Result {Ok(())} + fn wait_deadline(&self,clock:&mut WaitClock,bound:u64)->Result {Ok(clock.deadline(rt::now_ns(),false,bound))} +} +''' + code + r''' +fn run(mode:usize)->(Result,u64,usize) { + NOW.store(0,Ordering::SeqCst); GEN.store(0,Ordering::SeqCst); ATTEMPTS.store(0,Ordering::SeqCst); MODE.store(mode,Ordering::SeqCst); + let b=Backend{shared:Shared{renders:Renders}}; + let (mut observed,mut clock)=(None,WaitClock::default()); + let deadline=rt::WORK_RING_WAIT_NS; + let result=loop { + // One refused placement attempt of the caller's loop. + NOW.fetch_add(200,Ordering::SeqCst); + assert!(ATTEMPTS.fetch_add(1,Ordering::SeqCst)<200_000,"capacity wait spins"); + if rt::now_ns()>=deadline {break Err(EBUSY);} + if mode==RELEASE && rt::now_ns()>=RELEASE_AT {break Ok(());} + if let Err(e)=b.wait_render_capacity(&mut observed,&mut clock,true) {break Err(e);} + }; + (result,rt::now_ns(),ATTEMPTS.load(Ordering::SeqCst)) +} +fn main() { + let (r,t,n)=run(STORM); + assert_eq!(r,Err(EBUSY)); assert!(t>=2_000_000_000 && t<2_003_000_000,"storm ends at {t}"); assert!(n<=3_200,"storm attempts {n}"); + println!("storm: EBUSY at {:.3} s after {} attempts",t as f64/1e9,n); + let (r,t,n)=run(QUIET); + assert_eq!(r,Err(EBUSY)); assert!(t>=2_000_000_000 && t<2_011_000_000); assert!(n<=420,"quiet attempts {n}"); + println!("quiet: EBUSY at {:.3} s after {} attempts",t as f64/1e9,n); + let (r,t,n)=run(RELEASE); + assert_eq!(r,Ok(())); assert!(tResult {let b=Backend{shared:Shared{renders:Renders}}; b.wait_render_capacity(&mut None,&mut WaitClock::default(),false)} +''' + + +def main(): + source = (ROOT / 'runtime/render/submit.rs').read_text() + fixed = fixture(source) + cases = [('fixed', fixed), + ('retry-after-every-reclaim', fixed.replace('&& clock.fast_retry()', '')), + ('no-pause-after-wake', fixed.replace('if left != 0 {', 'if false {'))] + with tempfile.TemporaryDirectory(prefix='g17-capacity-backoff-') as tmp: + for name, code in cases: + assert name == 'fixed' or code != fixed, name + path = Path(tmp) / (name.replace('-', '_') + '.rs') + path.write_text(code) + binary = path.with_suffix('') + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', '-O', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30) + if name == 'fixed': + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) + else: + assert result.returncode != 0, name + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index c59c064f7f07e..5250d0337cf2d 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -127,6 +127,7 @@ def mutations(flag, names, base, expect=KILL): 'quarantine', 'epoch', 'cutoff', 'whole-batch', 'legacy-format', 'read-drain'), out()), 'g17-admission-deadline-tests.py': [([], PASS)], 'g17-admission-order-tests.py': [([], PASS)], + 'g17-capacity-backoff-tests.py': [([], PASS)], 'g17-capacity-fault-tests.py': [([], PASS)], 'g17-compute-constructor-tests.py': [([], PASS)], 'g17-compute-deadline-tests.py': [([], PASS)], From b84f5655f2ee380e57211e26cc09bb88ede28f2b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 20:45:00 -0500 Subject: [PATCH 366/392] soc: apple: clpc: Hold T6050 CPU load at DCS 5 and fabric 2 A busy T6050 cluster in the upper half of its range asked for the top DCS and fabric states, 6 and 3, even for a loop that touches no memory. macOS's CLPC votes one state lower on each: traced on J714sAP under macOS 26A428, its voter word reads 0x25 (DCS 5, fabric 2) for 99% of an 18-thread CPU loop, 0 for 89% of idle and mostly 0 with one busy thread. Map the top two quartiles to DCS 5 and fabric 2. GPU work keeps both top states: the GPU driver's input now takes the higher of the map's top and the SoC's gpu states, which only T6050 sets, because bandwidth-bound GPU work runs about 11% slower at DCS 5. Signed-off-by: Diljit Singh --- drivers/soc/apple/clpc.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 1c590aa538f23..7a062b7af400c 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -679,7 +679,8 @@ static void apple_clpc_gpu_floor_count(struct apple_clpc *clpc, bool released) /* * The GPU driver's input, where it drives demand instead of the GFX power - * state: every rail at its top state while the driver reports work. + * state: every rail at the top of @map, or at @gpu where that is higher, + * while the driver reports work. */ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, struct apple_clpc_gpu *gpu, @@ -692,7 +693,8 @@ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, !apple_clpc_gpu_active(gpu, &updated, &sequence)) return false; for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1]); + demand[rail] = max3(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1], + hw->gpu[rail]); return true; } @@ -1675,16 +1677,19 @@ static const struct apple_clpc_hw apple_clpc_hw_t8152 = { * The boot loader enables no agent on either rail, so without this floor * the DRAM controller and the fabric stay at state 0: CPU memcpy caps at * about 34 GB/s on the M5 Pro. A busy cluster in the upper half of its range - * asks for the top states; three busy samples in a row (12 ms) are needed, - * as on T6040. The GPU driver reports its work in flight and gets the top - * of the map while it has some. + * asks for DCS 5 and fabric 2, the floor macOS's CLPC holds through an + * 18-thread CPU loop (J714sAP, 26A428: 0x25 on its voter 99% of the time); + * three busy samples in a row (12 ms) are needed, as on T6040. The GPU driver + * reports its work in flight and gets both top states, DCS 6 and fabric 3, + * while it has some: bandwidth-bound GPU work runs about 11% slower at DCS 5. */ static const struct apple_clpc_hw apple_clpc_hw_t6050 = { .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, .map = { - [APPLE_CLPC_DCS] = { 2, 4, 6, 6 }, - [APPLE_CLPC_SOC] = { 1, 2, 3, 3 }, + [APPLE_CLPC_DCS] = { 2, 4, 5, 5 }, + [APPLE_CLPC_SOC] = { 1, 2, 2, 2 }, }, + .gpu = { [APPLE_CLPC_DCS] = 6, [APPLE_CLPC_SOC] = 3 }, .sustain = 3, .gpu_input = true, }; From b2783e0fa0c122ece1fba5045f40d5bbe3e717db Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:56:37 +1100 Subject: [PATCH 367/392] usb: typec: tipd: Wait for the power role swap to settle tps6598x_pr_set() reads STATUS as soon as the SWSk or SWSr task completes and returns -EPROTO unless the port role has already changed. The task completes when the PR_Swap has been accepted, but the port role changes only after the power supply transition that follows. On a MacBook Air (M2) with its port linked to another Mac, echo sink > /sys/class/typec/port1/power_role fails with "Protocol error", yet the swap completes: a moment later the port reads sink, the battery charges again and the link stays up. Poll STATUS until the port role matches, for at most the time the PR_Swap timers allow the two PS_RDY messages, and report -EPROTO only if it never does. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/usb/typec/tipd/core.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 3121d95142c47..fed6ea4097662 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -460,12 +461,21 @@ static int tps6598x_dr_set(struct typec_port *port, enum typec_data_role role) return ret; } +/* + * After a PR_Swap is accepted, the initial sink waits up to tPSSourceOff + * (920 ms) for the initial source's PS_RDY, and the initial source then + * waits up to tPSSourceOn (480 ms) for the new source's PS_RDY (USB Power + * Delivery, PR_Swap timers). + */ +#define TPS_PR_SWAP_TIMEOUT_US (1400 * USEC_PER_MSEC) +#define TPS_PR_SWAP_POLL_US (10 * USEC_PER_MSEC) + static int tps6598x_pr_set(struct typec_port *port, enum typec_role role) { const char *cmd = (role == TYPEC_SINK) ? "SWSk" : "SWSr"; struct tps6598x *tps = typec_get_drvdata(port); u32 status; - int ret; + int ret, err; mutex_lock(&tps->lock); @@ -473,11 +483,18 @@ static int tps6598x_pr_set(struct typec_port *port, enum typec_role role) if (ret) goto out_unlock; - ret = tps6598x_read32(tps, TPS_REG_STATUS, &status); + /* + * The task completes once the swap has been accepted, but the port + * role in STATUS changes only after the power supply transition that + * follows, so wait for it before calling the swap a failure. + */ + err = read_poll_timeout(tps6598x_read32, ret, + ret || TPS_STATUS_TO_TYPEC_PORTROLE(status) == role, + TPS_PR_SWAP_POLL_US, TPS_PR_SWAP_TIMEOUT_US, false, + tps, TPS_REG_STATUS, &status); if (ret) goto out_unlock; - - if (role != TPS_STATUS_TO_TYPEC_PORTROLE(status)) { + if (err) { ret = -EPROTO; goto out_unlock; } From a34f50bad4fea82ad5e2731ed9116db98e592a21 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 9 Nov 2026 21:56:21 -0500 Subject: [PATCH 368/392] soc: apple: clpc: Take the GPU floor with max(), not max3() The host test that builds the CLPC demand code outside the kernel has no max3() in its shim. Nest max() instead, as the line did before. Signed-off-by: Diljit Singh --- drivers/soc/apple/clpc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 7a062b7af400c..96fc0550bc027 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -693,8 +693,8 @@ static bool apple_clpc_gpu_input_demand(const struct apple_clpc_hw *hw, !apple_clpc_gpu_active(gpu, &updated, &sequence)) return false; for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) - demand[rail] = max3(demand[rail], hw->map[rail][APPLE_CLPC_LEVELS - 1], - hw->gpu[rail]); + demand[rail] = max(demand[rail], + max(hw->map[rail][APPLE_CLPC_LEVELS - 1], hw->gpu[rail])); return true; } From 7d3d4e60de0bb3ba784e36c7a9a02db789e6c08d Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:57:44 -0600 Subject: [PATCH 369/392] arm64: dts: apple: t6050-j714c: Describe the left-front USB-C port (#1403) Enable the ATC1 template of t6050-dieX.dtsi on the J714c and link it to the port's PD controller: usb-pd@a on nub-spmi-a0, an SN201202x at SPMI address 0xa. The J714cAP ADT names it hpm1, port-number 2, port-location "left-front", acio-parent acio1, with atc-phy1, usb-drd1 and dart-usb1. The board file no longer disables nub-spmi-a0, atc1_phy, atc1_phy_mbox, usb_drd1 and its two DARTs. The atcphy1 alias lets the boot loader fill the PHY tunables. The left-back port (ATC0, hpm0) is not described. The combo PHY binding and driver exclude port index 0, the debug port that carries the boot link, on purpose. The port's USB4/Thunderbolt router (acio1) and its DisplayPort routes are not described; the PHY runs in USB mode. The other ten T6050 board DTBs are byte-identical. --- arch/arm64/boot/dts/apple/t6050-j714c.dts | 93 +++++++++++++++++------ 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-j714c.dts b/arch/arm64/boot/dts/apple/t6050-j714c.dts index bd4ea3fc0d288..2afe054d92c03 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714c.dts @@ -109,35 +109,11 @@ }; }; -/* ATC1, ATC3 and the PD controller buses nub-spmi-a0 and -a2: not yet described for this board */ -&nub_spmi_a0 { - status = "disabled"; -}; - +/* ATC3 and the PD controller bus nub-spmi-a2: not on this board's ports */ &nub_spmi_a2 { status = "disabled"; }; -&atc1_phy_mbox { - status = "disabled"; -}; - -&atc1_phy { - status = "disabled"; -}; - -&usb_drd1 { - status = "disabled"; -}; - -&usb_drd1_dart_0 { - status = "disabled"; -}; - -&usb_drd1_dart_1 { - status = "disabled"; -}; - &atc3_phy_mbox { status = "disabled"; }; @@ -278,6 +254,73 @@ #include "t6050-max-atc2.dtsi" +/* + * ==== The left-front USB-C port (ATC1) ==== + * + * ADT /arm-io/nub-spmi-a0/hpm1 (SN201202x, SPMI address 0xa, port-number 2, port-location + * "left-front", acio-parent acio1), with atc-phy1, usb-drd1 and dart-usb1. The PHY, the DWC3 and + * their DARTs and mailbox are the ATC1 template of t6050-dieX.dtsi; only the links to the PD + * controller are the board's. The port's USB4/Thunderbolt router (/arm-io/acio1) and its + * DisplayPort routes are not described yet; the PHY runs in USB mode. + * + * The left-back port (ATC0, hpm0) is the boot link's debug port. The combo PHY binding and + * driver exclude it on purpose (apple,port-index 0), so it is not described either. + */ +/ { + aliases { + atcphy1 = &atc1_phy; + }; +}; + +&atc1_phy { + port { + atc1_phy_ep: endpoint { + remote-endpoint = <&hpm1_ss>; + }; + }; +}; + +&usb_drd1 { + port { + usb_drd1_hs: endpoint { + remote-endpoint = <&hpm1_hs>; + }; + }; +}; + +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C Left Front"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + hpm1_hs: endpoint { + remote-endpoint = <&usb_drd1_hs>; + }; + }; + + port@1 { + reg = <1>; + hpm1_ss: endpoint { + remote-endpoint = <&atc1_phy_ep>; + }; + }; + }; + }; + }; +}; + /* The host DROM from the J714cAP ADT (/arm-io/acio2 thunderbolt-drom); the boot loader replaces it */ &acio2_nhi { apple,thunderbolt-drom = [fd 02 00 00 00 00 00 00 00 90 19 c3 8c 00 5c 00 From 8e44895ce96f7b796f9ef06ea9a98f7b9553a5b7 Mon Sep 17 00:00:00 2001 From: Ryan <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:16:30 +1100 Subject: [PATCH 370/392] arm64: dts: apple: t8140-j700: reserve the configured AOP and always-on GPIO lines (#1407) The AOP and always-on GPIO banks are enabled with no consumer. Some of their lines are already configured by the firmware when Linux boots (non-reset mode, pulls or drive strength, one output driven high) and nothing says what they are for. Mark those lines in gpio-reserved-ranges so gpiolib never hands them out and the driver's register map leaves their pad registers alone. Lines still in the reset state stay available. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t8140-j700.dts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ba91829924008..ce94f916ff754 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -716,9 +716,12 @@ /* * The AOP's GPIO bank. The AOP keeps interrupt groups 0-3; the AP's are 4-6 - * (AIC 384-386). No consumer requests a line yet. + * (AIC 384-386). No consumer requests a line yet. The reserved lines are the + * ones the firmware has already configured away from the bank's reset state + * by the time Linux boots; their users are not known, so they stay off limits. */ &pinctrl_aop { + gpio-reserved-ranges = <18 1>, <21 1>, <27 9>, <46 2>, <50 7>; interrupts = , , ; @@ -728,10 +731,12 @@ /* * The always-on GPIO bank, as on the J616s: the AP takes none of its - * interrupts. Line 20 is the secure-repair enable and is never requested. + * interrupts. Line 20 is the secure-repair enable and is never requested; + * the other reserved lines are configured by the firmware before Linux + * boots (line 27 is driven high) and have no known user. */ &pinctrl_nub { - gpio-reserved-ranges = <20 1>; + gpio-reserved-ranges = <5 1>, <10 1>, <20 1>, <23 1>, <27 3>; status = "okay"; /delete-property/ interrupt-controller; /delete-property/ #interrupt-cells; From 50e5f040ba07a32c0df012559f054b593caa3c65 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:23:45 -0600 Subject: [PATCH 371/392] arm64: dts: apple: t6050-j714c: Describe the six speaker amplifiers (#1408) * arm64: dts: apple: t6050-j714c: Describe the six speaker amplifiers Enable i2c1 and i2c3 on the J714c and describe the six SN012776 speaker amplifiers from the J714cAP ADT: left woofer 1, left woofer 2 and left tweeter on i2c1 at 0x38, 0x39 and 0x3a, right woofer 1, right woofer 2 and right tweeter on i2c3 at 0x3b, 0x3c and 0x3d. The amplifiers share one shutdown line, AP pin 107 (function-reset of the first), described as the speaker_sdz fixed regulator with apple,sdz-group-fail-closed, and one fault interrupt, pin 106. Each amplifier's sense slots come from the ADT speaker-config, where the transmit slot N belongs to speakerN: voltage sense in slot 4N, current sense in slot 4N+2. ps_i2c3 has no described parent (SIO), as for i2c1 and i2c2. Pins 106 and 107 join pin 80 in the exposed range of pinctrl_ap. There is no sound card, no jack codec and no AOP audio change. The other ten T6050 board DTBs are byte-identical. * arm64: dts: apple: t6050-j714c: Clock i2c3 from the reference clock The i2c3 stub of t6050-peripherals.dtsi takes <&peripheral_clocks 4>. No driver in the tree provides apple,pmgr-clocks, so i2c3 and the three amplifiers on it would never probe. The ADT gives i2c1, i2c2 and i2c3 the same clock-ids (4), and i2c1 and i2c2 take <&clkref>, so the J714c does the same for i2c3. The other ten T6050 board DTBs are byte-identical. --- arch/arm64/boot/dts/apple/t6050-j714c.dts | 114 +++++++++++++++++++++- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-j714c.dts b/arch/arm64/boot/dts/apple/t6050-j714c.dts index 2afe054d92c03..89b4c00d310a2 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714c.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714c.dts @@ -134,8 +134,64 @@ status = "disabled"; }; +/* + * The six speaker amplifiers (ADT /arm-io/i2c1/audio-speaker* and /arm-io/i2c3/audio-speaker*, + * SN012776) share one shutdown line, AP pin 107 (function-reset of the first), and one fault + * interrupt, pin 106. The ADT's speaker-config gives each amp a transmit slot N with its + * voltage sense in slot 4N and current sense in 4N+2, and names the amps by speakerN: + * 0 left woofer 1, 1 right woofer 1, 2 left woofer 2, 3 right woofer 2, 4 left tweeter, + * 5 right tweeter. + * + * There is no sound card yet: the AOP audio driver is not on this branch, and the jack codec + * (i2c2) is not described. The amps stay in shutdown until a stream asks for them. + */ +/ { + speaker_sdz: regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + &i2c1 { - status = "disabled"; + status = "okay"; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <2>; + ti,vmon-slot-no = <0>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <10>; + ti,vmon-slot-no = <8>; + }; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <18>; + ti,vmon-slot-no = <16>; + }; }; &i2c2 { @@ -144,10 +200,11 @@ /* * Only the pins a node here names are exposed, as on the J716c: pin 112's register - * takes an SError when read. 80 is PCIe port 0's PERST#. + * takes an SError when read. 80 is PCIe port 0's PERST#, 106 the amps' fault interrupt and + * 107 their shutdown line. */ &pinctrl_ap { - gpio-reserved-ranges = <0 80>, <81 140>; + gpio-reserved-ranges = <0 80>, <81 25>, <108 113>; }; /* This board's SMC fan and power keys are not yet checked */ @@ -254,6 +311,57 @@ #include "t6050-max-atc2.dtsi" +/* The I2C power states have no described parent (SIO), as for i2c1 and i2c2 */ +&ps_i2c3 { + /delete-property/ power-domains; + status = "okay"; +}; + +&i2c3 { + /* + * The stub's pmgr clock controller has no driver. The ADT gives i2c1, i2c2 and i2c3 the same + * clock-ids (4), so i2c3 takes the reference clock as i2c1 and i2c2 do. + */ + clocks = <&clkref>; + status = "okay"; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <6>; + ti,vmon-slot-no = <4>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <14>; + ti,vmon-slot-no = <12>; + }; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + apple,sdz-group-fail-closed; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <22>; + ti,vmon-slot-no = <20>; + }; +}; + /* * ==== The left-front USB-C port (ATC1) ==== * From 4f5ecf698dcb9c36da8a9050e8955c119040dd35 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 10 Nov 2026 15:08:46 +1100 Subject: [PATCH 372/392] tools/aurora-sep: tests: take the refkey ACL argument in the identity fixture refkey.rs's sks_create_refkey() now takes an optional ACL (ed3d64765952, "sep-key: Touch ID keys the enclave guards"). The fixture in test_identity_preservation.py still declared the two-argument form, so rustc rejected the machine-refkey case. Declare the third argument. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_identity_preservation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/aurora-sep/tests/test_identity_preservation.py b/tools/aurora-sep/tests/test_identity_preservation.py index 3f15f10c7d191..f420ec74b167f 100644 --- a/tools/aurora-sep/tests/test_identity_preservation.py +++ b/tools/aurora-sep/tests/test_identity_preservation.py @@ -69,7 +69,7 @@ def body(source, signature): impl Cache {fn lock(&self)->Option<()>{None}} struct SepData {machine_refkey:Cache,creates:Cell} impl SepData { - fn sks_create_refkey(&self,_:u32,_:&[u8])->Option>{self.creates.set(self.creates.get()+1);None} + fn sks_create_refkey(&self,_:u32,_:&[u8],_:Option<&[u8]>)->Option>{self.creates.set(self.creates.get()+1);None} fn cache_machine_refkey(&self,_:KVec)->bool{true} METHOD } From 2fdb577f2d862f4c436aae7c0cd6ef4f78f40b53 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 10 Nov 2026 15:28:46 +1100 Subject: [PATCH 373/392] tools/asahi: mt7932: stub the AWDL event hook in the association harness mt_receive() now hands EID 0xf7 to mt_awdl_event() when CONFIG_MT7932_AWDL is enabled. The association-events harness compiles mt_receive() on its own and had neither IS_ENABLED() nor the hook, so it no longer built. Treat AWDL as disabled there and stub the hook. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/mt7932-association-events-tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index d5905d4d71e34..13d1262f601ed 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -122,6 +122,8 @@ def function(source, name): static void schedule_work(void*w){scheduled++;} static void mt_scan_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_bss_presence(struct mt7932*m,const struct mt7932_event*e){} +#define IS_ENABLED(option) 0 +static void mt_awdl_event(struct mt7932*m,const struct mt7932_event*e){} static void mt_data_complete(struct mt7932*m,const u8*p,size_t n){} static void mt_packet_receive(struct mt7932*m,const u8*p,size_t n){} static void mt_rf_fail_locked(struct mt7932*m,int e){m->rf_ready=false;m->link_failed=true;m->cal_state.error=e;m->cal_state.active=false;m->connect_error=e;complete_all(&m->assoc_done);complete_all(&m->cal_response);} From e8b0d6f93c1f764bf34a55b551a9cd23d7b686e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 10 Nov 2026 15:29:56 +1100 Subject: [PATCH 374/392] accel/ane: h16: restart a stopped startup at once On the J616s (T6041) the H16 firmware's CPU is stopped part way through its startup on about one boot in six: the ASC wrapper control word reads 0 (RUN cleared) and its status 0x30006a, 8 to 15 ms after RUN, while the firmware is still asking for its shared allocations. A stopped CPU answers nothing, so the startup only failed at the 3 s command deadline, and the verified restart that then brings the engine up cost every such boot (and every such cold resume) those 3 s. Give the shared transport an optional stopped() op, checked in each wait before it sleeps. The H16 backend reports a cleared RUN bit while the startup commands are out; the wait then ends at once with -ESHUTDOWN, which gets the same read-only evidence and the same bounded, verified restart as a timeout. The evidence now also names the wrapper control/status words and the time since RUN. J616s, 7.3.0-aurora-fleet48 with this change, one-shot boots: every stall seen read control 0 / status 0x30006a and was caught within a poll; the engine was ready 0.1 s after probe instead of 3.1 s, with MobileNet top-1 16/16 on every boot. Neither a power cycle of the sys domain before the first RUN nor busy-polling every reply during startup changed the stall rate, so the cause of the stop is not addressed here. KUnit: ane-h16-boot gains startup_stopped and startup_stopped_restart (x86_64, KASAN + UBSAN, boot and transport configs: 226/226 pass). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane_fw_transport.h | 8 +++ drivers/accel/ane/ane_h16_backend.c | 26 ++++++++-- drivers/accel/ane/ane_h16_boot.h | 41 ++++++++++++++-- drivers/accel/ane/ane_h16_boot_test.c | 71 +++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 9 deletions(-) diff --git a/drivers/accel/ane/ane_fw_transport.h b/drivers/accel/ane/ane_fw_transport.h index 527dff351aabf..ab1d7134eb3a6 100644 --- a/drivers/accel/ane/ane_fw_transport.h +++ b/drivers/accel/ane/ane_fw_transport.h @@ -102,6 +102,12 @@ struct ane_fw_transport_ops { void (*refused)(void *ctx, u64 word0, u64 word1, u64 word2, int error); /* Optional: a shared allocation was freed ("FREE") or served again ("reuse"). */ void (*shared_note)(void *ctx, const char *event, u64 dva, u64 size, u32 tag); + /* + * Optional: true once the firmware's CPU no longer runs, so no reply can + * come. A wait then ends at once with -ESHUTDOWN instead of at its + * deadline. + */ + bool (*stopped)(void *ctx); }; struct ane_fw_transaction { @@ -914,6 +920,8 @@ static inline int ane_fw_transport_wait(struct ane_fw_transport *t, unsigned int now = t->ops->now_ns(t->ctx); if (now >= t->pending[ch].deadline_ns) return ane_fw_transport_fail(t, -ETIMEDOUT); + if (t->ops->stopped && t->ops->stopped(t->ctx)) + return ane_fw_transport_fail(t, -ESHUTDOWN); t->polls++; t->ops->relax(t->ctx, now - start); } diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 9e0b061204a2b..4b52a4a5fe3f0 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -182,7 +182,7 @@ static void ane_h16_quarantine(void *ctx, int error) { struct ane_h16_device *h = ctx; - if (error == -ETIMEDOUT && h->boot.stage == ANE_H16_BOOT) + if ((error == -ETIMEDOUT || error == -ESHUTDOWN) && h->boot.stage == ANE_H16_BOOT) ane_h16_startup_fwlog(h); dev_err(h->dev, "H16 quarantine error %d; mappings, power and module held until reboot\n", @@ -1145,9 +1145,10 @@ static void ane_h16_startup_fwlog(struct ane_h16_device *h) } /* - * Power cycles allowed after a startup timeout at bring-up. On the J616s the - * firmware stops part way through startup on about one boot in ten: it posts - * no further request and never answers the pending command. + * Power cycles allowed after a startup timeout or stop at bring-up. On the + * J616s the firmware's CPU is stopped part way through startup on about one + * boot in six: it posts no further request and never answers the pending + * command. */ #define ANE_H16_STARTUP_RESTARTS 2 @@ -1192,6 +1193,22 @@ static const struct ane_h16_boot_ops ane_h16_ops = { static void ane_h16_busy(void *ctx, bool busy); +/* + * On the J616s the firmware's CPU is sometimes stopped part way through its + * startup, 8 to 12 ms after RUN: its wrapper control word reads 0 (RUN + * cleared) and the status 0x30006a. It then answers nothing until it is + * power cycled. Seen during startup only, so only watched there. + */ +static bool ane_h16_cpu_stopped(void *ctx) +{ + struct ane_h16_device *h = ctx; + const struct ane_h16_build *f = h->boot.build; + + if (!h->boot.starting || !h->engine_on) + return false; + return !(ane_h16_mmio_read32(h, ANE_H16_ENGINE, f->cpu_control) & f->run); +} + static const struct ane_h16_session_ops ane_h16_session_ops = { .map = ane_h16_program_map, .log = ane_h16_log, @@ -1212,6 +1229,7 @@ static const struct ane_fw_transport_ops ane_h16_transport_ops = { .alloc = ane_h16_alloc_shared, .refused = ane_h16_refused, .shared_note = ane_h16_shared_note, + .stopped = ane_h16_cpu_stopped, }; /* Pre-RUN rollback of everything a stage acquired. h itself stays. */ diff --git a/drivers/accel/ane/ane_h16_boot.h b/drivers/accel/ane/ane_h16_boot.h index 2ebe3caa21ca2..f12a8fb3d8e10 100644 --- a/drivers/accel/ane/ane_h16_boot.h +++ b/drivers/accel/ane/ane_h16_boot.h @@ -169,6 +169,10 @@ struct ane_h16_boot { bool failed; bool ran; bool quarantined; + /* Inside ane_h16_startup(): the CPU must keep running until it answers. */ + bool starting; + /* When the CPU was last released, for the startup evidence. */ + u64 run_ns; /* Startup timeouts answered with a verified restart, and the budget. */ unsigned int restarts; unsigned int max_restarts; @@ -524,6 +528,7 @@ static inline int ane_h16_scratch_boot(struct ane_h16_boot *b) /* Latch before the first RUN store, including a failure at that store. */ b->ran = true; b->ops->write32(b->ctx, ANE_H16_ENGINE, f->cpu_control, f->run); + b->run_ns = b->transport_ops->now_ns(b->ctx); ret = ane_h16_wait(b, 7, f->ready, false); if (ret) return ret; @@ -585,14 +590,23 @@ static inline int ane_h16_scratch_boot(struct ane_h16_boot *b) return ret; } -/* Read-only evidence: never acknowledge an IRQ or change a cursor on timeout. */ +/* + * Read-only evidence: never acknowledge an IRQ or change a cursor on timeout + * or when the CPU stopped. + */ static inline void ane_h16_startup_timeout(struct ane_h16_boot *b) { struct ane_fw_transport *t = &b->transport; const struct ane_fw_channel *c = &t->channels[ANE_FW_SHAREDMALLOC]; + const struct ane_h16_build *f = b->build; unsigned int i; u8 *p; + b->ops->log(b->ctx, "startup CPU control/status", + b->ops->read32(b->ctx, ANE_H16_ENGINE, f->cpu_control), + b->ops->read32(b->ctx, ANE_H16_ENGINE, f->cpu_control + 4)); + b->ops->log(b->ctx, "startup us since RUN/error", + div_u64(t->ops->now_ns(t->ctx) - b->run_ns, NSEC_PER_USEC), -t->error); b->ops->log(b->ctx, "startup timeout IRQ/polls", t->ops->read32(t->ctx, t->soc->irq_status), t->polls); b->ops->log(b->ctx, "startup SHAREDMALLOC head/slots", @@ -619,7 +633,7 @@ static inline void ane_h16_startup_timeout(struct ane_h16_boot *b) b->ops->log(b->ctx, "startup timeout scratch", i, ane_h16_read(b, i)); } -static inline int ane_h16_startup(struct ane_h16_boot *b) +static inline int ane_h16_startup_commands(struct ane_h16_boot *b) { const struct ane_h16_build *f = b->build; unsigned int i; @@ -639,7 +653,7 @@ static inline int ane_h16_startup(struct ane_h16_boot *b) reply = (u8 *)b->pool.cpu + (b->transport.pending[ANE_FW_IO].dva - b->pool.dva); if (ret) { - if (ret == -ETIMEDOUT) + if (ret == -ETIMEDOUT || ret == -ESHUTDOWN) ane_h16_startup_timeout(b); if (ret == -EREMOTEIO) b->ops->log(b->ctx, "startup refusal ID/ack", @@ -677,6 +691,22 @@ static inline int ane_h16_startup(struct ane_h16_boot *b) return 0; } +/* + * While the startup commands are out the CPU is watched for a stop (its RUN + * bit cleared, see ane_fw_transport_ops.stopped): a stopped CPU answers + * nothing, so the startup ends at once with -ESHUTDOWN and gets the same + * evidence and verified restart as a timeout. + */ +static inline int ane_h16_startup(struct ane_h16_boot *b) +{ + int ret; + + b->starting = true; + ret = ane_h16_startup_commands(b); + b->starting = false; + return ret; +} + /* * A startup that timed out left the firmware stopped part way, with requests * of its own outstanding. Its rings cannot be resumed, so the engine is @@ -703,7 +733,7 @@ static inline int ane_h16_restart(struct ane_h16_boot *b) } /* - * Run startup; after a timeout, restart the engine and run it again, at most + * Run startup; after a timeout or a stop, restart the engine and run it again, at most * b->max_restarts times in all against @used. The caller owns @used: one * budget for bring-up, a fresh one for each boot after system sleep. Logs * @what with the restart count and the budget. @@ -714,7 +744,8 @@ static inline int ane_h16_startup_restarting(struct ane_h16_boot *b, { int ret = ane_h16_startup(b); - while (ret == -ETIMEDOUT && b->ops->restart && *used < b->max_restarts) { + while ((ret == -ETIMEDOUT || ret == -ESHUTDOWN) && b->ops->restart && + *used < b->max_restarts) { (*used)++; b->ops->log(b->ctx, what, *used, b->max_restarts); ret = ane_h16_restart(b); diff --git a/drivers/accel/ane/ane_h16_boot_test.c b/drivers/accel/ane/ane_h16_boot_test.c index fa81df8f73103..58bf09123d3ce 100644 --- a/drivers/accel/ane/ane_h16_boot_test.c +++ b/drivers/accel/ane/ane_h16_boot_test.c @@ -66,6 +66,10 @@ struct ane_h16_sim { bool reset; bool unpowered; u32 quarantine_reads; + /* The stopped op, and the CPU control word the stop evidence logged. */ + struct ane_fw_transport_ops transport; + u64 stop_control; + u32 stopped_checks; }; static const u32 sim_power_words[] = { 0x3c0, 0xc008, 0xc000, 0xc010, 0xc018 }; @@ -410,6 +414,8 @@ static void sim_log(void *ctx, const char *item, u64 a, u64 b) s->timeout_scratch++; if (!strcmp(item, "quarantine scratch")) s->quarantine_reads++; + if (!strcmp(item, "startup CPU control/status")) + s->stop_control = a; } static bool sim_powered(void *ctx) @@ -514,6 +520,22 @@ static const struct ane_fw_transport_ops sim_transport_ops = { .refused = sim_refused, }; +/* As the backend: only while the startup commands are out. */ +static bool sim_stopped(void *ctx) +{ + struct ane_h16_sim *s = ctx; + + s->stopped_checks++; + return s->boot.starting && !(s->cpu & s->build.run); +} + +static void sim_use_stopped(struct ane_h16_sim *s) +{ + s->transport = sim_transport_ops; + s->transport.stopped = sim_stopped; + s->boot.transport_ops = &s->transport; +} + /* A programmed bank, ends without their low two bits. */ static void sim_program_dapf(struct ane_h16_sim *s) { @@ -854,6 +876,53 @@ static void ane_h16_startup_restart_budget_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EBUSY); } +/* + * A CPU that loses RUN during startup can answer nothing: the wait ends at + * once with -ESHUTDOWN and the evidence names the control word, instead of + * waiting out the command deadline. + */ +static void ane_h16_startup_stopped_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + u64 start; + + sim_use_stopped(s); + s->build.command_timeout_ns = 3ULL * NSEC_PER_SEC; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), 0); + /* Running and answering: never reported stopped. */ + KUNIT_EXPECT_FALSE(test, sim_stopped(s)); + s->no_startup_reply = true; + s->cpu = 0; + start = s->now; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -ESHUTDOWN); + KUNIT_EXPECT_LT(test, s->now - start, 2ULL * NSEC_PER_MSEC); + KUNIT_EXPECT_EQ(test, s->stop_control, 0ULL); + KUNIT_EXPECT_EQ(test, s->timeout_scratch, 8U); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_FALSE(test, s->boot.starting); +} + +/* A stopped CPU gets the same verified restart as a timed-out one. */ +static void ane_h16_startup_stopped_restart_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + u64 start; + + sim_use_restart(s, 1, 2); + sim_use_stopped(s); + s->build.command_timeout_ns = 3ULL * NSEC_PER_SEC; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), 0); + s->cpu = 0; + start = s->now; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), 0); + KUNIT_EXPECT_LT(test, s->now - start, 100ULL * NSEC_PER_MSEC); + KUNIT_EXPECT_EQ(test, s->boot.restarts, 1U); + KUNIT_EXPECT_EQ(test, s->power_cycles, 1U); + KUNIT_EXPECT_FALSE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->boot.stage, ANE_H16_STARTUP); + KUNIT_EXPECT_FALSE(test, s->boot.starting); +} + /* An unverified power cycle is not followed by another release of the CPU. */ static void ane_h16_startup_restart_refused_test(struct kunit *test) { @@ -1191,6 +1260,8 @@ static struct kunit_case ane_h16_boot_cases[] = { KUNIT_CASE(ane_h16_startup_timeout_test), KUNIT_CASE(ane_h16_startup_restart_test), KUNIT_CASE(ane_h16_startup_restart_budget_test), + KUNIT_CASE(ane_h16_startup_stopped_test), + KUNIT_CASE(ane_h16_startup_stopped_restart_test), KUNIT_CASE(ane_h16_startup_restart_refused_test), KUNIT_CASE(ane_h16_startup_restart_unpowered_test), KUNIT_CASE(ane_h16_startup_no_budget_test), From c4f57fc5fa8d374926e146fb7151fbefdeb0586f Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 9 Nov 2026 21:37:09 -0500 Subject: [PATCH 375/392] drm/apple: record the started swap on the firmware-27 dual-pipe path The flip-lost handling unarms the framebuffers of dcp->started_swap_id, which the swap-start callbacks set. The firmware-27 dual-pipe start records the swap id without it, so a lost flip on a display driven by two pipes unarmed the wrong swap's framebuffers. Set it there too. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc index c2a681328c102..a62d630d18a96 100644 --- a/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc +++ b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc @@ -120,6 +120,7 @@ static bool dcp_peer_start_second(struct apple_dcp *owner, return true; } owner->swap.h17p.swap.swap_id = resp->swap_id; + WRITE_ONCE(owner->started_swap_id, resp->swap_id); dcp_swap_start(peer, false, &req, dcp_peer_started, owner); return true; } From d2566e93444f7eb4a453a7c1b3f9f55444bdf153 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 9 Nov 2026 22:25:37 -0500 Subject: [PATCH 376/392] drm/asahi: g17: build the M5 thermal governor without cpufreq The governor's CPU ceiling calls cpufreq_cpu_get() and cpufreq_cpu_put() directly. Without CONFIG_CPU_FREQ those are inline stubs in the C header, which the Rust bindings do not carry, so the driver does not compile: drivers/soc/apple/sep.kunitconfig builds DRM_ASAHI that way and the KUnit run stops there. Take and drop the policy reference through two small wrappers that do what the C stubs do when cpufreq is off: no CPU has a policy, so the ceiling never takes a request. Nothing changes with cpufreq on. Tested: the Apple SEP KUnit suites build and pass on arm64 QEMU with sep.kunitconfig (193 tests), and the full J716c configuration with CONFIG_CPU_FREQ=y builds. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/gpu/drm/asahi/g17/thermal.rs | 42 ++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs index 0f5c21a77488a..6dd9bc7c2b15a 100644 --- a/drivers/gpu/drm/asahi/g17/thermal.rs +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -394,6 +394,38 @@ struct CpuRequest { // ceiling's lock. unsafe impl Send for CpuRequest {} +/// Takes a counted reference on `cpu`'s cpufreq policy, or returns null where it has none. +#[cfg(CONFIG_CPU_FREQ)] +fn policy_get(cpu: u32) -> *mut bindings::cpufreq_policy { + // SAFETY: cpufreq_cpu_get accepts any CPU number and returns a counted reference or null. + unsafe { bindings::cpufreq_cpu_get(cpu) } +} + +/// Without cpufreq no CPU has a policy, so the ceiling never takes a request. +#[cfg(not(CONFIG_CPU_FREQ))] +fn policy_get(_cpu: u32) -> *mut bindings::cpufreq_policy { + core::ptr::null_mut() +} + +/// Drops a reference taken by [`policy_get`]. +/// +/// # Safety +/// +/// `policy` must hold a reference from [`policy_get`] that has not been dropped yet. +#[cfg(CONFIG_CPU_FREQ)] +unsafe fn policy_put(policy: *mut bindings::cpufreq_policy) { + // SAFETY: the caller holds the reference. + unsafe { bindings::cpufreq_cpu_put(policy) } +} + +/// Without cpufreq there is no reference to drop. +/// +/// # Safety +/// +/// None; unsafe only to match the cpufreq variant. +#[cfg(not(CONFIG_CPU_FREQ))] +unsafe fn policy_put(_policy: *mut bindings::cpufreq_policy) {} + /// Shares the SoC's heat between the CPU and the GPU while the governor limits the GPU: when the /// GPU is budget-bound it lowers the maximum /// frequency of every CPU cluster one step at a time, and gives the steps back once the GPU no @@ -427,9 +459,7 @@ impl CpuCeiling { // SAFETY: nr_cpu_ids is set once during early boot. let cpus = unsafe { bindings::nr_cpu_ids }; for cpu in 0..cpus { - // SAFETY: cpufreq_cpu_get accepts any CPU number and returns a counted reference or - // null. - let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + let policy = policy_get(cpu); if policy.is_null() { continue; } @@ -444,7 +474,7 @@ impl CpuCeiling { let Some(mut request) = request.filter(|_| self.requests.reserve(1, GFP_KERNEL).is_ok()) else { // SAFETY: drops the reference taken above. - unsafe { bindings::cpufreq_cpu_put(policy) }; + unsafe { policy_put(policy) }; continue; }; // SAFETY: the policy outlives the request (the reference is held until removal) and @@ -459,7 +489,7 @@ impl CpuCeiling { }; if added < 0 { // SAFETY: drops the reference taken above. - unsafe { bindings::cpufreq_cpu_put(policy) }; + unsafe { policy_put(policy) }; continue; } // Capacity was reserved above, so this cannot fail. @@ -474,7 +504,7 @@ impl CpuCeiling { // SAFETY: the request was added in acquire and the policy reference is still held. unsafe { bindings::freq_qos_remove_request(&mut *entry.request); - bindings::cpufreq_cpu_put(entry.policy); + policy_put(entry.policy); } } self.level = 0; From b277bcffafa7ccbae0eae7e7175f9705cae2fe8f Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:21:50 +1100 Subject: [PATCH 377/392] wifi: mt7932: keep the station connection ids in one index space Since the AWDL interface split the peer allocator into station and WTBL bitmaps, mt_peer_reserve() hands the infrastructure connection WTBL indices 1-4, which overlap its station indices 1 and 2. On the MacBook Neo every connect then fails with CONNECT_FAILED -EINVAL and the access point is never joined. Draw the connection station and WTBL indices from the union of both allocators again, so a fresh connection gets stations 1-2 and WTBL entries 3-6 as before, while each index is still recorded in its own allocator for the AWDL per-table release. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/net/wireless/mediatek/mt7932/link.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 063288b38c768..4cf6062f3bdd8 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -121,23 +121,34 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) } } -/* One owned station and firmware handshake; no host key fabrication. */ +/* + * One owned station and firmware handshake; no host key fabrication. + * + * The infrastructure connection takes its station and WTBL indices from one + * index space, as it did before AWDL, so none of its WTBL entries reuses one + * of its station indices; the firmware refuses such a connect with -EINVAL. + * Each index is still recorded in its own allocator, so AWDL peers keep + * their per-table release. + */ static int mt_peer_reserve(struct mt7932 *m) { unsigned long station = m->station_alloc, wtbl = m->wtbl_alloc; + unsigned long used = station | wtbl; unsigned int i, id; for (i = 0; i < ARRAY_SIZE(m->peer_ids.station); i++) { - id = find_first_zero_bit(&station, 15); + id = find_first_zero_bit(&used, 15); if (id >= 15) return -ENOSPC; + __set_bit(id, &used); __set_bit(id, &station); m->peer_ids.station[i] = id; } for (i = 0; i < ARRAY_SIZE(m->peer_ids.wtbl); i++) { - id = find_first_zero_bit(&wtbl, 14); + id = find_first_zero_bit(&used, 14); if (id >= 14) return -ENOSPC; + __set_bit(id, &used); __set_bit(id, &wtbl); m->peer_ids.wtbl[i] = id; } From f2802e92e4dd6c4f920a69235fc7fed8c3530417 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:47:13 +1100 Subject: [PATCH 378/392] drm/apple: switch EDR on a panel without a mode set A desktop 26.x panel keeps its SDR link and shows PQ content as EDR: PQ only changes how the swaps tag their surfaces and the EDR values they carry. A change of Colorspace or of the HDR EOTF was still a mode set, so a compositor that turns HDR on at startup, after the boot splash drew in SDR, made the panel go black for a few frames. On such a panel let SDR and PQ through without a mode set and follow the connector's colour state at the next flush. Other EOTFs still go through the mode set and its check. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> --- drivers/gpu/drm/apple/apple_drv.c | 10 ++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 31 ++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b35ed086b4710..02ef41d19947a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -918,6 +918,16 @@ static int apple_connector_atomic_check(struct drm_connector *conn, to_apple_connector(conn)->dcp && dcp_hdr_metadata_live(to_apple_connector(conn)->dcp)) return 0; + /* + * A panel that shows PQ as EDR keeps its SDR link: SDR and PQ switch + * with the next swap, so a compositor turning HDR on at startup does + * not blank it. Other EOTFs still go through the mode set's check. + */ + if (old->crtc == new->crtc && to_apple_connector(conn)->dcp && + dcp_edr_live(to_apple_connector(conn)->dcp) && + (apple_connector_eotf(new) == HDMI_EOTF_TRADITIONAL_GAMMA_SDR || + apple_connector_eotf(new) == HDMI_EOTF_SMPTE_ST2084)) + return 0; crtc_state->mode_changed = true; return 0; } diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index aaaf2918c4b75..14563a26602ce 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -77,6 +77,7 @@ int dcp_activate(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); bool dcp_hdr_metadata_live(struct platform_device *pdev); +bool dcp_edr_live(struct platform_device *pdev); void dcp_retain_framebuffer(struct platform_device *pdev, struct dcp_fb_reference *entry); void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8bca8d2cebeca..2a7735a2c7654 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2570,6 +2570,33 @@ bool dcp_hdr_metadata_live(struct platform_device *pdev) !dcp_has_panel(dcp); } +/* + * Whether @pdev switches between SDR and PQ shown as EDR without a mode set: + * a desktop 26.x panel keeps its SDR link either way, and PQ only changes how + * the swaps tag their surfaces and the EDR values they carry + * (dcp_edr_prepare()). macOS turns EDR on and off on a live panel too. + */ +bool dcp_edr_live(struct platform_device *pdev) +{ + return dcp_panel_edr_supported(platform_get_drvdata(pdev)); +} + +/* Follow a colour state change that dcp_edr_live() let through without a mode set. */ +static void dcp_edr_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) +{ + bool edr; + + if (!dcp_panel_edr_supported(dcp)) + return; + edr = dcp_crtc_eotf(dcp, crtc, commit) == HDMI_EOTF_SMPTE_ST2084 && + dcp_crtc_bt2020(dcp, crtc, commit); + if (edr == dcp->output_edr) + return; + dev_dbg(dcp->dev, "EDR %s without a mode set\n", edr ? "on" : "off"); + dcp->output_edr = edr; +} + /* * Send the connector's HDR10 static metadata to the DCPAV service once the * link carries PQ: after the mode set that selected a PQ colour mode, and @@ -3361,8 +3388,10 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && READ_ONCE(dcp->mode_state.valid) && dcp->connector && - READ_ONCE(dcp->connector->connected)) + READ_ONCE(dcp->connector->connected)) { + dcp_edr_sync(dcp, crtc, commit); dcp_hdr_metadata_sync(dcp, crtc, commit); + } if (iomfb_uses_queue(dcp)) { iomfb_queue_atomic(dcp, crtc, commit); From 32246be58de9f3e10da0d71ce216d715c6ca6f6c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:18:37 -0600 Subject: [PATCH 379/392] test: register perf mmap accounting host check Register the alias lifecycle harness so the host-test runner executes it instead of rejecting the unregistered file. Verified with the runner before and after registration. Assisted-by: LLM Signed-off-by: Joshua Warren --- tools/asahi/run-host-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 6900b7fe82f5f..4b31cfd7761fa 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -165,7 +165,7 @@ def mutations(flag, names, base, expect=KILL): 'pmp-j613-hwmon-tests.py': [([], PASS)], 'pmu-rtc-tests.py': [(out(), PASS)] + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', - 'offset-sign', 'event-bit', 'wrong-control', + 'offset-sign', 'event-bit', 'wrong-base', 'wrong-control', 'maverick-event', 'stowe-offset'), out()), 'perf-mmap-alias-lifecycle-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], From cf0c7ee8e319ccc832409a5375e4da7a8286026b Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:33:31 -0600 Subject: [PATCH 380/392] perf: handle first AUX mmap on redirected events Initialize the first redirected AUX mmap count without incrementing a zero refcount, and use full close when that AUX map fails. Extend the source-level host harness with redirected AUX lifecycles and realistic saturated-refcount behavior. The unmodified stacked base fails nine redirect checks; the fixed source passes the registered host-runner selection. A focused arm64 core.o compile passed. These are single-threaded host-model tests, not booted-kernel or hardware validation. Assisted-by: LLM Signed-off-by: Joshua Warren --- kernel/events/core.c | 18 +- .../asahi/perf-mmap-alias-lifecycle-tests.py | 1366 +++++++++++++++-- 2 files changed, 1277 insertions(+), 107 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 93574bee7b4b2..91f680f173589 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7475,7 +7475,15 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) perf_mmap_account(vma, user_extra, extra); } - refcount_inc(&event->mmap_count); + /* + * A redirected event has mmap_count == 0 here; refcount_inc() + * from zero would WARN and saturate it. As in perf_mmap_rb(), + * the first mmap() sets the count, aliases increment it. + */ + if (!refcount_read(&event->mmap_count)) + refcount_set(&event->mmap_count, 1); + else + refcount_inc(&event->mmap_count); return 0; } @@ -7569,8 +7577,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) * Make sure to remove event->rb before releasing * event->mmap_mutex, such that any concurrent mmap() will not * attempt use this failed buffer. + * + * The minimal close only fits the base ring mmap() that + * created the rb: a redirected event's first AUX mmap() + * also sees mmap_count == 1, but shares another event's rb + * and needs the full perf_mmap_close(). */ - if (refcount_read(&event->mmap_count) == 1) { + if (!perf_mmap_offset(vma) && + refcount_read(&event->mmap_count) == 1) { /* * Minimal perf_mmap_close(); there can't be AUX or * other events on account of this being the first. diff --git a/tools/asahi/perf-mmap-alias-lifecycle-tests.py b/tools/asahi/perf-mmap-alias-lifecycle-tests.py index 55f385052ea29..4477a8cbba858 100755 --- a/tools/asahi/perf-mmap-alias-lifecycle-tests.py +++ b/tools/asahi/perf-mmap-alias-lifecycle-tests.py @@ -19,6 +19,43 @@ failure, early -EINVAL), and map_range() failures through the real perf_mmap() dispatcher error path, both for a first mapping (minimal perf_mmap_close()) and for an alias (full perf_mmap_close()), + - redirected events (perf_event_set_output): a second event with + mmap_count == 0 attached to another event's rb. Its first mmap can + only be the AUX mmap (a first ring mmap of a redirected event is + rejected -EBUSY in perf_mmap_rb). On the unmodified tree, + perf_mmap_aux() increments the zero count, which WARNs and + SATURATES (REFCOUNT_ADD_UAF); it does not produce 1. The scenarios + assert the desired behaviour (no saturation, R->mmap_count == 1, + full unwind on map_range failure: rb->mmap_count back to the + owner's count, the redirect detached, the AUX allocation freed, and + the shared user counter and both mms individually restored). + Note the dispatcher's "first mmap" minimal perf_mmap_close() branch + (mmap_count == 1) is therefore NOT reachable for a redirected + event's first AUX mmap on unmodified code -- the count reads + REFCOUNT_SATURATED, not 1 -- and the saturation itself is the + observed defect: later decrements cannot release that count, and + the failed mmap leaves R attached to the rb. Setting the count alone + would newly expose the minimal-close branch to redirected AUX mmaps, + whose unwinding does not fit it. + - focused redirected-AUX cases, each stating the exact expected + redirect mmap_count, rb counters, event list, AUX allocation + lifetime and charge at every step: + * the owner's AUX area is ALREADY mapped and the redirected event + maps it too (perf_mmap_aux()'s rb_has_aux() alias branch), + succeeding and failing at map_range(), + * the owner's ring VMA closes FIRST while the redirected AUX + mapping is still live, + * the redirected event maps the AUX area a SECOND time + (mmap_count 1 -> 2), + * the redirected event maps the NORMAL ring after its first AUX + map (perf_mmap_rb()'s alias branch). The current tree admits + this, and both events then share ONE user_page. The source + permits updates from either event to overwrite the other's + metadata; the harness does not test the contents of that page. + This change does not alter that pre-existing behavior. The ring + alias's charge follows the + tree under test (measured like the AUX alias charge) and is + recorded as known_ring_alias_charge. - real limit splitting through the tree's own perf_mmap_calc_limits() (partial spill and full spill into pinned_vm), - ring geometries: full data ring, metadata-only ring, and (user4k trees) @@ -38,8 +75,26 @@ a nonzero counter therefore FAILS and prints the exact leaked page count -- leaks are reported, not hidden. The tool exits nonzero if any invariant broke: a tree that charges every alias fails the duplicate-alias scenarios; -a tree whose aliases share the first map's single charge balances every -lifecycle and passes. +a tree whose aliases share the first map's single charge passes the tested +single-user, single-threaded lifecycles, subject to the cases below. + +On an older tree that charges per alias, a scenario that maps the same area +twice also picks up the known per-alias charge (#947/#948). Such a scenario +records that residual in known_alias_charge / known_ring_alias_charge, and +invariants_broken() prints it as KNOWN instead of failing on it, so the known +defect stays visible without masking the redirect-specific assertions. +Every other number in those scenarios is still checked exactly, and the +duplicate-alias scenarios themselves still FAIL on the known defect. + +RESULT lines distinguish the two clean outcomes: status=PASS means every +invariant held with no residual at all; status=PASS-KNOWN-RESIDUAL means +the scenario's own assertions held but it ended on the measured per-alias +residual. A FAIL is always a broken invariant. + +Each tree under test is reported with its exact provenance before the +run: worktree path, git HEAD commit and the dirty state of kernel/events +and tools/asahi, so a result line is never attributed to an unidentified +source. """ from pathlib import Path @@ -157,17 +212,63 @@ def rb_init_lines(ring): #define RLIMIT_MEMLOCK 0 #define PERF_EVENT_STATE_REVOKED (-1) -static int warn_count, warn_printed; +/* Warning model. warn_count counts every WARN event; warn_lines counts + * the lines the KERNEL would print. WARN_ON() prints every time, while + * WARN_ONCE() prints only the first time per call site -- and every + * refcount saturation type is its own WARN_ONCE site in lib/refcount.c. + * Each scenario models a fresh boot, so the "already printed" state is + * reset with the rest of the harness state. */ +#define WARN_SITES 64 +static int warn_count; /* every WARN event */ +static int warn_lines; /* lines the kernel would print */ +static int warn_printed; +static int warn_site_line[WARN_SITES]; +static int warn_site_n; + +static bool warn_site_seen(int line) +{ + int i; + + for (i = 0; i < warn_site_n; i++) + if (warn_site_line[i] == line) + return true; + return false; +} + +static void warn_line(const char *msg) +{ + warn_lines++; + if (warn_printed++ < 40) + printf(" WARN %s\n", msg); +} + static void warn_hit(int condition, const char *tag, int line) { + char msg[64]; + + if (!condition) + return; + warn_count++; + snprintf(msg, sizeof(msg), "%s line %d", tag, line); + warn_line(msg); +} + +static void warn_hit_once(int condition, const char *tag, int line) +{ + char msg[64]; + if (!condition) return; warn_count++; - if (warn_printed++ < 20) - printf(" WARN %s line %d fired\n", tag, line); + if (warn_site_seen(line)) + return; + if (warn_site_n < WARN_SITES) + warn_site_line[warn_site_n++] = line; + snprintf(msg, sizeof(msg), "%s line %d", tag, line); + warn_line(msg); } #define WARN_ON(c) warn_hit(!!(c), "WARN_ON", __LINE__) -#define WARN_ON_ONCE(c) warn_hit(!!(c), "WARN_ON_ONCE", __LINE__) +#define WARN_ON_ONCE(c) warn_hit_once(!!(c), "WARN_ON_ONCE", __LINE__) /* --- environment knobs the real code reads ---------------------------- */ static long sysctl_perf_event_mlock = 256UL << 20; /* KiB, kernel units */ @@ -249,22 +350,115 @@ def rb_init_lines(ring): static void rcu_read_lock(void) { } static void rcu_read_unlock(void) { } +/* refcount_t with the kernel's real saturating semantics + * (include/linux/refcount.h, lib/refcount.c): the counter clamps at + * REFCOUNT_SATURATED and never moves once there. + * + * Saturation warns through WARN_ONCE() in lib/refcount.c, so each + * saturation TYPE prints one line per boot however often it fires: + * warn_count counts the events, warn_lines the lines a real kernel + * would print. */ +#define REFCOUNT_SATURATED (INT_MIN / 2) + +enum refcount_saturation_type { + REFCOUNT_ADD_NOT_ZERO_OVF, /* "saturated; leaking memory" */ + REFCOUNT_ADD_OVF, /* "saturated; leaking memory" */ + REFCOUNT_ADD_UAF, /* "addition on 0; use-after-free" */ + REFCOUNT_SUB_UAF, /* "underflow; use-after-free" */ + REFCOUNT_DEC_LEAK, /* "decrement hit 0; leaking memory" */ +}; + +static const char *const refcount_sat_text[] = { + "saturated; leaking memory", + "saturated; leaking memory", + "addition on 0; use-after-free", + "underflow; use-after-free", + "decrement hit 0; leaking memory", +}; + typedef struct { int refs; } refcount_t; static void refcount_set(refcount_t *r, int v) { r->refs = v; } static int refcount_read(refcount_t *r) { return r->refs; } + +/* lib/refcount.c: refcount_warn_saturate() -- clamp and WARN_ONCE. */ +static bool refcount_sat_warned[5]; + +static void refcount_warn_saturate(refcount_t *r, int type) +{ + r->refs = REFCOUNT_SATURATED; + warn_count++; + if (!refcount_sat_warned[type]) { + refcount_sat_warned[type] = true; + warn_line(refcount_sat_text[type]); + } +} + +/* __refcount_add_not_zero(): no-op at 0, saturates on negative/overflow */ static bool refcount_inc_not_zero(refcount_t *r) { - if (r->refs <= 0) + int old = r->refs; + + if (!old) return false; + if (old < 0 || old + 1 < 0) { + refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF); + return true; /* the kernel still returns old (nonzero) */ + } r->refs++; return true; } -static void refcount_inc(refcount_t *r) { assert(r->refs > 0); r->refs++; } -static void refcount_dec(refcount_t *r) { assert(r->refs > 0); r->refs--; } +/* __refcount_add(): refcount_inc() from 0 warns (REFCOUNT_ADD_UAF) and + * saturates -- it does NOT yield 1 */ +static void refcount_inc(refcount_t *r) +{ + int old = r->refs++; + + if (!old) + refcount_warn_saturate(r, REFCOUNT_ADD_UAF); + else if (old < 0 || old + 1 < 0) + refcount_warn_saturate(r, REFCOUNT_ADD_OVF); +} +/* __refcount_sub_and_test(): success only on reaching exactly 0; warns + * (REFCOUNT_SUB_UAF) and fails to move when saturated or on underflow */ static bool refcount_dec_and_test(refcount_t *r) { - assert(r->refs > 0); - return --r->refs == 0; + int old = r->refs--; + + if (old - 1 == 0) + return true; + if (old <= 0 || old - 1 < 0) + refcount_warn_saturate(r, REFCOUNT_SUB_UAF); + return false; +} +/* __refcount_dec(): warns (REFCOUNT_DEC_LEAK) at old <= 1 and saturates */ +static void refcount_dec(refcount_t *r) +{ + int old = r->refs--; + + if (old <= 1) + refcount_warn_saturate(r, REFCOUNT_DEC_LEAK); +} + +/* lib/refcount.c: refcount_dec_not_one() -- decrement unless the count + * is 1, in which case it returns false and leaves the count alone. At + * REFCOUNT_SATURATED it returns true WITHOUT warning and WITHOUT moving + * the counter: a saturated count is not decremented and not reported. */ +static bool refcount_dec_not_one(refcount_t *r) +{ + unsigned int val = (unsigned int)r->refs; + + if (val == (unsigned int)REFCOUNT_SATURATED) + return true; /* saturated: silent, no decrement */ + if (val == 1) + return false; /* the caller takes it to zero */ + if (val == 0) { + /* underflow: WARN_ONCE, count left alone */ + warn_hit_once(1, "refcount_t: underflow; use-after-free", + __LINE__); + return true; + } + r->refs = (int)(val - 1); + return true; } struct mutex { int locked; }; @@ -276,12 +470,21 @@ def rb_init_lines(ring): if (*m) mutex_unlock(*m); /* scoped_guard step already nulled it */ } +/* lib/refcount.c: refcount_dec_and_mutex_lock() -- the dec_not_one() + * fast path runs FIRST: a count above 1 is decremented there (without the + * mutex) and the function reports failure, a saturated count is left + * alone and reports failure SILENTLY, and only a count of exactly 1 + * reaches the mutex-and-drop-to-zero path. */ static bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *m) { - assert(r->refs > 0); - if (--r->refs != 0) + if (refcount_dec_not_one(r)) return false; + mutex_lock(m); + if (!refcount_dec_and_test(r)) { + mutex_unlock(m); + return false; + } return true; } #define H_CAT_(a, b) a##b @@ -507,6 +710,39 @@ def rb_init_lines(ring): rb_free(rb); } +/* Number of events currently attached to the rb (the tree's + * rb->event_list membership: the owner's mmap plus every set_output + * redirect still attached). */ +static int event_list_len(struct perf_buffer *rb) +{ + struct perf_event *pos; + int n = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(pos, &rb->event_list, rb_entry) + n++; + rcu_read_unlock(); + return n; +} + +/* perf_event_set_output(R, A) attach side, signature-faithful to the + * tree: ring_buffer_get(output_event) takes one rb->refcount for the + * redirect and ring_buffer_attach() puts R on rb->event_list. The + * set_output() validation is syscall plumbing outside the extracted + * accounting code and is satisfied by construction here; the tree + * requires: R != A, same cpu (same task when cpu == -1), same clockid, + * same write_backward, same PMU when both events generate AUX, + * output_event not REVOKED, and R->mmap_count == 0 ("Can't redirect + * output if we've got an active mmap()"). That last precondition is + * what makes R's next mmap() its FIRST mmap of the shared rb. */ +static void set_output(struct perf_event *from, struct perf_event *to) +{ + struct perf_buffer *rb = ring_buffer_get(to); + + assert(refcount_read(&from->mmap_count) == 0); + ring_buffer_attach(from, rb); +} + static void perf_event_update_time(struct perf_event *event) { (void)event; } static void perf_event_init_userpage(struct perf_event *event) { (void)event; } static void perf_event_update_userpage(struct perf_event *event) { (void)event; } @@ -528,6 +764,12 @@ def rb_init_lines(ring): static struct perf_event_context ctx; static struct mm_struct mm; static struct file file; +/* the set_output-redirected event: a second event sharing the owner's + * rb, with its own mm (its AUX vma charges/uncharges that mm) */ +static struct perf_event ev_redirect; +static struct perf_event_context ctx_redirect; +static struct mm_struct mm_redirect; +static struct file file_redirect; static void reset_all(unsigned int mm_shift) { @@ -542,13 +784,32 @@ def rb_init_lines(ring): atomic_long_set(&ev.refcount, 2); mm.page_shift = mm_shift; file.private_data = &ev; + memset(&ev_redirect, 0, sizeof(ev_redirect)); + memset(&ctx_redirect, 0, sizeof(ctx_redirect)); + memset(&mm_redirect, 0, sizeof(mm_redirect)); + memset(&file_redirect, 0, sizeof(file_redirect)); + ev_redirect.cpu = 0; + ev_redirect.ctx = &ctx_redirect; + ev_redirect.state = 0; + mutex_init(&ev_redirect.mmap_mutex); + atomic_long_set(&ev_redirect.refcount, 2); + mm_redirect.page_shift = mm_shift; + file_redirect.private_data = &ev_redirect; test_user.perf_mlock = 0; test_user.locked_vm = 0; mm.pinned_vm = 0; + mm_redirect.pinned_vm = 0; rb_allocs = rb_frees = aux_allocs = aux_frees = 0; map_range_calls = 0; + known_alias_charge = 0; + known_ring_alias_charge = 0; warn_count = 0; + warn_lines = 0; warn_printed = 0; + warn_site_n = 0; + /* WARN_ONCE() is once per boot, and every scenario models its own + * boot, so the "already printed" state resets with the rest. */ + memset(refcount_sat_warned, 0, sizeof(refcount_sat_warned)); rb_alloc_aux_fail = 0; map_range_fail = 0; sysctl_perf_event_mlock = 256UL << 20; @@ -564,7 +825,10 @@ def rb_init_lines(ring): sysctl_perf_event_mlock = (long)pages << (PAGE_SHIFT - 10); } -static struct vm_area_struct make_vma_bytes(unsigned long bytes, unsigned long pgoff) +static struct vm_area_struct make_vma_bytes_mm(unsigned long bytes, + unsigned long pgoff, + struct mm_struct *vmm, + struct file *vfile) { struct vm_area_struct v; @@ -573,11 +837,16 @@ def rb_init_lines(ring): v.vm_end = v.vm_start + bytes; v.vm_pgoff = pgoff; v.vm_flags = VM_SHARED | VM_WRITE; - v.vm_mm = &mm; - v.vm_file = &file; + v.vm_mm = vmm; + v.vm_file = vfile; return v; } +static struct vm_area_struct make_vma_bytes(unsigned long bytes, unsigned long pgoff) +{ + return make_vma_bytes_mm(bytes, pgoff, &mm, &file); +} + static void dump(const char *tag) { struct perf_buffer *rb = ev.rb; @@ -593,6 +862,92 @@ def rb_init_lines(ring): static long user_perf_mlock(void) { return test_user.perf_mlock; } static long user_locked_vm(void) { return test_user.locked_vm; } +/* The redirect-side state the tree's own accounting must leave after a + * step: the redirected event's mmap_count, the shared rb's mapping and + * AUX mapping counts, the rb's reference count and event list, the AUX + * allocation's lifetime and the user-side charge. */ +struct redirect_state { + int redirect_mc; + int rb_mmap; + int rb_aux_mmap; + int rb_refcount; + int list; + int aux_allocs; + int aux_frees; + long charge; +}; + +static void dump_redirect(const char *tag, struct perf_buffer *rb) +{ + printf(" %-26s perf_mlock=%ld locked_vm=%ld pinned=%ld " + "redirect.mc=%d rb->mmap_count=%d rb->aux_mmap_count=%d " + "rb->refcount=%d list=%d aux=%d/%d warns=%d/%d\n", tag, + test_user.perf_mlock, test_user.locked_vm, (long)mm.pinned_vm, + ev_redirect.mmap_count.refs, + rb ? rb->mmap_count.refs : -1, + rb ? rb->aux_mmap_count.refs : -1, + rb ? rb->refcount.refs : -1, + rb ? event_list_len(rb) : -1, + aux_allocs, aux_frees, warn_count, warn_lines); +} + +/* The rb is freed by the last close, so this variant reports only state + * that outlives it. */ +static void dump_redirect_final(const char *tag) +{ + printf(" %-26s perf_mlock=%ld locked_vm=%ld pinned=%ld " + "redirect.mc=%d redirect.rb=%d rb=%d/%d aux=%d/%d warns=%d/%d\n", + tag, test_user.perf_mlock, test_user.locked_vm, + (long)mm.pinned_vm, ev_redirect.mmap_count.refs, + ev_redirect.rb != NULL, rb_allocs, rb_frees, aux_allocs, + aux_frees, warn_count, warn_lines); +} + +/* Only call this while the rb is alive (the last close frees it). */ +static int redirect_state_broken(const char *what, struct perf_buffer *rb, + const struct redirect_state *want) +{ + int bad = 0; + + if (ev_redirect.mmap_count.refs != want->redirect_mc) { + printf(" OBSERVED %s: redirect mmap_count %d, expected %d\n", + what, ev_redirect.mmap_count.refs, want->redirect_mc); + bad = 1; + } + if (rb->mmap_count.refs != want->rb_mmap) { + printf(" OBSERVED %s: rb->mmap_count %d, expected %d\n", + what, rb->mmap_count.refs, want->rb_mmap); + bad = 1; + } + if (rb->aux_mmap_count.refs != want->rb_aux_mmap) { + printf(" OBSERVED %s: rb->aux_mmap_count %d, expected %d\n", + what, rb->aux_mmap_count.refs, want->rb_aux_mmap); + bad = 1; + } + if (rb->refcount.refs != want->rb_refcount) { + printf(" OBSERVED %s: rb->refcount %d, expected %d\n", + what, rb->refcount.refs, want->rb_refcount); + bad = 1; + } + if (event_list_len(rb) != want->list) { + printf(" OBSERVED %s: rb event list %d, expected %d\n", + what, event_list_len(rb), want->list); + bad = 1; + } + if (aux_allocs != want->aux_allocs || aux_frees != want->aux_frees) { + printf(" OBSERVED %s: AUX allocations %d/%d, expected %d/%d\n", + what, aux_allocs, aux_frees, + want->aux_allocs, want->aux_frees); + bad = 1; + } + if (user_$charge_counter() != want->charge) { + printf(" OBSERVED %s: $charge_counter %ld, expected %ld\n", + what, user_$charge_counter(), want->charge); + bad = 1; + } + return bad; +} + /* The tree charges exactly one user counter ($charge_counter) and must * leave the other ($spare_counter) untouched: assert both individually, * so a transfer between them cannot pose as unchanged or balanced. */ @@ -617,17 +972,22 @@ def rb_init_lines(ring): { int bad = 0; struct perf_buffer *rb = ev.rb; + long known = known_alias_charge + known_ring_alias_charge; /* each user counter individually: a +leak on one and a -leak on * the other would hide behind a summed total */ - if (user_perf_mlock()) { - printf(" OBSERVED %s: perf_mlock leaked %ld pages\n", - what, user_perf_mlock()); + if (user_$charge_counter() != known) { + printf(" OBSERVED %s: $charge_counter leaked %ld pages\n", + what, user_$charge_counter()); bad = 1; + } else if (known) { + printf(" KNOWN residual: %ld pages " + "(aux alias %ld [#947/#948], ring alias %ld)\n", + known, known_alias_charge, known_ring_alias_charge); } - if (user_locked_vm()) { - printf(" OBSERVED %s: locked_vm leaked %ld pages\n", - what, user_locked_vm()); + if (user_$spare_counter()) { + printf(" OBSERVED %s: $spare_counter leaked %ld pages\n", + what, user_$spare_counter()); bad = 1; } if (mm.pinned_vm) { @@ -873,6 +1233,54 @@ def rb_init_lines(ring): return base; } +/* Measure how many pages the tree charges AGAIN for an alias mapping of + * the same AUX area: map the AUX area once, then map it again, and report + * the difference in the user counter. A tree with the known #947/#948 + * per-alias charge returns 8 (the AUX size); a tree that charges aliases + * only once returns 0. Measuring keeps a scenario's expected charge + * bound to the tree under test instead of to an assumption. */ +static long measure_alias_charge(void) +{ + struct vm_area_struct vring, vaux1, vaux2; + long before, after; + + reset_all(PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) + return -1; + aux_pgoff_target = aux_request(ev.rb, 8); + vaux1 = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux1)) + return -1; + before = user_$charge_counter(); + vaux2 = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux2)) + return -1; + after = user_$charge_counter(); + return after - before; +} + +/* Same measurement for a RING alias: map the ring twice through the + * same event and report the difference. A tree that charges every + * ring alias returns the ring charge; a tree where aliases share the + * first map's single charge returns 0. */ +static long measure_ring_alias_charge(void) +{ + struct vm_area_struct v1, v2; + long before, after; + + reset_all(PAGE_SHIFT); + v1 = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &v1)) + return -1; + before = user_$charge_counter(); + v2 = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &v2)) + return -1; + after = user_$charge_counter(); + return after - before; +} + static int sc_aux_baseline(void) { struct vm_area_struct vring, vaux; @@ -1201,6 +1609,708 @@ def rb_init_lines(ring): return invariants_broken("failed first aux mmap via map_range") || bad; } +/* Positive control: a redirected event whose first mmap is a SUCCESSFUL + * AUX mmap. Proves the harness reaches perf_mmap_aux() through the + * set_output redirect (R->rb was never created by R) and that the + * successful lifecycle balances. Run without limit split: per-mm + * pinned_vm uncharge follows the closing vma's mm, which for cross-mm + * aliases is the tree's existing behaviour outside this scenario's + * scope (user-side charges are what must balance here). */ +static int sc_aux_redirect_lifecycle(void) +{ + struct vm_area_struct vring, vaux; + struct perf_buffer *rb; + int bad = 0; + + reset_all(PAGE_SHIFT); + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + set_output(&ev_redirect, &ev); + if (ev_redirect.rb != rb || rb->refcount.refs != 2 || + event_list_len(rb) != 2 || ev_redirect.mmap_count.refs != 0) { + printf(" UNEXPECTED redirect state (rb=%d rc=%d list=%d mc=%d)\n", + ev_redirect.rb == rb, rb->refcount.refs, + event_list_len(rb), ev_redirect.mmap_count.refs); + return 1; + } + + vaux = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux)) { + printf(" UNEXPECTED redirected aux mmap failed\n"); + return 1; + } + dump("after redirected aux mmap"); + if (ev_redirect.mmap_count.refs != 1 || rb->mmap_count.refs != 2 || + rb->aux_mmap_count.refs != 1 || rb->aux_refcount.refs != 1) { + printf(" UNEXPECTED redirected aux mmap state\n"); + bad = 1; + } + bad |= check_split("after redirected aux mmap", + ring_charge(17, PAGE_SHIFT) + 8, 0); + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + + perf_mmap_close(&vaux); + dump("after redirected aux close"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0 || + rb->mmap_count.refs != 1 || rb->refcount.refs != 1 || + rb->aux_mmap_count.refs != 0 || aux_allocs != aux_frees || + event_list_len(rb) != 1) { + printf(" UNEXPECTED redirect unwind after aux close\n"); + bad = 1; + } + + perf_mmap_close(&vring); + dump("after ring close"); + bad |= invariants_broken("redirected aux lifecycle"); + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* The redirected-event first-map failure: R (set_output, mmap_count 0) + * AUX-mmaps the shared rb and map_range() fails in the perf_mmap() + * dispatcher. perf_mmap_aux() succeeded, ending in + * refcount_inc(&event->mmap_count) FROM ZERO -- real refcount_t + * semantics WARN (ADD_UAF) and leave the count SATURATED, so the + * dispatcher's minimal-close branch (mmap_count == 1) is not taken and + * the full perf_mmap_close() runs against a saturated per-event count: + * its refcount_dec_and_mutex_lock(&event->mmap_count) warns again, + * never reaches 0, and R is left ATTACHED to the rb. Desired state + * after the failure (what the assertions encode): no saturation at + * all, R fully detached with mmap_count 0, rb->mmap_count/refcount + * back to the owner's alone, the AUX allocation freed (aux_refcount 0, + * pages gone), user counters hold only the ring's charge, each mm's + * pinned_vm checked separately. */ +static int sc_aux_redirect_first_map_range_fail(long split) +{ + struct vm_area_struct vring, vaux; + struct perf_buffer *rb; + int bad = 0; + long ring_chg = ring_charge(17, PAGE_SHIFT); + long ring_user, ring_pinned; + + reset_all(PAGE_SHIFT); + if (split < 0) + set_user_lock_limit(0); + else if (split > 0) + /* the AUX charge of 8 pages splits (8 - split) user / + * split pinned against this limit; the owner's ring fits + * the user counter whole */ + set_user_lock_limit(ring_chg + 8 - split); + ring_user = split < 0 ? 0 : ring_chg; + ring_pinned = split < 0 ? ring_chg : 0; + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED first mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + set_output(&ev_redirect, &ev); + dump("after redirect"); + + map_range_fail = 1; + vaux = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (!perf_mmap(&file_redirect, &vaux)) { + printf(" UNEXPECTED failing redirected aux mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump("after FAILED redirected aux mmap (map_range)"); + + /* R detached, owner's refcounts restored */ + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect event still attached after " + "failure (rb=%d mc=%d)\n", + ev_redirect.rb != NULL, ev_redirect.mmap_count.refs); + bad = 1; + } + if (rb->mmap_count.refs != 1 || rb->refcount.refs != 1 || + event_list_len(rb) != 1) { + printf(" OBSERVED rb not restored to owner-only " + "(mmap_count=%d refcount=%d event_list=%d)\n", + rb->mmap_count.refs, rb->refcount.refs, + event_list_len(rb)); + bad = 1; + } + /* the AUX allocation must be gone */ + if (rb->aux_mmap_count.refs != 0 || rb->aux_refcount.refs != 0 || + rb->aux_nr_pages != 0 || aux_allocs != aux_frees) { + printf(" OBSERVED AUX allocation leaked after failure " + "(auxmc=%d auxrc=%d pages=%d allocs=%d frees=%d)\n", + rb->aux_mmap_count.refs, rb->aux_refcount.refs, + rb->aux_nr_pages, aux_allocs, aux_frees); + bad = 1; + } + /* user counters hold only the ring's charge; the AUX charge is + * fully undone, on both users and both mms individually */ + bad |= check_split("after FAILED redirected aux mmap (map_range)", + ring_user, ring_pinned); + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + + /* the owner then closes normally: the rb must free with nothing + * left over */ + perf_mmap_close(&vring); + dump("after ring close"); + bad |= invariants_broken("failed redirected first aux mmap via " + "map_range"); + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* --- focused redirected-AUX scenarios ---------------------------------- + * The redirect fix is about the redirected event's OWN mmap_count: it + * must read 1 after the redirected event's first AUX mmap (not a + * saturated count) and 0 after the close, and the redirect must detach + * from the shared rb on every path. Each scenario below spells out the + * exact expected numbers for the redirect's count, the shared rb's + * counters, the AUX allocation's lifetime and the user-side charge. + * + * The known #947/#948 per-alias charge is NOT hidden: a scenario whose + * AUX area is mapped twice is charged twice by the tree and uncharged + * once, so it states that residual as known_alias_charge and + * invariants_broken() prints it as the KNOWN residual. Every other + * number still has to be exact, and the pre-existing alias scenarios + * still fail on the known defect. + */ + +static int sc_aux_redirect_owner_aux_mapped(void) +{ + struct vm_area_struct vring, vaux_owner, vaux_r; + struct perf_buffer *rb; + struct redirect_state want; + int bad = 0; + + reset_all(PAGE_SHIFT); + /* the owner's AUX area is mapped first, so the redirect's AUX map + * takes perf_mmap_aux()'s alias branch: charged again by a tree + * with the known #947/#948 per-alias charge, uncharged once */ + known_alias_charge = alias_charge; + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED owner ring mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + /* the owner maps the AUX area first */ + vaux_owner = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux_owner)) { + printf(" UNEXPECTED owner aux mmap failed\n"); + return 1; + } + dump_redirect("after owner aux mmap", rb); + + set_output(&ev_redirect, &ev); + + /* now the redirect maps the same AUX area (rb_has_aux() alias) */ + vaux_r = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux_r)) { + printf(" UNEXPECTED redirected aux mmap failed\n"); + return 1; + } + dump_redirect("after redirected aux mmap", rb); + + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 3, + .rb_aux_mmap = 2, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_alias_charge, + }; + bad |= redirect_state_broken("after redirected aux mmap", rb, &want); + + /* the redirect closes its AUX mapping first */ + perf_mmap_close(&vaux_r); + dump_redirect("after redirected aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_alias_charge, + }; + bad |= redirect_state_broken("after redirected aux close", rb, &want); + + /* the owner closes the AUX area: the AUX allocation is freed */ + perf_mmap_close(&vaux_owner); + dump_redirect("after owner aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 1, + .rb_aux_mmap = 0, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 1, + .charge = 17 + known_alias_charge, + }; + bad |= redirect_state_broken("after owner aux close", rb, &want); + + /* the owner closes the ring: the rb is freed */ + perf_mmap_close(&vring); + dump_redirect_final("after owner ring close"); + bad |= invariants_broken("redirected aux over an already mapped aux area"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect not detached at the end " + "(rb=%d mc=%d)\n", ev_redirect.rb != NULL, + ev_redirect.mmap_count.refs); + bad = 1; + } + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* The same shape, with the redirect's AUX map failing at map_range(): + * the dispatcher must unwind it completely -- the redirect detached, + * the rb back to the owner's counts and the AUX allocation left to the + * owner. On unmodified code the redirect's mmap_count reads saturated + * after the AUX map, so the minimal-close branch is not taken and the + * full perf_mmap_close() runs against a saturated count: it warns + * again, never reaches 0 and the redirect stays attached. */ +static int sc_aux_redirect_owner_aux_mapped_fail(void) +{ + struct vm_area_struct vring, vaux_owner, vaux_r; + struct perf_buffer *rb; + struct redirect_state want; + int bad = 0; + + reset_all(PAGE_SHIFT); + known_alias_charge = alias_charge; + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED owner ring mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + vaux_owner = make_vma_bytes(8 << PAGE_SHIFT, aux_pgoff_target); + if (perf_mmap(&file, &vaux_owner)) { + printf(" UNEXPECTED owner aux mmap failed\n"); + return 1; + } + dump_redirect("after owner aux mmap", rb); + + set_output(&ev_redirect, &ev); + + map_range_fail = 1; + vaux_r = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (!perf_mmap(&file_redirect, &vaux_r)) { + printf(" UNEXPECTED failing redirected aux mmap succeeded\n"); + map_range_fail = 0; + return 1; + } + map_range_fail = 0; + dump_redirect("after FAILED redirected aux mmap", rb); + + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_alias_charge, + }; + bad |= redirect_state_broken("after FAILED redirected aux mmap", rb, + &want); + + /* the owner closes the AUX area: the AUX allocation is freed */ + perf_mmap_close(&vaux_owner); + dump_redirect("after owner aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 1, + .rb_aux_mmap = 0, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 1, + .charge = 17 + known_alias_charge, + }; + bad |= redirect_state_broken("after owner aux close", rb, &want); + + perf_mmap_close(&vring); + dump_redirect_final("after owner ring close"); + bad |= invariants_broken("failed redirected aux over an already " + "mapped aux area"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect not detached after the failure " + "(rb=%d mc=%d)\n", ev_redirect.rb != NULL, + ev_redirect.mmap_count.refs); + bad = 1; + } + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* The owner's ring VMA closes FIRST while the redirect's AUX mapping + * is still live: the rb survives on the redirect's mapping, the AUX + * allocation is freed by the redirect's close and the ring's charge is + * undone only when the rb's mapping count reaches 0. */ +static int sc_aux_redirect_owner_ring_closes_first(void) +{ + struct vm_area_struct vring, vaux_r; + struct perf_buffer *rb; + struct redirect_state want; + int bad = 0; + + reset_all(PAGE_SHIFT); + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED owner ring mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + set_output(&ev_redirect, &ev); + + /* the redirect maps the AUX area (the rb's first AUX mapping) */ + vaux_r = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux_r)) { + printf(" UNEXPECTED redirected aux mmap failed\n"); + return 1; + } + dump_redirect("after redirected aux mmap", rb); + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25, + }; + bad |= redirect_state_broken("after redirected aux mmap", rb, &want); + + /* the owner closes its ring while the redirect's AUX stays live */ + perf_mmap_close(&vring); + dump_redirect("after owner ring close (aux live)", rb); + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 1, + .rb_aux_mmap = 1, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25, + }; + bad |= redirect_state_broken("after owner ring close (aux live)", rb, + &want); + if (ev.rb != NULL || ev.mmap_count.refs != 0) { + printf(" OBSERVED owner not detached after its ring close " + "(rb=%d mc=%d)\n", ev.rb != NULL, ev.mmap_count.refs); + bad = 1; + } + + /* the redirect closes the AUX area: the AUX allocation and the + * ring's charge are undone here, the rb is freed */ + perf_mmap_close(&vaux_r); + dump_redirect_final("after redirected aux close"); + bad |= invariants_broken("owner ring closed before the redirected aux"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect not detached at the end " + "(rb=%d mc=%d)\n", ev_redirect.rb != NULL, + ev_redirect.mmap_count.refs); + bad = 1; + } + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* The redirect maps the AUX area a second time: the AUX area is shared + * with its own first mapping (rb_has_aux() alias branch) and the + * redirect's mmap_count must go 1 -> 2, not saturate. */ +static int sc_aux_redirect_second_aux_map(void) +{ + struct vm_area_struct vring, vaux_r1, vaux_r2; + struct perf_buffer *rb; + struct redirect_state want; + int bad = 0; + + reset_all(PAGE_SHIFT); + /* the second mapping of the same AUX area is charged again by a + * tree with the known #947/#948 per-alias charge, uncharged once */ + known_alias_charge = alias_charge; + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED owner ring mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + set_output(&ev_redirect, &ev); + + vaux_r1 = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux_r1)) { + printf(" UNEXPECTED first redirected aux mmap failed\n"); + return 1; + } + dump_redirect("after first redirected aux mmap", rb); + + vaux_r2 = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux_r2)) { + printf(" UNEXPECTED second redirected aux mmap failed\n"); + return 1; + } + dump_redirect("after second redirected aux mmap", rb); + want = (struct redirect_state){ + .redirect_mc = 2, + .rb_mmap = 3, + .rb_aux_mmap = 2, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_alias_charge, + }; + bad |= redirect_state_broken("after second redirected aux mmap", rb, + &want); + + /* close the second mapping first, then the first */ + perf_mmap_close(&vaux_r2); + dump_redirect("after second redirected aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_alias_charge, + }; + bad |= redirect_state_broken("after second redirected aux close", rb, + &want); + + perf_mmap_close(&vaux_r1); + dump_redirect("after first redirected aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 1, + .rb_aux_mmap = 0, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 1, + .charge = 17 + known_alias_charge, + }; + bad |= redirect_state_broken("after first redirected aux close", rb, + &want); + + perf_mmap_close(&vring); + dump_redirect_final("after owner ring close"); + bad |= invariants_broken("redirected event mapping the aux area twice"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect not detached at the end " + "(rb=%d mc=%d)\n", ev_redirect.rb != NULL, + ev_redirect.mmap_count.refs); + bad = 1; + } + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* The redirect maps the NORMAL ring (offset 0, the owner's geometry) + * after its first AUX map: perf_mmap_rb()'s alias branch, reachable + * once the redirect holds mmap_count 1. The current tree admits this, + * and both events then share ONE user_page: a userpage update from + * either event overwrites the other's metadata. That shared-metadata + * exposure is a pre-existing limitation of the rb-alias design, + * documented and asserted here as observed -- not changed by the + * redirect fix. The ring alias's charge follows the tree under test + * (measured like the AUX alias charge) and is recorded in + * known_ring_alias_charge, so the scenario states the tree's own + * per-alias ring policy instead of assuming one. */ +static int sc_aux_redirect_then_ring_map(void) +{ + struct vm_area_struct vring, vaux_r, vring_r; + struct perf_buffer *rb; + struct redirect_state want; + int bad = 0; + + reset_all(PAGE_SHIFT); + known_ring_alias_charge = ring_alias_charge; + + vring = make_vma_bytes(17 << PAGE_SHIFT, 0); + if (perf_mmap(&file, &vring)) { + printf(" UNEXPECTED owner ring mmap failed\n"); + return 1; + } + rb = ev.rb; + aux_pgoff_target = aux_request(rb, 8); + + set_output(&ev_redirect, &ev); + + /* the redirect's first mmap: the AUX area (allocates it, + * mmap_count 0 -> 1) */ + vaux_r = make_vma_bytes_mm(8 << PAGE_SHIFT, aux_pgoff_target, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vaux_r)) { + printf(" UNEXPECTED redirected aux mmap failed\n"); + return 1; + } + dump_redirect("after redirected aux mmap", rb); + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25, + }; + bad |= redirect_state_broken("after redirected aux mmap", rb, &want); + + /* now the redirect maps the ring itself (rb alias branch) */ + vring_r = make_vma_bytes_mm(17 << PAGE_SHIFT, 0, + &mm_redirect, &file_redirect); + if (perf_mmap(&file_redirect, &vring_r)) { + printf(" UNEXPECTED redirected ring mmap failed\n"); + return 1; + } + dump_redirect("after redirected ring mmap", rb); + want = (struct redirect_state){ + .redirect_mc = 2, + .rb_mmap = 3, + .rb_aux_mmap = 1, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_ring_alias_charge, + }; + bad |= redirect_state_broken("after redirected ring mmap", rb, &want); + + /* close in reverse order: ring alias, AUX, owner ring */ + perf_mmap_close(&vring_r); + dump_redirect("after redirected ring close", rb); + want = (struct redirect_state){ + .redirect_mc = 1, + .rb_mmap = 2, + .rb_aux_mmap = 1, + .rb_refcount = 2, + .list = 2, + .aux_allocs = 1, + .aux_frees = 0, + .charge = 25 + known_ring_alias_charge, + }; + bad |= redirect_state_broken("after redirected ring close", rb, &want); + + perf_mmap_close(&vaux_r); + dump_redirect("after redirected aux close", rb); + want = (struct redirect_state){ + .redirect_mc = 0, + .rb_mmap = 1, + .rb_aux_mmap = 0, + .rb_refcount = 1, + .list = 1, + .aux_allocs = 1, + .aux_frees = 1, + .charge = 17 + known_ring_alias_charge, + }; + bad |= redirect_state_broken("after redirected aux close", rb, &want); + + perf_mmap_close(&vring); + dump_redirect_final("after owner ring close"); + bad |= invariants_broken("redirected event mapping the ring after " + "its aux"); + if (ev_redirect.rb != NULL || ev_redirect.mmap_count.refs != 0) { + printf(" OBSERVED redirect not detached at the end " + "(rb=%d mc=%d)\n", ev_redirect.rb != NULL, + ev_redirect.mmap_count.refs); + bad = 1; + } + if (mm_redirect.pinned_vm != 0) { + printf(" OBSERVED redirect mm pinned_vm %ld, expected 0\n", + (long)mm_redirect.pinned_vm); + bad = 1; + } + return bad; +} + +/* One RESULT line per scenario: FAIL on any broken invariant, otherwise + * PASS-KNOWN-RESIDUAL when the scenario ended on the recorded per-alias + * residual, plain PASS only when nothing was left over. */ +static int result(const char *scenario, const char *geo, const char *limits, + int bad) +{ + const char *status = bad ? "FAIL" + : (known_alias_charge || known_ring_alias_charge) + ? "PASS-KNOWN-RESIDUAL" : "PASS"; + + printf("RESULT scenario=%s%s%s%s%s status=%s\n", + scenario, + geo ? " geometry=" : "", geo ? geo : "", + limits ? " limits=" : "", limits ? limits : "", + status); + return bad; +} + int main(void) { int failures = 0; @@ -1214,18 +2324,43 @@ def rb_init_lines(ring): const char *lim_names[] = { "nosplit", "partialsplit3", "fullspill" }; int g, s; + /* how much does THIS tree charge again for an alias mapping of the + * same AUX area? Measured, so the alias scenarios' expected charge + * follows the tree under test. */ + alias_charge = measure_alias_charge(); + printf("-- measured per-alias AUX charge: %ld pages%s\n", + alias_charge, + alias_charge ? " (known #947/#948 per-alias charge)" : ""); + if (alias_charge < 0) { + /* a failed probe mmap makes the measurement invalid: the + * alias scenarios' expected charges would be built on + * nothing. Hard fail instead of running them. */ + printf("RESULT scenario=alias-charge-probe status=FAIL " + "(measurement mmap failed, run aborted)\n"); + printf("SUMMARY failures=1\n"); + return 1; + } + printf("RESULT scenario=alias-charge-probe status=PASS\n"); + + /* the tree's own per-alias RING policy, measured the same way */ + ring_alias_charge = measure_ring_alias_charge(); + printf("-- measured per-alias ring charge: %ld pages%s\n", + ring_alias_charge, + ring_alias_charge ? " (charged per ring alias)" : ""); + if (ring_alias_charge < 0) { + printf("RESULT scenario=ring-alias-charge-probe status=FAIL " + "(measurement mmap failed, run aborted)\n"); + printf("SUMMARY failures=1\n"); + return 1; + } + printf("RESULT scenario=ring-alias-charge-probe status=PASS\n"); + for (g = 0; g < 3; g++) { if (g == 2 && !RB_USER4K) continue; printf("-- scenario rb-baseline geometry=%s\n", geo_names[g]); - if (sc_rb_baseline(geos[g][0], geos[g][1], 0)) { - printf("RESULT scenario=rb-baseline geometry=%s status=FAIL\n", - geo_names[g]); - failures++; - } else { - printf("RESULT scenario=rb-baseline geometry=%s status=PASS\n", - geo_names[g]); - } + failures += result("rb-baseline", geo_names[g], NULL, + sc_rb_baseline(geos[g][0], geos[g][1], 0)); } for (s = 0; s < 3; s++) { for (g = 0; g < 3; g++) { @@ -1253,93 +2388,66 @@ def rb_init_lines(ring): } printf("-- scenario rb-duplicate geometry=%s limits=%s\n", geo_names[g], lim); - if (sc_rb_duplicate(geos[g][0], geos[g][1], split)) { - printf("RESULT scenario=rb-duplicate geometry=%s limits=%s status=FAIL\n", - geo_names[g], lim); - failures++; - } else { - printf("RESULT scenario=rb-duplicate geometry=%s limits=%s status=PASS\n", - geo_names[g], lim); - } + failures += result("rb-duplicate", geo_names[g], lim, + sc_rb_duplicate(geos[g][0], + geos[g][1], split)); } } for (s = 0; s < 3; s += 2) { /* nosplit and fullspill */ printf("-- scenario rb-alias-map-range-fail limits=%s\n", lim_names[s]); - if (sc_rb_alias_map_range_fail(splits[s])) { - printf("RESULT scenario=rb-alias-map-range-fail limits=%s status=FAIL\n", - lim_names[s]); - failures++; - } else { - printf("RESULT scenario=rb-alias-map-range-fail limits=%s status=PASS\n", - lim_names[s]); - } + failures += result("rb-alias-map-range-fail", NULL, lim_names[s], + sc_rb_alias_map_range_fail(splits[s])); printf("-- scenario rb-first-map-range-fail limits=%s\n", lim_names[s]); - if (sc_rb_first_map_range_fail(splits[s])) { - printf("RESULT scenario=rb-first-map-range-fail limits=%s status=FAIL\n", - lim_names[s]); - failures++; - } else { - printf("RESULT scenario=rb-first-map-range-fail limits=%s status=PASS\n", - lim_names[s]); - } + failures += result("rb-first-map-range-fail", NULL, lim_names[s], + sc_rb_first_map_range_fail(splits[s])); } printf("-- scenario aux-baseline\n"); - if (sc_aux_baseline()) { - printf("RESULT scenario=aux-baseline status=FAIL\n"); - failures++; - } else { - printf("RESULT scenario=aux-baseline status=PASS\n"); - } + failures += result("aux-baseline", NULL, NULL, sc_aux_baseline()); for (s = 0; s < 3; s++) { printf("-- scenario aux-duplicate limits=%s\n", lim_names[s]); - if (sc_aux_duplicate(splits[s])) { - printf("RESULT scenario=aux-duplicate limits=%s status=FAIL\n", - lim_names[s]); - failures++; - } else { - printf("RESULT scenario=aux-duplicate limits=%s status=PASS\n", - lim_names[s]); - } + failures += result("aux-duplicate", NULL, lim_names[s], + sc_aux_duplicate(splits[s])); } printf("-- scenario aux-limit-reject\n"); - if (sc_aux_limit_fail()) { - printf("RESULT scenario=aux-limit-reject status=FAIL\n"); - failures++; - } else { - printf("RESULT scenario=aux-limit-reject status=PASS\n"); - } + failures += result("aux-limit-reject", NULL, NULL, sc_aux_limit_fail()); printf("-- scenario aux-alloc-fail\n"); - if (sc_aux_alloc_fail()) { - printf("RESULT scenario=aux-alloc-fail status=FAIL\n"); - failures++; - } else { - printf("RESULT scenario=aux-alloc-fail status=PASS\n"); - } + failures += result("aux-alloc-fail", NULL, NULL, sc_aux_alloc_fail()); printf("-- scenario aux-alias-size-reject\n"); - if (sc_aux_alias_size_reject()) { - printf("RESULT scenario=aux-alias-size-reject status=FAIL\n"); - failures++; - } else { - printf("RESULT scenario=aux-alias-size-reject status=PASS\n"); - } + failures += result("aux-alias-size-reject", NULL, NULL, + sc_aux_alias_size_reject()); for (s = 0; s < 3; s += 2) { printf("-- scenario aux-alias-map-range-fail limits=%s\n", lim_names[s]); - if (sc_aux_alias_map_range_fail(splits[s])) { - printf("RESULT scenario=aux-alias-map-range-fail limits=%s status=FAIL\n", - lim_names[s]); - failures++; - } else { - printf("RESULT scenario=aux-alias-map-range-fail limits=%s status=PASS\n", - lim_names[s]); - } + failures += result("aux-alias-map-range-fail", NULL, lim_names[s], + sc_aux_alias_map_range_fail(splits[s])); } printf("-- scenario aux-first-map-range-fail\n"); - if (sc_aux_first_map_range_fail()) { - printf("RESULT scenario=aux-first-map-range-fail status=FAIL\n"); - failures++; - } else { - printf("RESULT scenario=aux-first-map-range-fail status=PASS\n"); + failures += result("aux-first-map-range-fail", NULL, NULL, + sc_aux_first_map_range_fail()); + printf("-- scenario aux-redirect-lifecycle\n"); + failures += result("aux-redirect-lifecycle", NULL, NULL, + sc_aux_redirect_lifecycle()); + for (s = 0; s < 3; s++) { + printf("-- scenario aux-redirect-first-map-range-fail limits=%s\n", + lim_names[s]); + failures += result("aux-redirect-first-map-range-fail", NULL, + lim_names[s], + sc_aux_redirect_first_map_range_fail(splits[s])); } + printf("-- scenario aux-redirect-owner-aux-mapped\n"); + failures += result("aux-redirect-owner-aux-mapped", NULL, NULL, + sc_aux_redirect_owner_aux_mapped()); + printf("-- scenario aux-redirect-owner-aux-mapped-fail\n"); + failures += result("aux-redirect-owner-aux-mapped-fail", NULL, NULL, + sc_aux_redirect_owner_aux_mapped_fail()); + printf("-- scenario aux-redirect-owner-ring-closes-first\n"); + failures += result("aux-redirect-owner-ring-closes-first", NULL, NULL, + sc_aux_redirect_owner_ring_closes_first()); + printf("-- scenario aux-redirect-second-aux-map\n"); + failures += result("aux-redirect-second-aux-map", NULL, NULL, + sc_aux_redirect_second_aux_map()); + printf("-- scenario aux-redirect-then-ring-map\n"); + failures += result("aux-redirect-then-ring-map", NULL, NULL, + sc_aux_redirect_then_ring_map()); printf("SUMMARY failures=%d\n", failures); return failures ? 1 : 0; @@ -1349,9 +2457,40 @@ def rb_init_lines(ring): HARNESS_GLOBALS = ''' static unsigned long aux_pgoff_target; static long charge_expect; +/* Pages the tree charges AGAIN for an alias mapping of the same AUX area: + * the per-alias charge defect fixed outside this change (#947/#948). It + * is MEASURED from the tree under test (measure_alias_charge()), not + * assumed, so a scenario's expected charge states the tree's own policy. + * A scenario that exercises an alias mapping sets known_alias_charge to + * it: the residual is reported, not hidden, and every other invariant + * is still exact. known_ring_alias_charge is the same shape for a ring + * mapping taken as an alias (charged per mapping by this tree, + * uncharged once). */ +static long known_alias_charge; +static long known_ring_alias_charge; +static long alias_charge; +static long ring_alias_charge; ''' +def provenance(tree): + """Exact provenance of the tree under test: path, git HEAD, dirty state.""" + def git(*a): + return subprocess.run(['git', '-C', str(tree), *a], + capture_output=True, text=True).stdout.strip() + head = git('rev-parse', 'HEAD') + if not head: + import hashlib + core = Path(tree) / 'kernel/events/core.c' + digest = hashlib.sha256(core.read_bytes()).hexdigest()[:16] \ + if core.is_file() else 'no kernel/events/core.c' + return '%s (not a git worktree; core.c sha256[:16]=%s)' % (tree, digest) + dirty = git('status', '--porcelain', '--', + 'kernel/events', 'tools/asahi').splitlines() + note = ('dirty: %s' % ', '.join(dirty)) if dirty else 'kernel/events clean' + return '%s @ %s (%s)' % (tree, head, note) + + def source_notes(core): """Probe the source shape so the tree under test is reported honestly.""" notes = [] @@ -1399,17 +2538,23 @@ def build(tree, out): body = definition(internal, 'perf_metadata_size') if body: helpers.append(body) - body = definition(core, 'perf_mmap_offset') - if body is None: - raise SystemExit('user4k tree lacks perf_mmap_offset()') - helpers.append(body) + extracted = [] for name in ('perf_mmap_calc_limits', 'perf_mmap_account', 'perf_mmap_unaccount', 'perf_mmap_rb', 'perf_mmap_aux', 'perf_mmap_close', 'perf_mmap'): body = definition(core, name) if body is None: raise SystemExit('missing %s in core.c' % name) + extracted.append(body) + # perf_mmap() and perf_mmap_aux() call perf_mmap_offset() in the trees + # that have it, so extract it whenever they need it -- and before them, + # since the extracted code calls it. + if any('perf_mmap_offset' in body for body in extracted): + body = definition(core, 'perf_mmap_offset') + if body is None: + raise SystemExit('tree lacks perf_mmap_offset()') helpers.append(body) + helpers.extend(extracted) aux_decl = re.search(r'extern int rb_alloc_aux\(([^;]+)\);', internal) if aux_decl is None: @@ -1445,7 +2590,8 @@ def build(tree, out): rb_alloc_decl = ('struct perf_buffer *rb_alloc(int nr_pages, ' 'long watermark, int cpu, int flags);') - source = HARNESS_GLOBALS + Template(HARNESS).substitute({ + template = Template(HARNESS_GLOBALS + HARNESS) + source = template.substitute({ 'RB_USER4K': '1' if user4k else '0', 'rb_alloc_decl': rb_alloc_decl, 'rb_alloc_aux_decl': 'static int rb_alloc_aux(%s);' % params, @@ -1486,6 +2632,7 @@ def build(tree, out): print('harness source kept at %s' % src) overall = 2 continue + print('provenance: %s' % provenance(tree)) print('profile: %s, user counter: %s' % ('user4k (data_size rb_alloc, per-mm page shift)' if user4k else 'plain (nr_pages rb_alloc, PAGE_SHIFT)', counter)) @@ -1495,10 +2642,19 @@ def build(tree, out): sys.stdout.write(result.stdout) if result.stderr.strip(): sys.stdout.write(result.stderr) + if result.returncode < 0: + # a crashing binary (signal) loses the tempdir; keep the + # extracted source for a backtrace against the exact code + keep = Path.cwd() / binary.with_suffix('.c').name + try: + keep.write_text(binary.with_suffix('.c').read_text()) + print('harness source kept at %s' % keep) + except OSError as error: + print('could not keep harness source: %s' % error) if result.returncode != 0 and overall == 0: overall = 1 print('=' * 72) print('OVERALL: %s' % ('tested invariants failed' if overall == 1 else 'build/source error' if overall == 2 - else 'tested single-user, single-mm lifecycles balanced')) + else 'tested lifecycles balanced')) raise SystemExit(overall) From 8fc6e1a5485441edf1bb0c77e723b62016fd136e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 10 Nov 2026 18:11:56 +1100 Subject: [PATCH 381/392] drm/asahi: hide the teardown errors of a closing client again Since "drm/asahi: finish an exiting client's work before its mappings go", a DRM file drops its queues before its VMs. Destroying a G17 queue fails its queued work with ENOENT (VM context gone) or ECANCELED (queue destroyed), which nobody can see once the client is closing, and the VM status hides those errors only after the client's VM handle is gone. With the VMs now dropped last, every such failure of a client that simply exited was logged as "GPU work of a client failed (ENOENT, ...); the client sees a lost device". gpubench-stress xq.exit and life.churn fail on that kernel-log line on a J616s, where fleet46 passed them. Mark the status of every VM of the file as closed when the file starts closing, before any queue goes, as closing the VM handle first used to. Device loss of such a client (EIO, ETIMEDOUT, ENODEV) is still warned about. Fixes: dbe881e3847a ("drm/asahi: finish an exiting client's work before its mappings go") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/file.rs | 15 +++++++++++++++ drivers/gpu/drm/asahi/g17/status.rs | 22 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index ffe5f0e846930..cc3ccb374ea11 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1623,5 +1623,20 @@ impl File { impl Drop for File { fn drop(&mut self) { mod_pr_debug!("[File {}]: Closing...\n", self.id); + // The queues drop before the VMs (see the field order), and destroying a queue fails + // its queued work with ENOENT or ECANCELED. Nobody sees those any more: the client + // closed its file. Say so before any queue goes, as a VM handle closed first did. + // SAFETY: `drop` runs on the pinned file in place, and nothing here moves it. + let this = unsafe { Pin::new_unchecked(&*self) }; + let mut index = 0; + while let Some((idx, vm)) = this.vms().find(index, usize::MAX) { + if let Some(status) = vm.borrow().vm.status() { + status.mark_client_closed(); + } + if idx == usize::MAX { + break; + } + index = idx + 1; + } } } diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 75df8ad0480ae..db9dffc5b3d4b 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -188,10 +188,16 @@ impl VmStatus { Ok(()) } + /// The client closed its DRM file: nobody sees the teardown errors of its queued work any + /// more, even before its VM handle goes (the file drops its queues first). + pub(crate) fn mark_client_closed(&self) { + self.client_closed.store(true, Ordering::Release); + } + /// The client's VM handle is gone and nobody reads the mirror any more. The /// mapping and object reference drop outside the mutex (GEM release may sleep). pub(crate) fn release_mirror(&self) { - self.client_closed.store(true, Ordering::Release); + self.mark_client_closed(); let taken = self.mirror.lock().take(); drop(taken); } @@ -218,6 +224,20 @@ mod tests { assert!(!status.unobserved_teardown(ENODEV)); } + #[test] + fn a_closing_file_hides_teardown_errors_before_its_vm_goes() { + let Ok(status) = KBox::pin_init(VmStatus::new(), GFP_KERNEL) else { + panic!("allocation failed"); + }; + status.mark_client_closed(); + assert!(status.unobserved_teardown(ENOENT)); + assert!(status.unobserved_teardown(ECANCELED)); + assert!(!status.unobserved_teardown(ETIMEDOUT)); + status.report_failure(ENOENT); + assert!(!status.warned.load(Ordering::Acquire)); + assert!(status.mirror.lock().is_none()); + } + #[test] fn a_closed_client_still_keeps_its_first_error() { let Ok(status) = KBox::pin_init(VmStatus::new(), GFP_KERNEL) else { From ccbfa492825580ef375b2b098dc84bd69deccd5d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 10 Nov 2026 18:30:03 +1100 Subject: [PATCH 382/392] tools/asahi: run-host-tests: drop the removed pmu-rtc wrong-base mutation pmu-rtc-tests.py no longer has a wrong-base mutation (integration replaced it with wrong-control, maverick-event and stowe-offset), so '--mutation wrong-base' exits 2 and run-host-tests fails. The entry came back with the perf mmap accounting registration (32246be58de9). Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/run-host-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 4b31cfd7761fa..6900b7fe82f5f 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -165,7 +165,7 @@ def mutations(flag, names, base, expect=KILL): 'pmp-j613-hwmon-tests.py': [([], PASS)], 'pmu-rtc-tests.py': [(out(), PASS)] + mutations('--mutation', ('ack-clobber', 'arm-ack', 'offset-seconds', - 'offset-sign', 'event-bit', 'wrong-base', 'wrong-control', + 'offset-sign', 'event-bit', 'wrong-control', 'maverick-event', 'stowe-offset'), out()), 'perf-mmap-alias-lifecycle-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], From 135768af3b02b6ee53356f171bfd66ac4bb57bbe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:22 -0500 Subject: [PATCH 383/392] arm64: dts: apple: t6050-j714s: Add the keyboard and trackpad The internal keyboard and trackpad are HID devices behind the MTP's DockChannel FIFO (ADT /arm-io/dockchannel-mtp/mtp-transport, with the multi-touch, keyboard, stm, actuator and tp-accel children). Describe the transport as t6050-j716s.dts does, with the J714s trackpad firmware name. Its AFE reset is the ADT mtp function-afe-reset, SMC pcIO template (0x10000, 0x11c0000), which is SMC GPIO gp1c, so the unused pcIO node goes. The keyboard alias lets the loader fill the layout from the ADT's keyboard node. The keyboard backlight is the ADT pwm0/kbd-backlight at 25 kHz (default-hz 25000), channel 0 of the fast PWM, as on J716s. The M5 Pro's default SSD boot has run these nodes from its DT overlay since 2026-11-09: Hyprland sees apple-mtp-keyboard and apple-mtp-multi-touch. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..32005ea0bcc50 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -13,6 +13,7 @@ /dts-v1/; +#include #include "t6050-max.dtsi" / { @@ -44,6 +45,79 @@ status = "disabled"; }; +/* The keyboard backlight: ADT pwm0/kbd-backlight, channel 0 at 25 kHz */ +/ { + aliases { + /* The loader fills the keyboard layout from the ADT's keyboard node */ + keyboard = &keyboard; + }; + + led-controller { + compatible = "pwm-leds"; + + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&fpwm0 { + status = "okay"; +}; + +/* + * The keyboard and trackpad: the HID transport on the MTP's DockChannel FIFO + * (ADT dockchannel-mtp/mtp-transport: multi-touch, keyboard, stm, actuator, + * tp-accel), as on J716s. + */ +&mtp_dockchannel { + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + /* + * ADT mtp function-afe-reset is the SMC pcIO template (0x10000, 0x11c0000), + * the same line as SMC GPIO gp1c, active low, which gpio-macsmc drives. + */ + apple,afe-reset-gpios = <&smc_gpio_low 28 GPIO_ACTIVE_LOW>; + + multi-touch { + apple,power-method = <2>; + firmware-name = "apple/tpmtfw-j714s.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp-accel { + }; + }; +}; + +/* Nothing here drives the SMC pcIO outputs, and the AFE reset above uses gp1c */ +/delete-node/ &smc_pcio; + #define T6050_J714S #include "t6050-peripherals.dtsi" From e0834cfff9dc7ef41718dc883e0ff232846b4361 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:23 -0500 Subject: [PATCH 384/392] arm64: dts: apple: t6050-j714s: Add the N1 Wi-Fi and Bluetooth PCIe port 0 carries the N1 (ADT /arm-io/apcie0/pci-bridge0 with centauri-control, centauri-alpha and centauri-beta). apcie0 gives port 0's registers and IRQ 1523, and pci-bridge0's function-perst is GPIO 80. Describe the port and the N1's three functions as t6050-j716s.dts does, with the J714s firmware table and platform ID (ADT /chosen board-id 8, so 0x00086050). The boot loader trains the link and then enables port 0 and its DART; the controller itself is enabled here. Pin 112 takes an SError when read (as on J716c and J716s), so only pin 80, the PERST#, is exposed. The M5 Pro's default SSD boot has run these nodes from its DT overlay since 2026-11-09: the N1 at 01:00.0-2, Wi-Fi at 5 GHz through NetworkManager and Bluetooth pairing. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 78 ++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..b2d5c3300165a 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -39,9 +39,83 @@ }; }; -/* PCIe port 0 is not described for this board yet */ +/* + * PCIe port 0: the N1 (Wi-Fi and Bluetooth), as on J716s. ADT apcie0 gives port + * 0's registers and IRQ 1523, and pci-bridge0's PERST# is GPIO 80; the N1's three + * functions are on DART streams 1-3. The boot loader trains the link and then + * enables port 0 and its DART. Port 1 is the SD reader, not described yet. + */ +/ { + aliases { + /* The loader fills the N1's factory addresses through these */ + wifi0 = &n1_alpha; + bluetooth0 = &n1_beta; + }; +}; + &pcie0 { - status = "disabled"; + reg = <0x1c 0xb0000000 0x0 0x10000000>, + <0x4 0x14000000 0x0 0x4000>, + <0x4 0x10028000 0x0 0x8000>, + <0x4 0x17010000 0x0 0x4000>, + <0x4 0x10024000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "phy0", "intr2axi0"; + interrupts = ; + interrupt-names = "port0"; + iommu-map = <0x100 &dart_apcie0 1 3>; + bus-range = <0 4>; + status = "okay"; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0 0 0 0>; + bus-range = <1 1>; + apple,link-trained-by-bootloader; + /* PERST#: ADT pci-bridge0 function-perst, GPIO 80 */ + reset-gpios = <&pinctrl_ap 80 GPIO_ACTIVE_LOW>; + status = "disabled"; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + + /* platform ID (ADT /chosen board-id 0x08 << 16) | chip 0x6050 */ + n1_control: control@0,0 { + compatible = "pci106b,1900"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + firmware-name = "apple/centauri/j714s-ftab.bin"; + apple,centauri-platform-id = <0x00086050>; + apple,centauri-protocol = <27>; + }; + + n1_alpha: wifi@0,1 { + compatible = "pci106b,1902"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + local-mac-address = [00 00 00 00 00 00]; + }; + + n1_beta: bluetooth@0,2 { + compatible = "pci106b,1903"; + reg = <0x10200 0x0 0x0 0x0 0x0>; + local-bd-address = [00 00 00 00 00 00]; + }; + }; +}; + +/* + * Pin 112 takes an SError when read (as on J716c and J716s), so only the pin a + * node here names is exposed: 80, PCIe port 0's PERST#. + */ +&pinctrl_ap { + gpio-reserved-ranges = <0 80>, <81 140>; }; #define T6050_J714S From 794fd92b3292ed65a36c57ac875c03b2c010c453 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:23 -0500 Subject: [PATCH 385/392] arm64: dts: apple: t6050-j714s: Add the SMC sensors Add the shared SMC fan, power, voltage and current channels and the temperature discovery t6050-die0.dtsi provides, as t6050-j716s.dts does: the shared temperature nodes and apple,soc-die-keys go so that discovery runs, and the fans use the T6050 mode keys F0md/F1md. The SMC (ADT /arm-io/smc) keeps driving the fans. The M5 Pro's default SSD boot has run this from its DT overlay since 2026-11-09; its Total System Power channel (PSTR) is the one the load measurements read. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..e0226053f12db 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -100,5 +100,37 @@ #include "t6050-j714s-pmp-tunables.dtsi" +/* + * SMC sensors, as on J716s: the shared fan, power, voltage and current keys, plus + * every readable float T* key through apple,discover-temperatures in + * t6050-die0.dtsi, labelled with the key name. A key this SMC lacks only drops + * its channel. + * + * Discovery refuses to run once any temperature channel exists, so the shared + * temperature nodes go (their keys are discovered anyway), and so does + * apple,soc-die-keys, whose channel would also count. Only the macsmc_soc_die + * thermal zone is lost. + */ +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" + +&smc_hwmon { + /delete-property/ apple,soc-die-keys; + /delete-node/ temperature-TH0x; + /delete-node/ temperature-TB0T; + /delete-node/ temperature-TCHP; + /delete-node/ temperature-TW0P; + + /* The T6050 SMC spells the fan mode keys F0md/F1md. The SMC drives the fans. */ + fan-F0Ac { + apple,fan-mode = "F0md"; + }; + + fan-F1Ac { + apple,fan-mode = "F1md"; + }; +}; + /* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ #include "t6050-j716c-isp.dtsi" From bd38e9dd45dcb156d5e768a709ef44f540a05dc9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:23 -0500 Subject: [PATCH 386/392] arm64: dts: apple: t6050-j714s: Describe the two USB-C ports From the J714sAP ADT: hpm0 (nub-spmi-a0 0x0c, left-back) is the debug port; hpm1 (nub-spmi-a0 0x0a, left-front) sits behind ATC1 / usb-drd1; hpm2 (nub-spmi-a1 0x0c, right) behind ATC2 / usb-drd2; hpm5 (0x08) is MagSafe. There is no usb-drd3 and no nub-spmi-a2. Describe the left-front and right ports as t6050-j716s.dts does: each SN201202x with its connector graph into its dwc3 (USB2) and combo PHY (SuperSpeed), and ATC2's PHY, mailbox, controller, DARTs and power states enabled. The inventory's nub-spmi-a1 carries the generation 4 bus's three register windows and IRQ (ADT nub-spmi-a1); the driver polls the bus and refuses an interrupt, so it takes the first window only, as J716s's node does. ATC3's USB chain and nub-spmi-a2 are turned off: the board has neither. The M5 Pro's default SSD boot has run these nodes from its DT overlay since 2026-11-09: Type-C port0 and port1 with a partner, two dwc3 controllers bound. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 174 ++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..f7f7efe535ac6 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -47,6 +47,180 @@ #define T6050_J714S #include "t6050-peripherals.dtsi" +/* + * Type-C, from the ADT: + * ATC0 usb-drd0 left-back hpm0 nub-spmi-a0 0x0c (the debug port, not described) + * ATC1 usb-drd1 left-front hpm1 nub-spmi-a0 0x0a + * ATC2 usb-drd2 right hpm2 nub-spmi-a1 0x0c + * hpm5 nub-spmi-a0 0x08 (MagSafe, not a USB-C connector) + * There is no usb-drd3 and no nub-spmi-a2, so ATC3's USB chain and that bus stay + * off. Both described ports are wired as on J716s. + */ +&atc3_phy_mbox { + status = "disabled"; +}; + +&atc3_phy { + status = "disabled"; +}; + +&usb_drd3 { + status = "disabled"; +}; + +&usb_drd3_dart_0 { + status = "disabled"; +}; + +&usb_drd3_dart_1 { + status = "disabled"; +}; + +&nub_spmi_a2 { + status = "disabled"; +}; + +/* Left-front port (ATC1): the SN201202x hpm1 and the connector graph */ +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec1_hs: endpoint { + remote-endpoint = <&usb_drd1_hs>; + }; + }; + + port@1 { + reg = <1>; + typec1_ss: endpoint { + remote-endpoint = <&atc1_phy_typec>; + }; + }; + }; + }; + }; +}; + +&usb_drd1 { + port { + usb_drd1_hs: endpoint { + remote-endpoint = <&typec1_hs>; + }; + }; +}; + +&atc1_phy { + port { + atc1_phy_typec: endpoint { + remote-endpoint = <&typec1_ss>; + }; + }; +}; + +/* + * Right port (ATC2): hpm2 on nub-spmi-a1, USB only (dcpext0's DisplayPort route + * through ATC2 stays off). The inventory's nub-spmi-a1 has the generation 4 bus's + * three register windows and IRQ (ADT nub-spmi-a1); the driver polls it and + * refuses an interrupt, so it takes the first window only, as on J716s. + */ +&nub_spmi_a1 { + reg = <0x2 0x8a1a8000 0x0 0x4000>; + /delete-property/ interrupts; + status = "okay"; + + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec2_hs: endpoint { + remote-endpoint = <&usb_drd2_hs>; + }; + }; + + port@1 { + reg = <1>; + typec2_ss: endpoint { + remote-endpoint = <&atc2_phy_typec>; + }; + }; + }; + }; + }; +}; + +&ps_atc2_common { + status = "okay"; +}; + +&ps_atc2_usb_aon { + status = "okay"; +}; + +&ps_atc2_phymxwrap { + status = "okay"; +}; + +&ps_atc2_usb { + status = "okay"; +}; + +&atc2_phy_mbox { + status = "okay"; +}; + +&atc2_phy { + status = "okay"; + + port { + atc2_phy_typec: endpoint { + remote-endpoint = <&typec2_ss>; + }; + }; +}; + +&usb_drd2 { + status = "okay"; + + port { + usb_drd2_hs: endpoint { + remote-endpoint = <&typec2_hs>; + }; + }; +}; + +&usb_drd2_dart_0 { + status = "okay"; +}; + +&usb_drd2_dart_1 { + status = "okay"; +}; + /* * The PMP (ADT pmp0, /chosen, /arm-io/pmgr). iBoot loads its firmware and the driver starts it * with these boot arguments and the firmware properties below. Its report tells it which devices From 1d6c883a599731be393b5d26042812df55df7c29 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:24 -0500 Subject: [PATCH 387/392] arm64: dts: apple: t6050-j714s: Enable the SEP's mailbox and power chain The SEP (ADT /arm-io/sep, iop-sep,ascwrap-v6, and dart-sep) runs from iBoot and gates its own domain. Enable its mailbox and its power state, and leave the two fabric parents (fab8-soc, fab0-piogw) to the firmware as well, so genpd votes them on and never turns them off. The boot loader enables the SEP node itself through the sep alias. The M5 Pro's default SSD boot has run this from its DT overlay since 2026-11-09, with apple_sep loaded read-only: SEP platform profile T6050/J714s, xART "writes disabled". Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..e7d5d62da555e 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -98,6 +98,29 @@ power-domains = <&pmp0_gfx>; }; +/* + * The SEP (ADT sep, dart-sep): its mailbox, and its power domain with the two + * fabric parents left to the firmware, which genpd votes on and never turns off. + * The boot loader enables the SEP itself through its alias. + */ +&sep_mbox { + status = "okay"; +}; + +&ps_sep { + status = "okay"; +}; + +&ps_fab8_soc { + apple,firmware-owned; + status = "okay"; +}; + +&ps_fab0_piogw { + apple,firmware-owned; + status = "okay"; +}; + #include "t6050-j714s-pmp-tunables.dtsi" /* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ From ac9dcf8c281099f46b15d7da8e58bdc1e1cdff9f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:36:24 -0500 Subject: [PATCH 388/392] arm64: dts: apple: t6050-j714s: Leave the audio I2C buses off Audio is not brought up on J714s. ADT i2c1 holds the left sn012776 amplifiers (audio-speaker*) and i2c2 the cs42l84 codec (audio-codec-output), so turn both buses off, as t6050-j716s.dts does. The M5 Pro's default SSD boot has run with both off since 2026-11-09. Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..94c54819c53c5 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -102,3 +102,15 @@ /* The same T6050 camera as J716c (J714sAP ADT); the loader sets the J714s sensor type. */ #include "t6050-j716c-isp.dtsi" + +/* + * Audio is not brought up: ADT i2c1 has the left sn012776 amplifiers and i2c2 the + * cs42l84 codec, so both buses stay off. + */ +&i2c1 { + status = "disabled"; +}; + +&i2c2 { + status = "disabled"; +}; From e57b81c4c84c891aa0526e1bd159e3fad1db55df Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 02:37:37 -0500 Subject: [PATCH 389/392] arm64: dts: apple: t6050-j714s: Enable the DockChannel console The J714s console is the DockChannel UART on the USB debug link (ADT /arm-io/dockchannel-uart: aapl,dock-channels, regs from 0x288e28000 and 0x288e0c000, IRQ 360), the die 0 node t6050-die0.dtsi already describes. Enable it as serial0 and the stdout path, with apple,tx-never-wait: the link drains only while a host reads it, and the machine also boots with nothing attached there. As on t6050-j716s.dts. The M5 Pro's default SSD boot has run this node from its DT overlay since 2026-11-09 (root login on ttyDC0, earlycon on the same channel). Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t6050-j714s.dts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6050-j714s.dts b/arch/arm64/boot/dts/apple/t6050-j714s.dts index 6fc1b66157762..68135877b9282 100644 --- a/arch/arm64/boot/dts/apple/t6050-j714s.dts +++ b/arch/arm64/boot/dts/apple/t6050-j714s.dts @@ -15,6 +15,26 @@ #include "t6050-max.dtsi" +/* The console on the USB debug link: ADT dockchannel-uart, channel 1 */ +/ { + aliases { + serial0 = &dockchannel; + }; + + chosen { + stdout-path = "serial0:1500000n8"; + }; +}; + +/* + * It drains only while a host reads it; the machine also boots with nothing + * attached there. + */ +&dockchannel { + apple,tx-never-wait; + status = "okay"; +}; + / { compatible = "apple,j714s", "apple,t6050", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M5 Pro, 2026)"; From 8017a47c92940c138ecffaab537c1e25fdbac26e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 03:49:58 -0500 Subject: [PATCH 390/392] tools/aurora-sep: update diagnostics source harnesses The passive diagnostics state now carries timing and endpoint discovery fields. Supply their types in the policy extraction and their kernel primitives in the full-source sensor harness, so both still compile the real diagnostics code. Signed-off-by: Diljit Singh --- tools/aurora-sep/tests/test_diag_policy.py | 8 ++- tools/aurora-sep/tests/test_diag_sensor.py | 61 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/tests/test_diag_policy.py b/tools/aurora-sep/tests/test_diag_policy.py index be4b93f55c2bf..27ace56de9056 100644 --- a/tools/aurora-sep/tests/test_diag_policy.py +++ b/tools/aurora-sep/tests/test_diag_policy.py @@ -36,8 +36,13 @@ def test_selected_policy_and_identity_source_only(self): prefix = '''#include #include #include +#include #include +typedef uint64_t u64; +typedef uint32_t u32; typedef int atomic_t; +typedef int64_t atomic64_t; +#define DECLARE_BITMAP(name, bits) unsigned long name[((bits) + 63) / 64] struct device { int unused; }; struct device_attribute { int unused; }; #define atomic_read(p) (*(p)) @@ -47,6 +52,7 @@ def test_selected_policy_and_identity_source_only(self): va_list args; va_start(args, fmt); int n = vsnprintf(out, 4096, fmt, args); va_end(args); return n; } ''' + endpoint_limit = re.search(r'^#define SEP_DIAG_MAX_EP\s+\d+', text, re.M).group(0) main = '''int main(void) { char out[4096]; sep_diag.xart = true; sep_diag.xart_guard = 1; sep_diag_set_policy(false, false, false, true, false, false); @@ -60,7 +66,7 @@ def test_selected_policy_and_identity_source_only(self): ''' with tempfile.TemporaryDirectory() as tmp: source, program = Path(tmp) / 'policy.c', Path(tmp) / 'policy' - source.write_text(prefix + state + body + main) + source.write_text(prefix + endpoint_limit + '\n' + state + body + main) subprocess.run([compiler, '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', str(source), '-o', str(program)], check=True) lines = subprocess.check_output([str(program)], text=True).splitlines() diff --git a/tools/aurora-sep/tests/test_diag_sensor.py b/tools/aurora-sep/tests/test_diag_sensor.py index 16623f1f3e358..1f0098d8ccf17 100644 --- a/tools/aurora-sep/tests/test_diag_sensor.py +++ b/tools/aurora-sep/tests/test_diag_sensor.py @@ -35,6 +35,8 @@ typedef uint64_t __u64; typedef uint32_t __u32; typedef uint16_t __u16; typedef uint8_t __u8; typedef int64_t __s64; typedef int32_t __s32; #define __user +#define READ_ONCE(x) __atomic_load_n(&(x), __ATOMIC_RELAXED) +#define WRITE_ONCE(x, v) __atomic_store_n(&(x), (v), __ATOMIC_RELAXED) ''', 'linux/atomic.h': r''' #include @@ -53,10 +55,50 @@ __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } #define atomic_inc(v) __atomic_add_fetch(&(v)->counter, 1, __ATOMIC_SEQ_CST) +#define atomic_inc_return(v) atomic_inc(v) +#define atomic_xchg(v, i) __atomic_exchange_n(&(v)->counter, (i), __ATOMIC_SEQ_CST) +typedef struct { int64_t counter; } atomic64_t; +#define atomic64_read(v) __atomic_load_n(&(v)->counter, __ATOMIC_SEQ_CST) +static inline int64_t atomic64_cmpxchg(atomic64_t *v, int64_t old, int64_t new_) +{ + __atomic_compare_exchange_n(&v->counter, &old, new_, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + return old; +} +''', + 'linux/bitmap.h': r''' +#include +#define BITS_PER_LONG (8 * sizeof(unsigned long)) +#define DECLARE_BITMAP(name, bits) unsigned long name[((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG] +static inline void set_bit(unsigned int bit, unsigned long *map) +{ + map[bit / BITS_PER_LONG] |= 1UL << (bit % BITS_PER_LONG); +} +#define for_each_set_bit(bit, map, size) \ + for ((bit) = 0; (bit) < (size); (bit)++) \ + if ((map)[(bit) / BITS_PER_LONG] & (1UL << ((bit) % BITS_PER_LONG))) +''', + 'linux/bitops.h': r''' +#define smp_rmb() __atomic_thread_fence(__ATOMIC_ACQUIRE) +#define smp_wmb() __atomic_thread_fence(__ATOMIC_RELEASE) +''', + 'linux/ctype.h': '#include \n', + 'linux/ktime.h': r''' +#include +#include +#define NSEC_PER_MSEC 1000000ULL +static inline u64 ktime_get_boottime_ns(void) +{ + struct timespec now; + clock_gettime(CLOCK_BOOTTIME, &now); + return (u64)now.tv_sec * 1000000000ULL + now.tv_nsec; +} +static inline u64 div_u64(u64 value, u32 divisor) { return value / divisor; } ''', 'linux/device.h': r''' #include struct device { int unused; }; +#define dev_info(dev, ...) ((void)(dev)) struct attribute { const char *name; umode_t mode; }; struct device_attribute { struct attribute attr; @@ -88,8 +130,25 @@ va_end(args); return n; } +static inline int sysfs_emit_at(char *buf, int at, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + int n = vsnprintf(buf + at, 4096 - at, fmt, args); + va_end(args); + return n; +} +static inline int scnprintf(char *buf, size_t size, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + int n = vsnprintf(buf, size, fmt, args); + va_end(args); + return n < 0 ? 0 : ((size_t)n < size ? n : (size ? (int)size - 1 : 0)); +} ''', - 'linux/minmax.h': '#define min(a, b) ((a) < (b) ? (a) : (b))\n', + 'linux/minmax.h': '#include \n#define min(a, b) ((a) < (b) ? (a) : (b))\n' + '#define min_t(type, a, b) min((type)(a), (type)(b))\n', 'linux/string.h': '#include \n', } From 837ce59414876e36fa5befee95a3097dd00c0451 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 04:04:21 -0500 Subject: [PATCH 391/392] arm64: smp: include the crash handoff stub without kexec arm64_nmi_cpu_stop calls arm64_crash_handoff_wait even when kexec is disabled. linux/kexec.h only includes the architecture header with CONFIG_KEXEC_CORE, so the SEP KUnit configuration fails with an undeclared function. Include asm/kexec.h directly to expose its existing no-op stub in that configuration. Signed-off-by: Diljit Singh --- arch/arm64/kernel/smp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 78d9b1857e147..f6b5decff53da 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include From 5bb2876f2ce3def3c0af8fc5a6394ee9362f0e70 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 10 Nov 2026 04:06:39 -0500 Subject: [PATCH 392/392] drm/asahi: adapt the 25G83 profile to optional graph tables The M5 stack makes the performance and region C tables optional because its firmware graph supplies them. Preserve the newer integration 25G83 profile tables as Some when combining the branches. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/hw/t6041_25g83.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs index 370561d906bf2..e23b419364a78 100644 --- a/drivers/gpu/drm/asahi/hw/t6041_25g83.rs +++ b/drivers/gpu/drm/asahi/hw/t6041_25g83.rs @@ -44,8 +44,8 @@ use crate::g17::{Config, CoreLayout}; pub(super) const CONFIG: Config = Config { windows: ®ISTER_WINDOWS, reserved_slots: &[], - perf: &PERF, - region_c: ®ION_C, + perf: Some(&PERF), + region_c: Some(®ION_C), chip: Some(&CHIP), firmware: fw::variant::Firmware::Identified(&[t6041::BUILD_25G83]), core_layout: CoreLayout {