From c406b40d5b37d27d720edbca6e0faab8fc1f4bc5 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Sat, 24 Oct 2026 09:26:42 -0300 Subject: [PATCH 001/171] drm/asahi: pgtable: check the struct page of a table outside the reserved regions With reserved table regions (no-map systems such as the T6040), a page-table page outside them went through Page::borrow_phys_unchecked(), which is undefined behaviour when the page is other no-map memory without a struct page. Use the checked borrow there and refuse such a page with -EIO; systems without reserved regions keep the unchecked borrow of an allocated page-table page. Document the single cache invalidate of the reserved regions at boot. It runs after the host has already written table entries, which is safe because the T6040 cleans every host table store to the PoC (UatQuirks::clean_tables), so the invalidate drops only clean lines. Not invalidating again assumes the firmware stops writing the tables the host walks once it has booted; say that this is an assumption. The wrapper's comment claimed the invalidate came before the first host edit, which it doesn't. Link: https://github.com/aurora-silicon/linux/pull/408 Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit bec833e0bc581f74744265f0613832e8291db0cc in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu.rs | 2 +- drivers/gpu/drm/asahi/pgtable.rs | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ed032ab146dd06..79dc1d50755d8b 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3557,7 +3557,7 @@ impl Uat { } /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). pub(crate) fn invalidate_reserved_tables(&self) { if let Some(reserved) = self.inner.reserved.as_ref() { reserved.invalidate(); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index c5a47c24ec9286..7244f12a43d398 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -183,6 +183,15 @@ impl ReservedTables { /// Drops the CPU's cached lines of every region, so that later reads see what the firmware /// wrote while booting. Lines the host dirtied are written back first, never discarded. + /// + /// This runs once, after the firmware booted. The host has written table entries by then + /// (fixed kernel objects are mapped before the boot), but the T6040, the only SoC with + /// reserved tables, also sets `UatQuirks::clean_tables`: every host table store is cleaned + /// to the PoC as it is made, so this drops only clean lines and loses no host write. + /// + /// The regions are not invalidated again, which assumes the firmware no longer writes the + /// tables the host walks once it has booted. That is not proven by a trace; the J616s has + /// run this way since its GPU came up without a fault traced to a stale table. pub(crate) fn invalidate(&self) { for map in self.regions.iter() { clean_invalidate_poc(map.ptr(), map.size()); @@ -192,7 +201,9 @@ impl ReservedTables { } /// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the -/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// reserved mapping when the page lies in a reserved region, else through `struct page`. With +/// reserved regions, a page outside them that is not System RAM with a `struct page` (other +/// no-map memory) is refused with `EIO`. /// /// # Safety /// @@ -211,8 +222,14 @@ unsafe fn with_table( // SAFETY: lookup() checked that the whole page is mapped. return f(unsafe { base.add(offset) }); } - // SAFETY: The caller guarantees an allocated page-table page in RAM. - let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + let page = if reserved.is_some() { + // SAFETY: The caller guarantees the page stays allocated; borrow_phys() checks that it + // is System RAM with a struct page. + unsafe { Page::borrow_phys(&phys) }.ok_or(EIO)? + } else { + // SAFETY: The caller guarantees an allocated page-table page in RAM. + unsafe { Page::borrow_phys_unchecked(&phys) } + }; page.with_pointer_into_page(offset, len, f) } From bc9e3aa6a8c580553111988865b3b82eb2d71552 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 23 Oct 2026 23:20:01 -0300 Subject: [PATCH 002/171] drm/asahi: identify the T6040 GPU firmware by its loaded text The T6040 selected its firmware build only by apple,firmware-version, the macOS release the boot loader reports. That says which build to expect, not that it is the build loaded: a loader can write the version for a boot firmware it merely recognises, and the init-data hash covers only the driver's own constants. The G16 backend already proves its build from the loaded TEXT: the SHA-256 of the image with the entropy bytes of its GKTS and ECAP records zeroed, plus the segment sizes. Move that check out of g16 into a shared fwid module with its SHA-256 helper, built for every asahi build. A build entry names the offsets of its two entropy records, which are checked and zeroed there, as G16 did; an unknown image is reported with the records found by their tags, so its digest is stable across boots and a new variant is one more entry. G16 uses it as before. A G17 firmware build now carries its image identity, and the T6040 refuses the GPU unless the fw-text region holds the 25G83 image it was written for (measured on a J616s, macOS 26.6.2), with the DATA size of the fw-data region. A tree without fw-text no longer gets the GPU; the J616s tree ships it disabled until a boot loader fills it in. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: integration/aurora does not carry the G16 backend: the g16/ changes are left out, and CRYPTO_LIB_SHA256 and fwid.o go to DRM_ASAHI and integration/aurora's asahi-y] (cherry picked from commit a335c77de52a274029bb4e42f96d6afde5712a2f in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/Kconfig | 1 + drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/fwid.c | 13 +++ drivers/gpu/drm/asahi/fwid.rs | 149 ++++++++++++++++++++++++ drivers/gpu/drm/asahi/g15/mod.rs | 1 - drivers/gpu/drm/asahi/g17/fw/variant.rs | 9 +- drivers/gpu/drm/asahi/g17/mod.rs | 19 +++ drivers/gpu/drm/asahi/hw/t6040.rs | 11 ++ 9 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/asahi/fwid.c create mode 100644 drivers/gpu/drm/asahi/fwid.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 0e989189206bad..8d2c50f4475330 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -21,6 +21,7 @@ config DRM_ASAHI select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM select RUST_APPLE_RTKIT + select CRYPTO_LIB_SHA256 help DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 037ffb71a9d4cf..33d575639d81c4 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o +asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index b6ddde4e3bcace..1d14ac3692a460 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -12,6 +12,7 @@ mod driver; mod event; mod file; mod float; +mod fwid; mod fw; mod gem; mod g17; diff --git a/drivers/gpu/drm/asahi/fwid.c b/drivers/gpu/drm/asahi/fwid.c new file mode 100644 index 00000000000000..5779380ff716ab --- /dev/null +++ b/drivers/gpu/drm/asahi/fwid.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// +// SHA-256 for the firmware identity check (fwid.rs). + +#include +#include + +void asahi_fw_sha256(const u8 *data, size_t len, u8 *out); + +void asahi_fw_sha256(const u8 *data, size_t len, u8 *out) +{ + sha256(data, len, out); +} diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs new file mode 100644 index 00000000000000..bcc5626d7f0bb8 --- /dev/null +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact GPU firmware identity. +//! +//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its two entropy records +//! (`GKTS` and `ECAP`, each a tag, the length 8 and the bytes) zeroed, plus the TEXT and DATA +//! sizes and the records' offsets. An unknown build is refused with its +//! measured identity, so nothing is written to a firmware the driver was not written for. + +use core::ops::Range; +use kernel::{bindings, device, fmt, prelude::*}; + +/// A qualified GPU firmware build, identified by its loaded TEXT. +pub(crate) struct FirmwareBuild { + pub(crate) name: &'static str, + pub(crate) text_size: u64, + pub(crate) data_size: u64, + /// Offsets of the GKTS and ECAP entropy records in TEXT; their 8 entropy bytes are zeroed + /// before hashing. + pub(crate) entropy: [u32; 2], + pub(crate) text_sha256: [u8; 32], +} + +extern "C" { + fn asahi_fw_sha256(data: *const u8, len: usize, out: *mut u8); +} + +/// SHA-256 (`sha256()`). +pub(crate) fn sha256(data: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + // SAFETY: `data` is valid for its length and `out` holds the 32-byte digest. + unsafe { asahi_fw_sha256(data.as_ptr(), data.len(), out.as_mut_ptr()) }; + out +} + +/// Lowercase hex of a byte string, as `%*phN` prints it. +pub(crate) struct Hex<'a>(pub(crate) &'a [u8]); + +impl fmt::Display for Hex<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for b in self.0 { + write!(f, "{:02x}", b)?; + } + Ok(()) + } +} + +/// Copies `size` bytes at physical `phys`, a range outside the kernel's System RAM, into a +/// private copy through a temporary write-combined mapping. +pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { + // SAFETY: memremap() of a boot-loader range creates a temporary mapping; it has no other + // side effects. + let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + if map.is_null() { + return Err(ENOMEM); + } + let copy = KVVec::from_elem(0u8, size, GFP_KERNEL).map(|mut copy| { + // SAFETY: `map` is a live mapping of `size` bytes and `copy` holds `size` bytes; the two do + // not overlap. + unsafe { core::ptr::copy_nonoverlapping(map as *const u8, copy.as_mut_ptr(), size) }; + copy + }); + // SAFETY: `map` came from memremap() above and is not used afterwards. + unsafe { bindings::memunmap(map) }; + Ok(copy?) +} + +/// Entropy record tags, in TEXT order. +const ENTROPY_TAGS: [[u8; 4]; 2] = [*b"GKTS", *b"ECAP"]; + +/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` zeroed, or `None` if a +/// record (tag, then the length 8) is not there. `image` is left as it was. +fn digest_at(image: &mut [u8], at: [u32; 2]) -> Option<[u8; 32]> { + let mut saved = [[0u8; 8]; 2]; + for ((off, tag), save) in at.into_iter().zip(ENTROPY_TAGS).zip(saved.iter_mut()) { + let off = off as usize; + if image.get(off..off + 4) != Some(tag.as_slice()) + || image.get(off + 4..off + 8) != Some(8u32.to_le_bytes().as_slice()) + { + return None; + } + save.copy_from_slice(image.get(off + 8..off + 16)?); + } + for off in at { + image[off as usize + 8..off as usize + 16].fill(0); + } + let digest = sha256(image); + for (off, save) in at.into_iter().zip(saved) { + image[off as usize + 8..off as usize + 16].copy_from_slice(&save); + } + Some(digest) +} + +/// The first record of each tag in `image`, to report an unknown build. +fn find_entropy(image: &[u8]) -> Option<[u32; 2]> { + let mut found = [0u32; 2]; + for (slot, tag) in found.iter_mut().zip(ENTROPY_TAGS) { + let mut record = [0u8; 8]; + record[..4].copy_from_slice(&tag); + record[4..].copy_from_slice(&8u32.to_le_bytes()); + *slot = u32::try_from(image.windows(8).position(|w| w == record)?).ok()?; + } + Some(found) +} + +/// Identifies the firmware build loaded at `text` (DATA `data_size` bytes) among `builds`, +/// before any firmware-visible write. +pub(crate) fn identify( + dev: &device::Device, + text: Range, + data_size: u64, + builds: &'static [FirmwareBuild], +) -> Result<&'static FirmwareBuild> { + let size = usize::try_from(text.end - text.start)?; + let mut copy = copy_physical(text.start, size)?; + let again = copy_physical(text.start, size)?; + if copy.as_slice() != again.as_slice() { + dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); + return Err(EAGAIN); + } + for build in builds { + if build.text_size == size as u64 + && build.data_size == data_size + && digest_at(&mut copy, build.entropy) == Some(build.text_sha256) + { + dev_info!(dev, "GPU firmware build: {}\n", build.name); + return Ok(build); + } + } + let Some((entropy, digest)) = + find_entropy(©).and_then(|at| Some((at, digest_at(&mut copy, at)?))) + else { + dev_err!(dev, "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", size); + return Err(EINVAL); + }; + dev_err!( + dev, + "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} entropy {:#x}/{:#x} sha256 {}\n", + size, + data_size, + entropy[0], + entropy[1], + Hex(&digest) + ); + for build in builds { + dev_err!(dev, "supported: {}\n", build.name); + } + Err(EINVAL) +} diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 010e5f0e5af4fc..de1dfa898eaae2 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -59,7 +59,6 @@ pub(crate) mod fw; mod gate; mod graph; pub(crate) mod hw; -mod identity; mod initdata; mod job; mod object; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 2f7b5987333688..59f1bb4714fc0f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,12 +64,15 @@ pub(crate) struct Variant { pub(crate) programs: super::render::RegisterPrograms, } -/// A firmware build a SoC supports, identified by the macOS release it ships with. +/// A firmware build a SoC supports, selected by the macOS release it ships with and proven by +/// its loaded TEXT. pub(crate) struct Build { /// macOS version (major, minor, patch). pub(crate) os_version: [u32; 3], /// Name, for the log. pub(crate) name: &'static str, + /// The loaded image, measured from the `fw-text` region. + pub(crate) image: crate::fwid::FirmwareBuild, /// Layouts and rules of the build. pub(crate) variant: &'static Variant, } @@ -79,8 +82,8 @@ pub(crate) enum Firmware { /// The SoC runs a single build, which is not identified (A18 Pro). Fixed(&'static Variant), /// One of these builds, selected by the macOS version that the boot loader reports in - /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and - /// with it this firmware build. Any other version is refused. + /// `apple,firmware-version` (the release whose boot firmware loaded the GPU firmware) and + /// then proven by the loaded TEXT. Any other version or image is refused. Identified(&'static [Build]), } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 31c242a1323f8d..740c1f7d9f5d4c 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1239,6 +1239,7 @@ impl Gpu { let version = Self::os_firmware_version(dev); if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + Self::identify_image(dev, build).map_err(|_| ENODEV)?; return Ok(build.variant); } dev_err!( @@ -1252,6 +1253,24 @@ impl Gpu { Err(ENODEV) } + /// Checks that the loaded TEXT (the `fw-text` region) is `build`'s image, with the DATA size + /// of the `fw-data` region. The version property only says which build to expect. + fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; + let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let start = text.start(); + let end = start.checked_add(text.size()).ok_or(EINVAL)?; + crate::fwid::identify( + dev.as_ref().as_ref(), + start..end, + data.size(), + core::slice::from_ref(&build.image), + ) + .inspect_err(|e| dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e))?; + Ok(()) + } + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 7a4d6f288dc3da..8df2cff4f65414 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -62,6 +62,17 @@ pub(crate) const CONFIG: Config = Config { firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { os_version: [26, 6, 2], name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + image: crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, + 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, 0x10, 0xe0, + 0x63, 0x87, 0x6d, 0xcd, + ], + }, variant: &fw::variant::V25G83, }]), // Measured on the 20-core M4 Pro; other core counts are refused at probe. From c3666fc36bd4dcf66cda9f8b2d66af792e291725 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 25 Oct 2026 11:06:22 +1100 Subject: [PATCH 003/171] drm/asahi: test shared firmware identity normalization Cover the shared identity code moved out of G16 with synthetic TEXT. Entropy changes must preserve the normalized digest, non-entropy changes must change it, malformed records must be refused, and measurement must leave its input unchanged. No firmware image is included in the tests. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit da42c7cf4fc83e68261f1c238b2a3b870f94a128 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/fwid.rs | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index bcc5626d7f0bb8..36a6fa45581be5 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -147,3 +147,42 @@ pub(crate) fn identify( } Err(EINVAL) } + +#[kunit_tests(asahi_fwid)] +mod tests { + use super::*; + + fn image() -> [u8; 64] { + let mut image = [0x5a; 64]; + for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { + image[off..off + 4].copy_from_slice(&tag); + image[off + 4..off + 8].copy_from_slice(&8u32.to_le_bytes()); + } + image + } + + #[test] + fn entropy_is_normalized_without_changing_the_image() { + let mut first = image(); + let original = first; + let digest = digest_at(&mut first, [0, 32]).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[8..16].fill(0xff); + second[40..48].fill(0x01); + assert_eq!(digest_at(&mut second, [0, 32]), Some(digest)); + second[63] ^= 1; + assert_ne!(digest_at(&mut second, [0, 32]), Some(digest)); + } + + #[test] + fn malformed_records_are_refused_without_changing_the_image() { + let mut image = image(); + assert_eq!(digest_at(&mut image, [0, 56]), None); + image[36] = 7; + let original = image; + assert_eq!(digest_at(&mut image, [0, 32]), None); + assert_eq!(image, original); + assert_eq!(find_entropy(&image), None); + } +} From 65a4c71118fb807ad7dcabd90bb417595df9aea1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 06:37:44 +1100 Subject: [PATCH 004/171] drm/asahi: admit the alternate T6040 loaded 25G83 image Keep a list of admitted images on each versioned firmware Build and pass that list to fwid::identify(). Version selection uses the first matching Build, so another Build for macOS 26.6.2 would leave its image unreachable. Add the second normalized T6040 identity measured over two J616s boots. The boots differ only in their entropy payloads. Compared with the qualified 25G83 image, the remaining differences are the field at 0x23d and the iBoot-filled patchbay/tunables area. The executable code outside those areas and DATA are unchanged. This follows the J713 precedent in #442, while keeping whole-TEXT hashing and exact identity checks. Factor the existing lookup so KUnit can exercise the real T6040 and J713 lists with digest measurements, without embedding firmware. Cover both T6040 identities, both J713 identities, unknown digests and size/offset mismatches. Keep the entropy-normalization tests. Link: https://github.com/aurora-silicon/linux/pull/442 Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 26f5eb68dbce039de470b789d754d2ec3541c15e in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/fwid.rs | 98 ++++++++++++++++++++++--- drivers/gpu/drm/asahi/g17/fw/variant.rs | 4 +- drivers/gpu/drm/asahi/g17/mod.rs | 4 +- drivers/gpu/drm/asahi/hw/t6040.rs | 47 ++++++++---- 4 files changed, 127 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 36a6fa45581be5..bb1fd3c36f6233 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -103,6 +103,20 @@ fn find_entropy(image: &[u8]) -> Option<[u32; 2]> { Some(found) } +/// Finds an exact identity, measuring each candidate at its own entropy offsets. +fn lookup( + builds: &'static [FirmwareBuild], + text_size: u64, + data_size: u64, + mut digest: impl FnMut([u32; 2]) -> Option<[u8; 32]>, +) -> Option<&'static FirmwareBuild> { + builds.iter().find(|build| { + build.text_size == text_size + && build.data_size == data_size + && digest(build.entropy) == Some(build.text_sha256) + }) +} + /// Identifies the firmware build loaded at `text` (DATA `data_size` bytes) among `builds`, /// before any firmware-visible write. pub(crate) fn identify( @@ -118,19 +132,20 @@ pub(crate) fn identify( dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); return Err(EAGAIN); } - for build in builds { - if build.text_size == size as u64 - && build.data_size == data_size - && digest_at(&mut copy, build.entropy) == Some(build.text_sha256) - { - dev_info!(dev, "GPU firmware build: {}\n", build.name); - return Ok(build); - } + if let Some(build) = lookup(builds, size as u64, data_size, |at| { + digest_at(&mut copy, at) + }) { + dev_info!(dev, "GPU firmware build: {}\n", build.name); + return Ok(build); } let Some((entropy, digest)) = find_entropy(©).and_then(|at| Some((at, digest_at(&mut copy, at)?))) else { - dev_err!(dev, "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", size); + dev_err!( + dev, + "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", + size + ); return Err(EINVAL); }; dev_err!( @@ -185,4 +200,69 @@ mod tests { assert_eq!(image, original); assert_eq!(find_entropy(&image), None); } + + fn measured( + builds: &'static [FirmwareBuild], + data_size: u64, + entropy: [u32; 2], + digest: [u8; 32], + ) -> Option<&'static FirmwareBuild> { + lookup(builds, 0x60000, data_size, |at| { + (at == entropy).then_some(digest) + }) + } + + #[test] + fn t6040_accepts_both_loaded_images() { + let crate::g17::fw::variant::Firmware::Identified(builds) = + crate::hw::t6040::CONFIG.firmware + else { + panic!("T6040 must identify its loaded firmware"); + }; + let build = builds.iter().find(|b| b.os_version == [26, 6, 2]).unwrap(); + let digests = [ + [ + 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, + 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, 0x10, 0xe0, + 0x63, 0x87, 0x6d, 0xcd, + ], + [ + 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, 0x96, + 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, 0xc9, 0xa5, + 0x2a, 0xcc, 0xd9, 0xac, + ], + ]; + for (i, digest) in digests.into_iter().enumerate() { + let found = measured(build.images, 0x138000, [0x59839, 0x5991c], digest).unwrap(); + assert!(core::ptr::eq(found, &build.images[i])); + assert!(measured(build.images, 0x13c000, [0x59839, 0x5991c], digest).is_none()); + assert!(measured(build.images, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(lookup(build.images, 0x60001, 0x138000, |_| Some(digest)).is_none()); + } + assert!(measured(build.images, 0x138000, [0x59839, 0x5991c], [0; 32]).is_none()); + } + + #[cfg(CONFIG_DRM_ASAHI_G16)] + #[test] + fn j713_accepts_both_loaded_images() { + let builds = crate::g16::hw::t8132::CONFIG.firmware; + let digests = [ + [ + 0x5f, 0x65, 0xf6, 0x14, 0x58, 0x72, 0x54, 0x70, 0xb4, 0xb0, 0xc0, 0x20, 0x28, 0x47, + 0x7b, 0xf2, 0xea, 0x57, 0x3a, 0xfb, 0x4e, 0x0c, 0x49, 0x6e, 0xf9, 0x95, 0xe9, 0x6a, + 0xff, 0xb0, 0xd8, 0x6c, + ], + [ + 0x3b, 0x36, 0x8e, 0xe4, 0x2d, 0xcc, 0xc6, 0x26, 0xe8, 0x49, 0x67, 0xff, 0x40, 0x87, + 0x64, 0x71, 0x0a, 0x64, 0xbf, 0xf1, 0x59, 0x4f, 0x3c, 0xfd, 0x2c, 0x6e, 0x4d, 0x34, + 0xd0, 0x65, 0x24, 0x8b, + ], + ]; + for (i, digest) in digests.into_iter().enumerate() { + let found = measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], digest).unwrap(); + assert!(core::ptr::eq(found, &builds[i])); + assert!(measured(builds, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + } + assert!(measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], [0; 32]).is_none()); + } } diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 59f1bb4714fc0f..d2d6d2404dfc8e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -71,8 +71,8 @@ pub(crate) struct Build { pub(crate) os_version: [u32; 3], /// Name, for the log. pub(crate) name: &'static str, - /// The loaded image, measured from the `fw-text` region. - pub(crate) image: crate::fwid::FirmwareBuild, + /// The admitted loaded images, measured from the `fw-text` region. + pub(crate) images: &'static [crate::fwid::FirmwareBuild], /// Layouts and rules of the build. pub(crate) variant: &'static Variant, } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 740c1f7d9f5d4c..7c476e1400a48a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1253,7 +1253,7 @@ impl Gpu { Err(ENODEV) } - /// Checks that the loaded TEXT (the `fw-text` region) is `build`'s image, with the DATA size + /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size /// of the `fw-data` region. The version property only says which build to expect. fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; @@ -1265,7 +1265,7 @@ impl Gpu { dev.as_ref().as_ref(), start..end, data.size(), - core::slice::from_ref(&build.image), + build.images, ) .inspect_err(|e| dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e))?; Ok(()) diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 8df2cff4f65414..0007fdf132949c 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -62,17 +62,30 @@ pub(crate) const CONFIG: Config = Config { firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { os_version: [26, 6, 2], name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", - image: crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, - 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, 0x10, 0xe0, - 0x63, 0x87, 0x6d, 0xcd, - ], - }, + images: &[ + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, + 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, + 0x10, 0xe0, 0x63, 0x87, 0x6d, 0xcd, + ], + }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, + 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, + 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, + ], + }, + ], variant: &fw::variant::V25G83, }]), // Measured on the 20-core M4 Pro; other core counts are refused at probe. @@ -122,7 +135,11 @@ const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> /// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), - instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + instances( + window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), + 4, + 0x200_0000, + ), window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), @@ -131,7 +148,11 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), - instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + instances( + window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), From 8926476b66bd2a5677dc4d7c8aeb20ae59e1f80a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 11:48:37 +1100 Subject: [PATCH 005/171] drm/asahi: use the loaded image to select identified firmware Treat the OS version as a candidate-order hint. Continue to require an exact admitted loaded TEXT image and preserve #453's images list. This extends Jonathan Goyvaerts' shared identity work. Originally-by: Jonathan Goyvaerts Link: https://github.com/aurora-silicon/linux/pull/428 Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 804bba86ca55cb0598e90c99b3dafc3e48b40c78 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/variant.rs | 10 ++++------ drivers/gpu/drm/asahi/g17/mod.rs | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index d2d6d2404dfc8e..78951f6d10a7e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,10 +64,9 @@ pub(crate) struct Variant { pub(crate) programs: super::render::RegisterPrograms, } -/// A firmware build a SoC supports, selected by the macOS release it ships with and proven by -/// its loaded TEXT. +/// A firmware build a SoC supports, identified by its loaded TEXT. pub(crate) struct Build { - /// macOS version (major, minor, patch). + /// macOS version (major, minor, patch), used to order image candidates. pub(crate) os_version: [u32; 3], /// Name, for the log. pub(crate) name: &'static str, @@ -81,9 +80,8 @@ pub(crate) struct Build { pub(crate) enum Firmware { /// The SoC runs a single build, which is not identified (A18 Pro). Fixed(&'static Variant), - /// One of these builds, selected by the macOS version that the boot loader reports in - /// `apple,firmware-version` (the release whose boot firmware loaded the GPU firmware) and - /// then proven by the loaded TEXT. Any other version or image is refused. + /// One of these exact loaded images. The loader's `apple,firmware-version` is only + /// a candidate-ordering hint because multiple OS releases can share an iBoot string. Identified(&'static [Build]), } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 7c476e1400a48a..f29dd51006fa2e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1237,24 +1237,26 @@ impl Gpu { fw::variant::Firmware::Identified(builds) => builds, }; let version = Self::os_firmware_version(dev); - if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - Self::identify_image(dev, build).map_err(|_| ENODEV)?; - return Ok(build.variant); + // iBoot strings can be shared by OS releases. Prefer the reported release's + // candidate, but admit only an exact loaded image, regardless of that hint. + let preferred = builds.iter().filter(|build| Some(build.os_version) == version); + let others = builds.iter().filter(|build| Some(build.os_version) != version); + for build in Iterator::chain(preferred, others) { + if Self::identify_image(dev, build).is_ok() { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } } dev_err!( dev.as_ref(), - "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + "Unsupported GPU firmware image (reported macOS version {:?})\n", version ); - for build in builds.iter() { - dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); - } Err(ENODEV) } /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size - /// of the `fw-data` region. The version property only says which build to expect. + /// of the `fw-data` region. The version property is only a selection hint. fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; From 801d2533c20e84254196007347c71dafdd2ea9e5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 11:48:39 +1100 Subject: [PATCH 006/171] drm/asahi: build T6040 initdata from validated per-chip calibration Wire the reader into probe and firmware construction, validating OPPs and enabled cores before firmware starts. Use the maps, leakage and power ceiling in the 25G83 objects, replacing bin-specific output hashes with layout and host encoder checks. Keep the Neo path unchanged. The old ceiling declaration remains only for the hardware-default reference removed by the next patch. Firmware uses the calibration. Retain GPL-2.0-only and the Gravity Linux encoder attribution to Cody Ho and Niklas Sheth. The original T6040 work is by Jonathan Goyvaerts. Originally-by: Jonathan Goyvaerts Link: https://github.com/aurora-silicon/linux/pull/428 Link: https://github.com/GravityLinux/linux Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 53877eaebd4b9d4d41ef9e1ce56c00d038192cff in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/float.rs | 5 + .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 111 ++++-------------- drivers/gpu/drm/asahi/g17/initdata.rs | 6 +- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 44 +++---- drivers/gpu/drm/asahi/g17/mod.rs | 12 ++ 5 files changed, 66 insertions(+), 112 deletions(-) diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index a740ad3bb945df..998ddb9ed9b2d9 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,6 +44,11 @@ struct F32U { } impl F32 { + /// Return the IEEE-754 binary32 representation. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index c237896a49fb1a..db554b0d9b8fc1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: GPL-2.0-only //! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). //! @@ -17,9 +17,9 @@ //! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. use super::{ - fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, - Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, - REGISTER_MAP_SLOTS, + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, + RegisterMapEntry, Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, + COMPLETION_RING_INDEX, PERF_STATES, REGISTER_MAP_SLOTS, }; use kernel::prelude::*; @@ -379,28 +379,15 @@ impl HwData25G83 { } } +/// Transitional hardware default, removed with the inert T6040 defaults. +pub(crate) const MAX_POWER_MW: u32 = 69_116; + /// Power configuration control period, in 24 MHz clocks (8 ms). const PERIOD_CLOCKS: u32 = 200_000; -/// Highest GPU power, in mW. -pub(crate) const MAX_POWER_MW: u32 = 69_116; -/// [`MAX_POWER_MW`] as an IEEE-754 single. -const MAX_POWER_MW_F32: u32 = 0x4786_fe00; /// Temperature sensors read by the die and MTR temperature limiters. const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; -/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays -/// of two dies. Not found in the ADT; constant for the SoC. -const LEAKAGE: [u32; 56] = { - let mut words = [0; 56]; - words[0] = 0x44f0_3000; // 1921.5 - words[1] = 0x44f3_0000; // 1944.0 - words[32] = 0x4373_0000; // 243.0 - words[33] = 0x4373_0000; // 243.0 - words[52] = 0x437b_0000; // 251.0 - words[54] = 0x43bd_c000; // 379.5 - words -}; -/// Offset of [`LEAKAGE`] in the power configuration. +/// Offset of the per-chip leakage coefficients in the power configuration. const LEAKAGE_OFFSET: usize = 0x642c; /// Offset in the power configuration of the GPU address of the configuration's status (the /// configuration named by the root, `+0x80`). @@ -433,12 +420,10 @@ const POWER_CONFIG_WORDS: Words = &[ (0x9c4, 0x3f7f2e9f, 1), (0x9cc, 0x3b516154, 1), (0x9d4, 0x3ca59586, 1), - (0x9e0, MAX_POWER_MW_F32, 1), (0x9e4, 0x40a90fdb, 1), (0x9ec, 0xbc6d0cfe, 1), (0x9f0, 0x64, 1), (0x9f4, 0x3e8, 2), - (0xa04, MAX_POWER_MW, 1), (0xa38, 0x3e8, 1), (0xa70, 0x8, 1), (0xa7c, 0x3f000000, 1), @@ -448,7 +433,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0xa9c, 0x3ecf2e49, 1), (0xaa8, 0x64, 1), (0xaac, 0x3e8, 1), - (0xabc, MAX_POWER_MW, 1), (0xac8, 0x10, 1), (0xad0, 0x5dc00, 1), (0xb28, 0x50, 1), @@ -508,7 +492,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x27b8, 0xc80000, 1), (0x2870, 0x3e8, 1), (0x28c0, 0x1, 1), - (0x28c8, MAX_POWER_MW, 3), (0x28e0, 0x3f800000, 1), (0x28e4, 0x4, 1), (0x28e8, 0x8, 1), @@ -521,8 +504,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x29e8, 0x3f4f2e49, 1), (0x29f4, 0x28, 1), (0x29f8, 0x3e8, 2), - (0x2a04, MAX_POWER_MW_F32, 1), - (0x2a08, MAX_POWER_MW, 1), (0x2a14, 0xf8, 1), (0x2a1c, 0x5ad200, 1), (0x2a3c, 0x3e8, 1), @@ -591,27 +572,34 @@ unsafe impl Zeroable for PowerConfig25G83 {} static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); impl PowerConfig25G83 { - /// Pointer and random words, which [`Self::hash`] leaves out. - const RUNTIME: [(usize, usize); 3] = - [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; - /// Initializes the power configuration. - pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + pub(crate) fn init( + &mut self, + config_va: u64, + aon_va: u64, + entropy: u32, + max_power_mw: u32, + leakage: &[u32; 56], + ) { + self.0.fill(0); fill_words(&mut self.0, POWER_CONFIG_WORDS); - for (index, word) in LEAKAGE.iter().enumerate() { + for (index, word) in leakage.iter().enumerate() { let at = LEAKAGE_OFFSET + 4 * index; self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); } + // All copies of the power ceiling must describe the same chip. + for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { + self.0[offset..offset + 4].copy_from_slice(&max_power_mw.to_le_bytes()); + } + let power = crate::float::F32::from(max_power_mw).to_bits(); + for offset in [0x9e0, 0x2a04] { + self.0[offset..offset + 4].copy_from_slice(&power.to_le_bytes()); + } self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] .copy_from_slice(&(config_va + 0x80).to_le_bytes()); self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); } - - /// Hash of the contents without the pointers and the random word. - pub(crate) fn hash(&self) -> u64 { - hash_without(&self.0, &Self::RUNTIME) - } } /// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU @@ -632,50 +620,3 @@ pub(crate) const AON_TABLE: [u32; 9] = [ /// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero /// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; - -impl HwData25G83 { - /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) - /// and what comes from the device: the performance tables built from its operating points, - /// which are binned per device, and its enabled core count. - const UNHASHED: [(usize, usize); 10] = [ - (COMPLETION_DESCRIPTORS, 0x80), - (QOS_POINTER, 8), - (core::mem::offset_of!(Self, unk_f88), size_of::()), - (core::mem::offset_of!(Self, unk_fc0), size_of::()), - (core::mem::offset_of!(Self, perf_a), size_of::()), - (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), - (core::mem::offset_of!(Self, cs_perf), size_of::()), - (core::mem::offset_of!(Self, afr_perf), size_of::()), - (core::mem::offset_of!(Self, perf_b), size_of::()), - (core::mem::offset_of!(Self, unk_2570), size_of::()), - ]; - - /// Hash of the constant contents (everything but [`Self::UNHASHED`]). - pub(crate) fn hash(&self) -> u64 { - // SAFETY: `HwData25G83` consists of integers only and has no padding. - let bytes = unsafe { - core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) - }; - hash_without(bytes, &Self::UNHASHED) - } -} - -/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. -pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { - let mut hash = 0xcbf2_9ce4_8422_2325u64; - for (offset, byte) in bytes.iter().enumerate() { - if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { - continue; - } - hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); - } - hash -} - -/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the -/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. -pub(crate) mod expected { - pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; - pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; - pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; -} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 7e0c36ece43ad5..3b224e330cdcb3 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1328,7 +1328,11 @@ impl InitData { /// The GPU's operating points: `count` of them, each with one voltage cell for the whole /// GPU or one per cluster. - fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + pub(super) fn opps( + dev: &AsahiDrmDevice, + count: usize, + clusters: u32, + ) -> Result> { let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; if opps.len() != count { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index 85aa6468fa311f..670f4b96dcb76a 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -8,7 +8,7 @@ //! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; -use crate::g17::{Config, Role}; +use crate::g17::{calibration::Calibration, Config, Role}; use crate::mmu; use abi::v25g83 as v; use core::sync::atomic::{fence, Ordering}; @@ -74,7 +74,7 @@ impl InitData { }) } - /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// Writes the 25G83 firmware objects using the chip calibration, then /// stages the opening control-ring state. fn initialize_25g83( &mut self, @@ -101,12 +101,14 @@ impl InitData { let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let calibration = Calibration::read(env.dev)?; + calibration.check_opps(&opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: &calibration.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -117,41 +119,28 @@ impl InitData { }; // SAFETY: Plain C entropy source. let entropy = unsafe { kernel::bindings::get_random_u32() }; - let mut hashes = [0; 3]; self.object_mut(self.bundle)? .initialize::(0, |hw| { hw.init(&hw_args, &hw_args_25g83)?; - hashes[0] = hw.hash(); Ok(()) })?; self.object_mut(self.bundle)? .initialize::(v::POWER_CONFIG, |power| { - power.init(config_va, aon_va, entropy); - hashes[1] = power.hash(); + power.init( + config_va, + aon_va, + entropy, + calibration.max_power_mw, + &calibration.leakage, + ); Ok(()) })?; self.object_mut(self.region_c)? .initialize::(0, |region| { region.init(cfg.region_c); + region.max_power_mw = calibration.max_power_mw; Ok(()) })?; - hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); - // The constant contents must equal what the Gravity Linux encoders produce on the proven - // setup; the operating points and the core count come from the device and are not hashed. - let expected = [ - v::expected::HW_DATA, - v::expected::POWER_CONFIG, - v::expected::REGION_C, - ]; - if hashes != expected { - dev_err!( - env.dev.as_ref(), - "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", - hashes, - expected - ); - return Err(EINVAL); - } for (index, word) in v::AON_TABLE.iter().enumerate() { self.object_mut(objects.aon)?.write(index * 4, *word)?; } @@ -167,7 +156,9 @@ impl InitData { v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); let cluster = self.object(self.cluster)?; cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); - cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + cluster + .dword(fwctl + 8)? + .store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); // One main configuration carrying both roles. let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; @@ -218,7 +209,8 @@ impl InitData { )?; for &(offset, value, repeat) in v::POWER_STATE_WORDS { for index in 0..repeat { - self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + self.object_mut(objects.power_state)? + .write(offset + 4 * index, value)?; } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f29dd51006fa2e..892e1bcd340f1b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -23,6 +23,7 @@ mod availability; mod buffer; +mod calibration; mod channel; mod command; pub(crate) use command::Attachments; @@ -1123,8 +1124,19 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { + Some(calibration::Calibration::read(dev).inspect_err(|_| { + dev_err!(dev.as_ref(), "Missing or invalid T6040 GPU calibration handoff\n"); + })?) + } else { + None + }; let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; + if let Some(calibration) = calibration { + calibration.check_cores(&id.core_masks)?; + calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; + } if id.family != cfg.identity.family || id.variant != cfg.identity.variant || id.num_dies != cfg.identity.num_dies From f6f90b208e1c0e7071ce8605bca6f2b788b8d7b9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 11:48:40 +1100 Subject: [PATCH 007/171] drm/asahi: remove single-unit T6040 power defaults Keep the structural hardware description while requiring the loader maps and power ceiling. Remove the last single-unit ceiling reference and its declaration together, keeping this commit buildable independently. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 415fd2e6ee2d5175dd4bf4148c391f7daf575328 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 3 --- drivers/gpu/drm/asahi/hw/t6040.rs | 24 ++++++------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index db554b0d9b8fc1..f4c3ee5db22e29 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -379,9 +379,6 @@ impl HwData25G83 { } } -/// Transitional hardware default, removed with the inert T6040 defaults. -pub(crate) const MAX_POWER_MW: u32 = 69_116; - /// Power configuration control period, in 24 MHz clocks (8 ms). const PERIOD_CLOCKS: u32 = 200_000; /// Temperature sensors read by the die and MTR temperature limiters. diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 0007fdf132949c..fa3e51efacadcb 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -7,7 +7,7 @@ use crate::g17::fw::{ self, - initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, + initdata::{PerfConfig, RegionCConfig, RegisterWindow}, }; use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; @@ -169,33 +169,23 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), ]; -/// Performance-table configuration. The device tree lists the sixteen operating points in -/// performance-state order, with one voltage per cluster. +/// Inert defaults. The 25G83 builder requires the loader's per-chip performance maps. pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], - states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], - relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], - relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + states_a: [0; 11], + states_b: [0; 11], + relative_a: [0; 11], + relative_b: [0; 11], sram_min_mv: 810, }; /// Operating points the performance tables reference. pub(crate) const OPPS: usize = 16; -const _: () = { - let mut index = 0; - while index < PERF_STATES { - assert!((PERF.states_a[index] as usize) < OPPS); - assert!((PERF.states_b[index] as usize) < OPPS); - index += 1; - } -}; - /// Region C. const REGION_C: RegionCConfig = RegionCConfig { unk_54: 0x0fa0, unk_78: [1, 0, 0], - max_power_mw: v25g83::MAX_POWER_MW, + max_power_mw: 0, unk_9c: 0, unk_a0: 0, unk_7c4: 1500, From 310861355992f105756d21ca4afbfb40cae9ad64 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 22:22:05 +1100 Subject: [PATCH 008/171] drm/asahi: cover the T6040 firmware queue floor in QID tests Run both dynamic compute growth and the full logical queue budget checks with the 25G83 floor of 32, alongside the existing floors. This checks render headroom and firmware-owned QID exclusion on the M4 Pro layout. Bring the host resource runner forward to the shared reserve_for helper and current render reserve, retaining its capacity and stale-ID controls. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of tools/asahi/g17-resource-tests.py] (cherry picked from commit 61aca8fb8212c8479a9e4830164ac2acf17c24e4 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/kick.rs | 4 +- tools/asahi/g17-resource-tests.py | 449 ++++++++++++++++++++++++++++++ 2 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/g17-resource-tests.py diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b55c283254b5e..6499cae106309b 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -314,7 +314,7 @@ mod accounting_tests { #[test] fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { - for floor in [0, 4] { + for floor in [0, 4, 32] { let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); let queues = 2; let mut owner = 1; @@ -333,7 +333,7 @@ mod accounting_tests { #[test] fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { - for floor in [0, 4] { + for floor in [0, 4, 32] { let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); let queues = queue_limit(floor) as usize; for owner in 1..=COMPUTE_FLOOR as u64 { diff --git a/tools/asahi/g17-resource-tests.py b/tools/asahi/g17-resource-tests.py new file mode 100644 index 00000000000000..cf93704966c33a --- /dev/null +++ b/tools/asahi/g17-resource-tests.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual G17 admission, queue-ID, and unused-graph eligibility methods. + +Run with Python and rustc; temporary host binaries are removed after the checks. +""" +from pathlib import Path +import os +import re +import subprocess +import tempfile + +DRIVER = Path(__file__).resolve().parents[2] / 'drivers/gpu/drm/asahi' + +def admission_tests(root, source): + s=(source/'g17/queue/admission.rs').read_text();cfg=(source/'hw/t8140.rs').read_text() + def item(start): + at=s.index(start);begin=s.index('{',at);depth=1;end=begin+1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + return s[at:end] + constants='\n'.join(re.search(r'pub\(crate\) const '+name+r': [^;]+;',cfg).group(0) for name in ['RENDER_SLOTS','RENDER_PAIRS_PER_OWNER','RENDER_DEPTH']) + mmu=(source/'mmu.rs').read_text() + limits='\n'.join(re.search(r'const '+name+r': [^;]+;',mmu).group(0) for name in ['EXECUTION_CONTEXT_MASK','MAX_EXECUTION_CONTEXTS']) + header='''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + mod g17 { pub mod capacity { + pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } + pub fn refused(_: Site) -> i32 { -28 } + }} + + const ENOSPC:i32=-28; const EIO:i32=-5; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + type KVec = Vec; struct InFlight; + mod cfg {\n'''+constants+'}\n'+limits+'\n' + methods=['const fn new','fn exhausted','fn refusal','fn advance','fn take_idle','fn acquire','fn claim_sibling','fn reassign_unused_spare','fn warmable_sibling','fn borrow_warm_sibling','fn cancel_empty_slot','fn release'] + code=header+item('struct State')+'\nimpl State {\n'+re.search(r' const MASK: u64 = .*;',s).group(0)+'\n'+'\n'.join(item(n) for n in methods)+'\n}\n'+''' + fn main(){ + if cfg::RENDER_PAIRS_PER_OWNER == 1 { + let mut state=State::new(); + let mut slots=Vec::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=state.acquire(owner).expect("every logical owner must fit");slots.push(slot); + assert_eq!(state.owners.iter().filter(|id|**id==owner).count(),1); + } + for slot in slots {state.release(slot);} + let mut state=State::new();let first=state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH {assert_eq!(state.acquire(1),Some(first));} + assert_eq!(state.acquire(1),None);state.release(first);assert_eq!(state.acquire(1),Some(first)); + println!("shared render admission: all58 owners fit; depth16 waits for exact retirement without consuming a spare"); + return; + } + let mut state=State::new(); + for owner in 1..=32 { + assert!(state.refusal(owner).is_ok()); + let slot=state.acquire(owner).expect("existing owner fits"); + state.release(slot); + } + assert_eq!(state.active,0,"all submitted work retired"); + let retained=state.owners.iter().filter(|o|**o!=0).count(); + assert_eq!(retained,63); + assert_eq!(state.refusal(33),Err(ENOSPC)); + assert!(state.acquire(33).is_none()); + println!("confirmed: 32 open idle owners retain all 63 usable pairs; owner33 receives ENOSPC"); + // Yield one unused spare for each additional logical owner. The runtime + // separately proves that these physical graphs have never been published. + for owner in 33..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot=(1..cfg::RENDER_SLOTS).find(|slot|state.reassign_unused_spare(*slot as u8,None,Some(owner))).unwrap(); + assert_eq!(state.acquire(owner),Some(slot as u8));state.release(slot as u8); + } + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) {let slot=state.acquire(owner).unwrap();state.release(slot);} + assert!(state.owners.iter().all(|owner|*owner<=u64::from(MAX_EXECUTION_CONTEXTS))); + println!("fixed: all{} logical owners render and resume without reducing slot count or depth",MAX_EXECUTION_CONTEXTS); + let mut guarded=State::new();let first=guarded.acquire(1).unwrap();guarded.release(first); + let spare=guarded.owners.iter().enumerate().find(|(s,o)|**o==1&&*s!=first as usize).unwrap().0; + guarded.take_idle(spare,1); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(1),Some(2)),"warm borrower excludes reclamation"); + guarded.release(spare as u8); + assert!(!guarded.reassign_unused_spare(spare as u8,Some(3),Some(2)),"owner mismatch"); + guarded.quarantined |= 1u64< = std::result::Result; + mod g17 { pub mod capacity { + pub enum Site { RenderSlots, LogicalQueues, ComputeReserve, PhysicalQids } + pub fn refused(_: Site) -> i32 { -28 } + }} + + mod mmu { pub const MAX_EXECUTION_CONTEXTS:u32=58; } + const ENOSPC:i32=-28; const EINVAL:i32=-22; const EOVERFLOW:i32=-75; + macro_rules! resource_exhausted { ($s:literal)=>{ENOSPC}; } + #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum DataMaster {Compute,Tiling,Fragment} + #[derive(Clone,Copy,PartialEq,Eq,Debug)] struct Id {qid:u8,owner:u64,engine:DataMaster} + #[derive(Clone,Copy)] struct Identity {id:Id,published:bool} + '''+constants+'\n'+item(s,'const fn render_reserve(')+'\n'+item(s,'pub(crate) struct QueueIds')+'\nimpl QueueIds {\n'+''.join(item(s,'pub(crate) fn '+name+'(')+'\n' for name in ['reserve_initial_render','reserve','reserve_for','publish','can_cancel_unpublished','cancel_unpublished'])+''.join(item(s, sig)+'\n' for sig in ['fn render_ids(', 'fn compute_room('])+'}\n' + test=''' + fn fresh()->QueueIds {QueueIds {entries:[None;QID_COUNT],initial_render:[None;2],leased:0,floor:0}} + fn main(){ + for floor in [0,32] { + for render in [0,2,16,32,80,96,112,126].into_iter().filter(|render|*render<=QID_COUNT-floor) { + let mut q=fresh();q.floor=floor; + for i in 0..render {q.reserve(i as u64+1,if i%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();} + let mut compute=0; + while q.reserve(1000+compute,DataMaster::Compute,None).is_ok(){compute+=1;} + assert_eq!(compute as usize,QID_COUNT-floor-render.max(FULL_RENDER_RESERVE-floor)); + assert!(q.entries[..floor].iter().all(Option::is_none)); + } + } + let mut q=fresh();let first=q.reserve(1,DataMaster::Tiling,None).unwrap(); + assert!(q.can_cancel_unpublished(first));q.publish(first).unwrap(); + assert!(!q.can_cancel_unpublished(first));assert_eq!(q.cancel_unpublished(first),Err(EINVAL));assert_eq!(q.leased,1); + let unused=q.reserve(2,DataMaster::Fragment,None).unwrap();q.cancel_unpublished(unused).unwrap();assert_eq!(q.leased,1); + let next=q.reserve(3,DataMaster::Fragment,None).unwrap();assert_eq!(next.qid,unused.qid); + assert!(!q.can_cancel_unpublished(unused));assert_eq!(q.cancel_unpublished(unused),Err(EINVAL)); + println!("queue IDs: render/compute capacity boundaries at zero and reserved floors; compute-only render reserve retained; published and stale identities protected"); + // Reproduce the hardware snapshot: 124 installed render identities, two + // installed compute identities, and two initial reservations fill128 IDs. + let mut q=fresh();q.reserve_initial_render().unwrap(); + assert_eq!(q.reserve_initial_render(),Err(EINVAL)); + let initial=q.initial_render; + for owner in 1000..1002 {let id=q.reserve(owner,DataMaster::Compute,Some(4)).unwrap();q.publish(id).unwrap();} + let mut installed=Vec::new(); + for owner in 3..127 {let id=q.reserve(owner,if owner%2==0 {DataMaster::Tiling}else{DataMaster::Fragment},None).unwrap();q.publish(id).unwrap();installed.push(id);} + assert_eq!(q.leased,QID_COUNT); + for (n,old) in initial.into_iter().enumerate() { + let old=old.unwrap();let id=q.reserve(2000+n as u64,DataMaster::Compute,None).unwrap(); + assert_eq!(id.qid,old.qid);assert_eq!(q.leased,QID_COUNT); + assert!(!q.can_cancel_unpublished(old));assert_eq!(q.cancel_unpublished(old),Err(EINVAL)); + q.publish(id).unwrap(); + } + assert_eq!(q.reserve(3000,DataMaster::Compute,None),Err(ENOSPC)); + for id in installed {assert!(!q.can_cancel_unpublished(id));assert_eq!(q.entries[id.qid as usize].unwrap().id,id);} + // A bootstrap token that became published is never replaced. + let mut q=fresh();q.reserve_initial_render().unwrap();let protected=q.initial_render[0].unwrap();q.publish(protected).unwrap(); + for owner in 3..129 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + assert_eq!(q.leased,QID_COUNT); + assert_eq!(q.reserve(1000,DataMaster::Compute,None).unwrap().qid,1); + assert_eq!(q.reserve(1001,DataMaster::Compute,None),Err(ENOSPC)); + assert_eq!(q.entries[0].unwrap().id,protected); + // Replacement must not consume the protected render minimum. + let mut q=fresh();q.reserve_initial_render().unwrap(); + for owner in 3..=FULL_RENDER_RESERVE as u64 {q.reserve(owner,DataMaster::Tiling,None).unwrap();} + for owner in 1000..1000+(QID_COUNT-FULL_RENDER_RESERVE) as u64 {q.reserve(owner,DataMaster::Compute,None).unwrap();} + assert_eq!(q.leased,QID_COUNT);assert_eq!(q.reserve(2000,DataMaster::Compute,None),Err(ENOSPC)); + assert!(q.initial_render.iter().all(Option::is_some)); + // Ordinary free capacity leaves the initialization reservations untouched. + let mut q=fresh();q.reserve_initial_render().unwrap();let initial=q.initial_render; + let id=q.reserve(1000,DataMaster::Compute,Some(4)).unwrap();assert_eq!(id.qid,4);assert_eq!(q.initial_render,initial); + println!("initial reservations: full-namespace compute admission restored; physical count unchanged; published and stale identities plus render minimum protected"); + + } + ''' + fixture=root/'queue-id-capacity-controls.rs';fixture.write_text(header+test) + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'queue-id-capacity-controls')],check=True) + subprocess.run([str(root/'queue-id-capacity-controls')],check=True) + s=(source/'queue/render/parked.rs').read_text();guard=item(s,'pub(crate) fn can_yield_unused') + fixture=root/'unused-render-guard-controls.rs';fixture.write_text('''#![allow(dead_code, unused_macros)] + type Result = std::result::Result; + struct Context; + struct Pair {owner:Option,ordinal:u64,registered:Option,outer_started:u8,previous:Option<()>,graph_started:bool,backing:Result} + impl Pair { + fn context(&self)->&Context {&Context} + fn can_park_backing(&self,_:&Context)->Result{self.backing} + '''+guard+''' + } + fn fresh()->Pair {Pair {owner:Some(1),ordinal:0,registered:None,outer_started:0,previous:None,graph_started:false,backing:Ok(true)}} + fn main(){ + assert_eq!(fresh().can_yield_unused(),Ok(true)); + let mut p=fresh();p.owner=None;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.ordinal=1;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.registered=Some(2);assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.outer_started=4;assert_eq!(p.can_yield_unused(),Ok(true)); + let mut p=fresh();p.graph_started=true;assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Ok(false);assert_eq!(p.can_yield_unused(),Ok(false)); + let mut p=fresh();p.backing=Err(-5);assert_eq!(p.can_yield_unused(),Err(-5)); + println!("unused graph: eight eligibility/error controls protect every publication boundary"); + } + ''') + subprocess.run([os.environ.get('RUSTC', 'rustc'),'--edition=2021',str(fixture),'-o',str(root/'unused-render-guard-controls')],check=True) + subprocess.run([str(root/'unused-render-guard-controls')],check=True) + +def lower_window_gate_tests(root, source): + s=(source/'g17/freelist.rs').read_text() + start=s.index('static LOWER_FREED');end=s.index('pub(crate) struct Pages {') + gate=s[start:end] + fixture=root/'lower-window-gate-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::sync::atomic::{AtomicU64, Ordering}; +type Result = std::result::Result; +const ENOSPC:i32=-28;const ENOMEM:i32=-12; +static CLOCK: AtomicU64 = AtomicU64::new(1_000_000_000); +fn now_ns()->u64 {CLOCK.load(Ordering::Relaxed)} +mod freelist { +use super::*; +'''+gate+''' +pub(crate) fn freed() {LOWER_FREED.fetch_add(1, Ordering::AcqRel);} +pub(crate) fn place(size: usize, f: impl FnOnce() -> Result) -> Result {place_lower(size, f)} +} +use freelist::*; +fn attempt(size:usize, result:Result, calls:&mut u32)->Result {place(size,||{*calls+=1;result})} +fn main(){ + let mut calls=0; + assert_eq!(attempt(100,Ok(1),&mut calls),Ok(1));assert_eq!(calls,1);assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,2);assert!(lower_pressure()); + for _ in 0..1000 {assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));} + assert_eq!(attempt(200,Ok(1),&mut calls),Err(ENOSPC)); + assert_eq!(calls,2,"a refused size is not placed again while nothing was freed"); + assert_eq!(attempt(50,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"a smaller size still probes"); + assert_eq!(attempt(60,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,3,"the smaller refusal now gates larger sizes"); + freed();assert!(!lower_pressure()); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,4,"freed backing lets a waiter retry"); + // One probe per interval across all callers, then the gate closes again. + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Err(ENOSPC),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + assert_eq!(attempt(100,Ok(1),&mut calls),Err(ENOSPC));assert_eq!(calls,5); + CLOCK.fetch_add(100_000_000,Ordering::Relaxed); + assert_eq!(attempt(100,Ok(7),&mut calls),Ok(7));assert_eq!(calls,6);assert!(!lower_pressure(),"a fitting placement clears pressure"); + assert_eq!(attempt(100,Err(ENOMEM),&mut calls),Err(ENOMEM));assert!(!lower_pressure(),"only window refusals gate"); + println!("lower window: refused free-list placements are not repeated until backing is freed or one probe per 100 ms"); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'lower-window-gate-controls')],check=True) + subprocess.run([str(root/'lower-window-gate-controls')],check=True) + +def watchdog_publication_guard_tests(source): + # A packet still in publication (capacity or graph waits) must never reach + # pair lookups, the timeout storm count or a collector allocation from its + # watchdog: each of them can stop the runtime for every client. + text = (source / 'g17/runtime/backend.rs').read_text() + at = text.index(' fn timed_out(') + body = text[at:text.index('\n }\n', at)] + guard = body.index('if !packet.is_published() {\n return sched::Status::NoHang;') + assert guard < body.index('self.release_backlog(&packet);') < body.index('self.shared.watchdog_timeout()'), 'timed_out' + for path, name in [('g17/runtime/render/settle.rs', 'pub(in crate::g17) fn timeout_render('), + ('g17/runtime/compute_control.rs', 'pub(super) fn timeout_compute(')]: + text = (source / path).read_text() + at = text.index(name) + body = text[at:text.index('\n }\n', at)] + assert 'let Ok(mut deferred) = DeferredBatch::worker() else {' in body, name + assert 'DeferredBatch::worker()?' not in body, name + print('watchdog: unpublished packets and collector allocation failure never stop the runtime') + +def shared_alias_tests(root, source): + s=(source/'g17/object.rs').read_text() + at=s.index(' pub(crate) fn lower_with_aliases(');begin=s.index('{',at);end=begin+1;depth=1 + while depth: + depth+=(s[end]=='{')-(s[end]=='}');end+=1 + method=s[at:end] + fixture=root/'shared-alias-controls.rs' + fixture.write_text('''#![allow(dead_code)] +use std::{cell::RefCell,rc::Rc,ops::Range}; +type Result = std::result::Result;type Error=i32; +const EINVAL:i32=-22;const EOVERFLOW:i32=-75;const ENOSPC:i32=-28;const EIO:i32=-5; +mod g17 {pub mod capacity {pub enum Site {LowerVa, AliasVa} pub fn refused(_: Site)->i32 {-28}}} +mod hw {pub mod t8140 {pub mod dynamic {pub const LOWER:std::ops::Range=0..256;}}} +mod mmu {pub const UAT_PGSZ:usize=16;pub type Prot=u8;pub type Vm=super::Vm;pub type KernelMapping=super::Mapping;} +#[derive(Clone,Copy)] enum CpuMap {WriteCombined} +#[derive(Clone)] struct Region {vm:usize,start:u64,size:u64,object:u64} +#[derive(Default)] struct State {regions:Vec,allocations:u64,attempts:usize,fault:Option} +struct Vm {id:usize,state:Rc>} +struct Uat {lower:Vm} +impl Uat {fn kernel_lower_vm(&self)->&Vm {&self.lower}} +struct Device {state:Rc>} +struct Mapping {vm:usize,address:u64,size:u64,object:u64,state:Rc>} +impl Mapping {fn iova(&self)->u64 {self.address}} +impl Drop for Mapping {fn drop(&mut self) { + let mut state=self.state.borrow_mut(); + if self.vm==0 {assert!(!state.regions.iter().any(|r|r.vm!=0&&r.object==self.object&&r.start==self.address),"aliases must retire before canonical reservation");} + let index=state.regions.iter().position(|r|r.vm==self.vm&&r.start==self.address&&r.object==self.object).unwrap(); + state.regions.remove(index); +}} +fn overlaps(s:&State,vm:usize,start:u64,size:u64)->bool {s.regions.iter().any(|r|r.vm==vm&&r.startResult { + let mut state=vm.state.borrow_mut(); + if state.fault==Some(vm.id) {return Err(EIO)} + if overlaps(&state,vm.id,address,size) {return Err(ENOSPC)} + state.regions.push(Region {vm:vm.id,start:address,size,object}); + Ok(Mapping {vm:vm.id,address,size,object,state:vm.state.clone()}) +} +struct ObjectRef {id:u64,size:usize} +impl ObjectRef { + fn map_mode(&self,vm:&Vm,range:Range,align:u64,guard:bool,lowest:bool)->Result { + if vm.id==0 {assert!(guard,"canonical guard must be preserved");vm.state.borrow_mut().attempts+=1;} + let length=self.size as u64+if guard {16}else{0}; + let mut used:Vec<_>=vm.state.borrow().regions.iter().filter(|r|r.vm==vm.id).cloned().collect();used.sort_by_key(|r|r.start); + let mut cursor=range.start;let mut holes=Vec::new(); + for r in used { + if r.start>cursor {holes.push(cursor..r.start.min(range.end));} + cursor=cursor.max(r.start+r.size); + } + if cursorclaim(vm,address,length,self.id),None=>Err(ENOSPC)} + } + fn map_into_range(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:bool)->Result { + self.map_mode(vm,range,align,guard,false) + } +} +struct KernelObject {obj:ObjectRef,mapping:Option} +impl KernelObject { + fn backing(dev:&Device,size:usize,_cpu:CpuMap)->Result {let mut s=dev.state.borrow_mut();s.allocations+=1;Ok(Self {obj:ObjectRef {id:s.allocations,size},mapping:None})} + fn map_alias(&self,vm:&Vm,address:u64,_prot:u8)->Result {claim(vm,address,self.obj.size as u64,self.obj.id)} + fn alias_in_lowest(&self,vm:&Vm,range:Range,align:u64,_prot:u8,guard:usize)->Result {self.obj.map_mode(vm,range,align,guard!=0,true)} +} +struct Allocator<'a> {dev:&'a Device,uat:&'a Uat} +impl Allocator<'_> { +'''+method+''' +} +fn case(obstacles:&[(usize,u64,u64)],fault:Option,size:usize,expected:std::result::Result,attempts:usize) { + let state=Rc::new(RefCell::new(State::default())); + for (vm,start,len) in obstacles {state.borrow_mut().regions.push(Region {vm:*vm,start:*start,size:*len,object:0});} + state.borrow_mut().fault=fault; + let dev=Device {state:state.clone()};let uat=Uat {lower:Vm {id:0,state:state.clone()}}; + let first=Vm {id:1,state:state.clone()};let second=Vm {id:2,state:state.clone()}; + let alloc=Allocator {dev:&dev,uat:&uat}; + let result=alloc.lower_with_aliases(size,16,3,CpuMap::WriteCombined,[&first,&second]); + match (expected,result) { + (Ok(address),Ok((object,aliases)))=>{ + assert_eq!(object.mapping.as_ref().unwrap().iova(),address); + assert!(aliases.iter().all(|m|m.iova()==address&&m.object==object.obj.id)); + assert_eq!(state.borrow().regions.iter().filter(|r|r.object!=0).count(),3); + drop(aliases);drop(object); + }, + (Err(expected),Err(actual))=>assert_eq!(actual,expected), + _=>panic!("unexpected common-address placement result"), + } + assert_eq!(state.borrow().regions.len(),obstacles.len(),"failed attempts leak no reservations or aliases"); + assert_eq!(state.borrow().allocations,if size==0||size%16!=0 {0}else{1},"all retries reuse one backing"); + assert_eq!(state.borrow().attempts,attempts); +} +fn exhaustive_common_holes() { + for canonical in 0u32..64 {for global in 0u32..64 {for client in 0u32..64 { + let masks=[canonical,global,client]; + let available=(0..4).any(|page|canonical&(7<{ + assert!(available,"allocator found a false common hole"); + let address=object.mapping.as_ref().unwrap().iova();assert_eq!(address%16,0);let page=address/16;assert!(page<4); + assert!(canonical&(7<{assert_eq!(error,ENOSPC);assert!(!available,"allocator skipped a common hole");}, + } + assert_eq!(state.borrow().allocations,1);assert_eq!(state.borrow().regions.len(),original); + }}} + println!("shared aliases: all262144 three-VM occupancy combinations match the independent common-hole oracle"); +} +fn main() { + case(&[],None,64,Ok(0),1); + case(&[(1,0,16)],None,64,Ok(16),2); + case(&[(2,0,16)],None,64,Ok(16),2); + case(&[(1,0,16),(2,64,16)],None,64,Ok(80),3); + case(&[(0,0,16)],None,64,Ok(16),1); + case(&[(2,0,256)],None,64,Err(ENOSPC),1); + case(&[(2,0,176)],None,64,Ok(176),2); + case(&[],Some(1),64,Err(EIO),1); + case(&[],Some(2),64,Err(EIO),1); + case(&[],Some(0),64,Err(EIO),1); + case(&[],None,256,Err(ENOSPC),1); + case(&[],None,0,Err(EINVAL),0); + case(&[],None,15,Err(EINVAL),0); + // Best-fit initially selects the small late hole, occupied in the client. + // The larger earlier canonical hole must remain eligible after that collision. + case(&[(0,128,16),(2,144,112)],None,64,Ok(0),2); + case(&[(1,0,256)],None,64,Err(ENOSPC),1); + case(&[(0,128,128),(2,0,128)],None,64,Err(ENOSPC),2); + println!("shared aliases: 16 best-fit, collision, rollback, lifetime, backing-reuse, guard and error controls pass"); + exhaustive_common_holes(); +} +''') + subprocess.run([os.environ.get('RUSTC','rustc'),'--edition=2021',str(fixture),'-o',str(root/'shared-alias-controls')],check=True) + subprocess.run([str(root/'shared-alias-controls')],check=True) + +if __name__ == '__main__': + with tempfile.TemporaryDirectory(prefix='asahi-resource-tests-') as temporary: + admission_tests(Path(temporary), DRIVER) + queue_tests(Path(temporary), DRIVER) + lower_window_gate_tests(Path(temporary), DRIVER) + watchdog_publication_guard_tests(DRIVER) + shared_alias_tests(Path(temporary), DRIVER) From 20971ad9feaf260a70d9cc17f1c66e83be557bff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 14:45:56 +1100 Subject: [PATCH 009/171] drm/asahi: require a compute stamp for record fallback across recovery An exact completion record that has aged for 50 ms permits compute retirement without a stamp in the ordinary generation. The recovery path can consume queue items without completing command writes; those cursors and an old record alone must not release command mappings. Restrict the record-only fallback to the publication's generation. After a restart, retain the command until a covering stamp and consumed items provide the release witness, or the existing quarantine/stop path takes ownership. Keep interrupted fence classification independent. Exercise aged records with missing, stale, exact and later stamps across a restart, consumed and busy rings, and the ordinary fallback control. This closes a retention hazard; it does not establish the cause of the T6040 CTS page faults or qualify recovery on that firmware. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit fcd39d542f865e7a0ae744da44748897dfdee75d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../drm/asahi/g17/queue/compute/retirement.rs | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 8a737546cb7cf9..5a89a1835c7404 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -112,7 +112,9 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery +/// generation. Recovery may consume items without completing their writes, so an old record +/// and consumed items cannot replace the stamp across a restart. In both cases its items /// must be consumed. #[expect( clippy::too_many_arguments, @@ -131,7 +133,9 @@ pub(crate) fn poll( if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let completed = stamp_covers(stamp, ticket.kick) + || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } @@ -542,6 +546,30 @@ mod tests { assert_eq!(result, Ok(None)); } + #[test] + fn recovery_cannot_turn_an_aged_record_into_a_release_witness() { + let record = Some(Observation { timestamps: [10, 20] }); + // Recovery can advance the consumer without completing cache maintenance. + // Even a record observed before the restart and aged past the fallback + // must retain its command's mappings until a real stamp covers it. + for stamp in [0, own(KICK - 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + } + // A real stamp restores the release witness; completion's interrupted + // classification independently prevents false successful fences. + for stamp in [own(KICK), own(KICK + 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(Some([10, 20]))); + } + let busy = Cursors { producer: 4, consumer: 2, count: 64 }; + assert_eq!(poll(&ticket(), record, busy, own(KICK), + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + // The existing same-generation fallback remains available. + assert_eq!(poll(&ticket(), record, consumed(), own(KICK - 1), + Recordless::Allowed, 0, None, true), Ok(Some([10, 20]))); + } + #[test] fn a_zero_duration_record_fails() { let record = Some(Observation { timestamps: [5, 5] }); From 0a142911f224fd24bdca92de8f2f91c59be7c515 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 14:46:45 +1100 Subject: [PATCH 010/171] drm/asahi: retain recovery progress when slot keys are unknown The 25G83 path has no validated slot-key address. It consequently lost progress observations too, even though they come from a separately validated firmware-owned view. Read those observations with an unknown key represented by zero; preserve firmware blame when ownership is unknown or a read fails. Report the bounded halted-firmware snapshot with its generation, reason, slot mask, known-key count and per-slot progress. Identify stampless retirement counters explicitly as lifetime totals so they cannot be mistaken for commands retired by a single recovery. Read no new MMIO addresses and change no firmware timeout, feature or voltage policy. Test unknown ownership, incomplete progress and page-fault fallback. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 90150b09b82066a3104df8bb7ec3a1dfeffc06c3 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 5 +- .../drm/asahi/g17/queue/compute/retirement.rs | 6 +- drivers/gpu/drm/asahi/g17/recovery.rs | 1 + .../gpu/drm/asahi/g17/recovery/attribution.rs | 58 +++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 3b224e330cdcb3..c8763f40871d16 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1563,7 +1563,10 @@ impl super::recovery::Memory for InitData { .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; + // Progress is independently available even when this firmware's key + // address is unknown. Zero keeps ownership unknown and cannot override + // firmware blame, but preserves the progress evidence for diagnostics. + let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 5a89a1835c7404..6b942dcd86a3e9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -94,13 +94,17 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(crate) fn stampless_total() -> u64 { + STAMPLESS.load(core::sync::atomic::Ordering::Relaxed) +} + /// Counts a command that retired on its record because its completion stamp never arrived; /// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: lifetime total {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, STAMP_FALLBACK_NS / 1_000_000, qid, diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 36f1ceac9f821c..7e5b84f6e630ee 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -241,6 +241,7 @@ pub(super) fn service( host.clear_timestamps()?; } let mut sources = host.fault_sources()?; + sources.report(request.generation, super::queue::compute::retirement::stampless_total()); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index 1755484219187e..be1c7ab9eb6621 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -30,6 +30,23 @@ pub(super) enum Attribution { } impl Sources { + /// Bounded evidence from the existing halted-firmware snapshot. No guessed + /// MMIO/key addresses or additional firmware reads are used for diagnostics. + pub(super) fn report(&self, generation: u64, stampless_total: u64) { + kernel::pr_warn!( + "G17: recovery snapshot gen {} reason {:?} slot_mask {:#x} sampled {} known_keys {}; stampless lifetime total {} before restart\n", + generation, self.reason, self.mask, self.count, + self.records[..self.count].iter().filter(|source| source.qid.is_some()).count(), + stampless_total + ); + for source in &self.records[..self.count] { + kernel::pr_warn!( + "G17: recovery slot {} queue {:?} kick {:#x} monitor {:#x}\n", + source.slot, source.qid, source.stamp, source.monitor + ); + } + } + /// Firmware is halted for every selected key and progress-record read. /// Failed reads stay unknown; incomplete evidence cannot override firmware blame. pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> Option<(u64, u32)>) -> Self { @@ -103,3 +120,44 @@ impl Sources { } } } + +#[kernel::macros::kunit_tests(asahi_g17_recovery_sources)] +mod tests { + use super::*; + + #[test] + fn progress_without_keys_preserves_firmware_blame() { + let mut reads = 0; + let mut sources = Sources::sample(0b101, |slot| { + reads += 1; + Some((0, (slot + 1) as u32)) + }); + sources.reason = Some(2); + assert_eq!(reads, 2); + assert_eq!(sources.count, 2); + assert_eq!(sources.records[1].slot, 2); + assert_eq!(sources.records[1].monitor, 3); + assert!(sources.records[..sources.count].iter().all(|source| source.qid.is_none())); + assert_eq!(sources.attribute(Some(38), |_, _| panic!("unknown key")), + Attribution::Firmware); + } + + #[test] + fn incomplete_progress_cannot_clear_firmware_blame() { + let mut sources = Sources::sample(0b11, |slot| { + (slot == 0).then_some(((38 << 40) | 1, 0)) + }); + sources.reason = Some(2); + assert_eq!(sources.attribute(Some(38), |_, _| Some(false)), Attribution::Firmware); + } + + #[test] + fn page_fault_keeps_firmware_blame_with_complete_keys() { + let mut sources = Sources::sample(0b11, |slot| { + Some(((((38 + slot) as u64) << 40) | 1, 0)) + }); + sources.reason = Some(PAGE_FAULT); + assert_eq!(sources.attribute(Some(38), |qid, _| Some(qid == 39)), + Attribution::Firmware); + } +} From 62c7346aa062ff8022050ed8449c630124c8f19b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 14:49:05 +1100 Subject: [PATCH 011/171] drm/asahi: preserve firmware blame when any selected slot lacks identity Readable progress is not complete ownership evidence. If even one selected slot lacks a valid key, do not use another slot's unstarted render to clear firmware blame. Require every sampled selected key to name a queue before attempting host attribution. Cover a known blamed queue with an unknown peer key, in addition to wholly unknown keys and failed progress reads. This also keeps a failed key read conservative on firmware builds with a known key address. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 34665a1107eb74adb99f8724ec5a94370e2fe918 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery/attribution.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index be1c7ab9eb6621..38ad8612a0b99c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -81,7 +81,10 @@ impl Sources { blamed: Option, mut started: impl FnMut(u8, u64) -> Option, ) -> Attribution { - if self.reason.is_none() || self.count as u32 != self.mask.count_ones() { + if self.reason.is_none() + || self.count as u32 != self.mask.count_ones() + || self.records[..self.count].iter().any(|source| source.qid.is_none()) + { return Attribution::Firmware; } for source in &mut self.records[..self.count] { @@ -151,6 +154,16 @@ mod tests { assert_eq!(sources.attribute(Some(38), |_, _| Some(false)), Attribution::Firmware); } + #[test] + fn a_missing_peer_key_cannot_clear_a_known_queues_blame() { + let mut sources = Sources::sample(0b11, |slot| { + Some((if slot == 0 { (38 << 40) | 1 } else { 0 }, 0)) + }); + sources.reason = Some(2); + assert_eq!(sources.count, 2); + assert_eq!(sources.attribute(Some(38), |_, _| Some(false)), Attribution::Firmware); + } + #[test] fn page_fault_keeps_firmware_blame_with_complete_keys() { let mut sources = Sources::sample(0b11, |slot| { From e35e0e4d36eb621fe822802637d84ad87f1d12bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 15:03:51 +1100 Subject: [PATCH 012/171] drm/asahi: initialize the G17 idle hold from the existing DT policy The G17 backend exported an idle-delay attribute but always started at the native two-millisecond policy, ignoring apple,idle-off-delay-ms. Use the existing DT property at probe and validate it against the same 2..1000 ms bounds as the existing attribute. Document that property in the binding. Missing policy preserves the current default; other GPUs need no platform flag or new module parameter. Exercise absent, valid and invalid policy values and depth-one gaps within a bounded sixteen-millisecond hold, followed by real idle release. This supplies a board policy through the existing power-control path; it changes neither firmware register layouts nor voltage/clock tables. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: the binding change keeps integration/aurora's reg-names and adds only apple,idle-off-delay-ms] (cherry picked from commit 6ca4244633eb9e7105685f99469332f8203a4adf in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../devicetree/bindings/gpu/apple,agx.yaml | 10 +++++ drivers/gpu/drm/asahi/g17/mod.rs | 6 +++ drivers/gpu/drm/asahi/g17/power.rs | 44 ++++++++++++++++++- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index e7afc4dd87e8f1..bfed6b4659d8ad 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -64,6 +64,16 @@ properties: - const: hw-cal-b - const: globals + apple,idle-off-delay-ms: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 2 + maximum: 1000 + default: 2 + description: + Continuous idle time before GPU power-off, in milliseconds. The G17 + backend extends the firmware's native delay with its host idle-hold + control; an absent property preserves the native policy. + operating-points-v2: true apple,firmware-abi: diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 892e1bcd340f1b..bb951a85ca9616 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1156,6 +1156,12 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let idle_delay = power::idle_delay( + node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?, + )?; + power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); + dev_info!(dev.as_ref(), "GPU idle power-off delay: {} ms\n", idle_delay); let channels = availability::Availability::new()?; let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index b0e21e70bbe18e..bbd10b204edd75 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -12,10 +12,22 @@ pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// the last job. Not measured on other firmware; at or below it the hold stays off. pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; -/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU -/// cores powered until the GPU has been idle this long ([`Hold`]). +/// Idle policy initialized from the existing DT `apple,idle-off-delay-ms` property. +/// The existing `idle_off_delay_ms` sysfs attribute may override it. Above +/// [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU cores powered until it has been idle +/// this long ([`Hold`]). pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); +/// Use the same bounds as the existing idle-delay attribute. Missing DT policy +/// preserves the native delay; malformed or out-of-range values fail probe. +pub(super) fn idle_delay(requested: Option) -> kernel::error::Result { + let delay = requested.unwrap_or(FIRMWARE_IDLE_OFF_MS); + if !(FIRMWARE_IDLE_OFF_MS..=1000).contains(&delay) { + return Err(kernel::error::code::EINVAL); + } + Ok(delay) +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -237,6 +249,34 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { mod tests { use super::*; + #[test] + fn dt_idle_delay_is_bounded_and_absence_preserves_native_policy() { + assert_eq!(idle_delay(None), Ok(FIRMWARE_IDLE_OFF_MS)); + for delay in [2, 16, 1000] { + assert_eq!(idle_delay(Some(delay)), Ok(delay)); + } + for delay in [0, 1, 1001, u32::MAX] { + assert!(idle_delay(Some(delay)).is_err()); + } + } + + #[test] + fn frame_hold_survives_depth_one_gaps_but_releases_at_idle() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 16), HoldAction::Assert); + hold.asserted(); + for submission in 1..=8 { + // A completed depth-one dispatch followed by another short gap. + let now = submission * 3_000_000; + assert_eq!(hold.step(now, false, submission, 16), HoldAction::Recheck(16_000_000)); + assert!(hold.active()); + } + assert_eq!(hold.step(39_999_999, false, 8, 16), HoldAction::Recheck(1)); + assert_eq!(hold.step(40_000_000, false, 8, 16), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + #[test] fn holds_until_idle_for_the_delay_then_releases() { let mut hold = Hold::new(); From d375201dc26a185fa40ad3a94ca92fe08505d7ce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:17:47 +1100 Subject: [PATCH 013/171] drm/asahi: preserve raw G17 firmware reports for clock and fault qualification G17 discarded statistics payloads and unknown event words before returning ring slots. Keep the latest full statistics sample per role/type with host monotonic time and count in read-only debugfs, and add a static-key raw report tracepoint for complete trace/statistics/event timelines. Copy before publishing the consumer; retain the existing tag and transport quarantine rules. Debugfs owns the session and samples under the device mutex without waking firmware or reading gated core registers. No decoded achieved-clock/power ABI is available for these firmware builds. Do not relabel legacy pstate fields, requested OPPs or advertised maxima as achieved frequency. Preserve raw observations so their semantics can be qualified independently. KUnit covers full payload preservation, role/type separation and invalid-tag bounds. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/channel.rs] (cherry picked from commit e4ee19363ec942580be56321b7fca5dee46eb0cc in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/channel.rs | 40 +++++++---- drivers/gpu/drm/asahi/g17/mod.rs | 9 ++- drivers/gpu/drm/asahi/g17/reports.rs | 97 +++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.c | 10 +++ drivers/gpu/drm/asahi/g17/trace.h | 21 ++++++ drivers/gpu/drm/asahi/g17/trace.rs | 16 +++++ drivers/gpu/drm/asahi/g17/trace_api.h | 3 + 7 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/reports.rs diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index d96924cbbf9f91..568647ebb444ae 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -43,16 +43,21 @@ impl ReportRing { /// Host-side receive-ring state. A quarantined ring stays stopped until the /// device is destroyed; firmware restart does not clear it. -#[derive(Debug)] pub(crate) struct Rings { quarantined: [[bool; 2]; 2], + reports: super::reports::Reports, } impl Rings { - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new() -> Result { + Ok(Self { quarantined: [[false; 2]; 2], - } + reports: super::reports::Reports::new()?, + }) + } + + pub(super) fn write_reports(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.reports.write(f) } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { @@ -68,7 +73,7 @@ impl Rings { if *quarantined { return; } - if let Err(error) = Self::consume_report(init, role, ring) { + if let Err(error) = Self::consume_report(init, role, ring, &mut self.reports) { *quarantined = true; dev_err!( dev.as_ref(), @@ -80,7 +85,7 @@ impl Rings { } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + fn consume_report(init: &InitData, role: Role, ring: ReportRing, reports: &mut super::reports::Reports) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; @@ -94,14 +99,17 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - let tag = match ring { - // Copy the entire trace entry before returning its slot. - ReportRing::Trace => init.status_read_words::(role, offset)?[0], - ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], - }; + // Copy all words before returning the slot. Interpretation is a + // separate firmware qualification step, never a scheduling input. + let words = init.status_read_words::(role, offset)?; + super::trace::report(role as u8, ring as u8, &words); + let tag = words[0]; if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { return Err(EIO); } + if matches!(ring, ReportRing::Statistics) { + reports.record(role, words, super::now_ns()); + } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); consumer_word.store(consumer, Ordering::Relaxed); @@ -147,6 +155,7 @@ impl Rings { /// the caller can service the copied event using the complete device owner. pub(crate) struct Events { role: Role, + graph: super::fw::variant::InitGraph, remaining: u32, unknown_reported: bool, } @@ -167,9 +176,10 @@ impl Events { Ok(consumer != producer) } - pub(crate) fn new(role: Role) -> Self { + pub(crate) fn new(role: Role, graph: super::fw::variant::InitGraph) -> Self { Self { role, + graph, remaining: EVENT_SLOTS, unknown_reported: false, } @@ -195,8 +205,10 @@ impl Events { // Observe the producer before reading the record it publishes. fence(Ordering::Acquire); let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; - let record = EventRecord::from_words(init.status_read_words(role, offset)?); - let event = record.decode(); + let words = init.status_read_words(role, offset)?; + super::trace::report(role as u8, 2, &words); + let record = EventRecord::from_words(words); + let event = record.decode_for(self.graph); fence(Ordering::SeqCst); consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index bb951a85ca9616..181ea4ab9ab73b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -50,6 +50,7 @@ mod kick; mod object; mod port; pub(crate) mod power; +mod reports; mod preparation; mod qos; mod queue; @@ -1114,6 +1115,7 @@ pub(crate) struct Gpu { ids: gpu::SequenceIDs, shared: Arc, _capacity: Pin>>, + _reports: Pin>>, } impl Gpu { @@ -1223,6 +1225,10 @@ impl Gpu { directory.read_only_file(c"capacity", capacity::View(shared.clone())), GFP_KERNEL, )?; + let reports = KBox::pin_init( + directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1232,6 +1238,7 @@ impl Gpu { ids: Default::default(), shared, _capacity: capacity, + _reports: reports, }, GFP_KERNEL, )?; @@ -1373,7 +1380,7 @@ impl Gpu { bridge, port: port::Port::new()?, seen_submissions: 0, - rings: channel::Rings::new(), + rings: channel::Rings::new()?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs new file mode 100644 index 00000000000000..7e1afef70cad2d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Preserve the firmware's statistics without assigning unqualified units or +//! treating another firmware's pstate report as an achieved GPU frequency. +//! Writers and readers hold the device mutex; reads never power the GPU up. + +use super::{fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, Role}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Default)] +struct Sample { + count: u64, + host_ns: u64, + words: [u32; EVENT_WORDS], +} + +pub(super) struct Reports { + samples: KVec, +} + +impl Reports { + pub(super) fn new() -> Result { + // Allocate before firmware starts; receive and debugfs paths allocate nothing. + let mut samples = KVec::with_capacity(64, GFP_KERNEL)?; + for _ in 0..64 { + samples.push(Sample::default(), GFP_KERNEL)?; + } + Ok(Self { samples }) + } + + pub(super) fn record(&mut self, role: Role, words: [u32; EVENT_WORDS], host_ns: u64) { + let tag = words[0]; + if tag >= u32::BITS || STATISTICS_TYPES & (1 << tag) == 0 { + return; + } + let sample = &mut self.samples[role as usize * 32 + tag as usize]; + sample.count = sample.count.saturating_add(1); + sample.host_ns = host_ns; + sample.words = words; + } + + pub(super) fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified")?; + for (index, sample) in self.samples.iter().enumerate().filter(|(_, s)| s.count != 0) { + writeln!(f, "role={} tag={:#x} count={} host_ns={} words={:08x?}", + index / 32, index % 32, sample.count, sample.host_ns, sample.words)?; + } + Ok(()) + } +} + +/// The view owns the session, not a pointer into a detachable Firmware. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; + firmware.rings.write_reports(f) + } +} + +#[kernel::macros::kunit_tests(asahi_g17_reports)] +mod tests { + use super::*; + + #[test] + fn retains_all_words_and_separates_role_and_tag() { + let mut reports = Reports::new().unwrap(); + let mut words = [0; EVENT_WORDS]; + words[0] = 16; + words[17] = 0xdeadbeef; + reports.record(Role::Primary, words, 100); + reports.record(Role::Primary, words, 101); + reports.record(Role::Secondary, words, 102); + assert_eq!(reports.samples[16].count, 2); + assert_eq!(reports.samples[16].host_ns, 101); + assert_eq!(reports.samples[16].words, words); + assert_eq!(reports.samples[48].count, 1); + words[0] = 17; + reports.record(Role::Primary, words, 103); + assert_eq!(reports.samples[17].count, 1); + assert_eq!(reports.samples[16].count, 2); + } + + #[test] + fn rejects_invalid_tags_before_indexing() { + let mut reports = Reports::new().unwrap(); + for tag in [0, 15, 29, 31, 32, u32::MAX] { + let mut words = [0; EVENT_WORDS]; + words[0] = tag; + reports.record(Role::Primary, words, 1); + } + assert!(reports.samples.iter().all(|s| s.count == 0)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index 9e93291f5114ec..59f0094d129e9e 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -60,3 +60,13 @@ void asahi_g17_trace_teardown(u8 kind, u64 a, u64 b, u64 c) if (trace_asahi_g17_teardown_enabled()) trace_asahi_g17_teardown(kind, a, b, c); } + +bool asahi_g17_trace_report_enabled(void) +{ + return trace_asahi_g17_report_enabled(); +} + +void asahi_g17_trace_report(u8 role, u8 ring, const u32 *words) +{ + trace_asahi_g17_report(role, ring, words, __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index d1529a2775fcca..36e1b445f1b775 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -158,6 +158,27 @@ TRACE_EVENT(asahi_g17_event, TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) ); +/* Raw 18-word records: trace=0, statistics=1, event=2. No unit inference. */ +TRACE_EVENT(asahi_g17_report, + TP_PROTO(u8 role, u8 ring, const u32 *words, u64 counter), + TP_ARGS(role, ring, words, counter), + TP_STRUCT__entry( + __field(u8, role) + __field(u8, ring) + __array(u32, words, 18) + __field(u64, counter) + ), + TP_fast_assign( + __entry->role = role; + __entry->ring = ring; + memcpy(__entry->words, words, sizeof(__entry->words)); + __entry->counter = counter; + ), + TP_printk("role=%u ring=%u words=%s counter=%llu", __entry->role, + __entry->ring, __print_array(__entry->words, 18, sizeof(u32)), + __entry->counter) +); + #endif #undef TRACE_INCLUDE_PATH diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index d5f61f50a93f68..1a99b41a80753b 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -90,3 +90,19 @@ pub(super) fn event(phase: u8) { // SAFETY: Plain values only. unsafe { asahi_g17_trace_event(phase) }; } + +unsafe extern "C" { + fn asahi_g17_trace_report_enabled() -> bool; + fn asahi_g17_trace_report(role: u8, ring: u8, words: *const u32); +} + +/// Copies a complete firmware record while the host still owns its slot. +/// Static-key gated; no MMIO reads or firmware requests are issued. +pub(super) fn report(role: u8, ring: u8, words: &[u32; super::fw::channels::EVENT_WORDS]) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_report_enabled() } { + return; + } + // SAFETY: C synchronously copies the complete initialized 18-word array. + unsafe { asahi_g17_trace_report(role, ring, words.as_ptr()) }; +} diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h index 0049aa00796342..be5eff4c8161bf 100644 --- a/drivers/gpu/drm/asahi/g17/trace_api.h +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -22,4 +22,7 @@ void asahi_g17_trace_message(u8 kind, u64 value); void asahi_g17_trace_event(u8 phase); void asahi_g17_trace_teardown(u8 kind, u64 a, u64 b, u64 c); +bool asahi_g17_trace_report_enabled(void); +void asahi_g17_trace_report(u8 role, u8 ring, const u32 *words); + #endif /* _ASAHI_G17_TRACE_API_H */ From 33c9504ac16e81c606e706fdc66dcd8b934baa50 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:18:33 +1100 Subject: [PATCH 014/171] drm/asahi: snapshot the blamed render before acknowledging GPU recovery Retain bounded first-fault evidence from the already mapped render graph: queue identities, completion stamps, free-list balance and the oldest four commands' ring cursors, pass timestamps and scheduler identity. Sample while firmware is halted at state 1 before publishing recovery targets and before acknowledging the restart. Explicitly report a borrowed/unavailable pair. The existing recovery reason and blame alone do not distinguish a stalled tiler from fragment execution, lost completion visibility or work that never started. This observation makes that distinction possible without new MMIO addresses, invented reason codes, timeout changes or weaker release rules. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 22c3c57a7c1597175e9424782b10595e40d0e382 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 13 +++++++++++++ drivers/gpu/drm/asahi/g17/recovery.rs | 5 +++++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 +++ .../gpu/drm/asahi/g17/runtime/render/recovery.rs | 14 ++++++++++++++ 4 files changed, 35 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3261f2f0640b32..fe72c71129b03c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1033,6 +1033,19 @@ impl Pair { }, ] } + /// Bounded first-fault evidence while the recovery owner has firmware + /// halted, before it acknowledges state 1. Never changes retirement. + pub(crate) fn report_recovery(&self, stamps: Result<[u32; 2]>) { + pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", + self.slot, self.qids(), self.active.len(), self.prepared.is_some(), + self.quarantined, stamps, self.free_list.counters()); + for active in self.active.iter().take(4) { + pr_warn!("G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, active.outer_published, active.packet.context.id(), + active.packet.order.sequence, observation::snapshot(&self.memory, active.ticket.ordinal)); + } + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 7e5b84f6e630ee..9eb14fe521d3ee 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -103,6 +103,8 @@ pub(super) trait Queues { /// partly executed, so it must never report success; see /// `Completion::note_interrupted`. fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; + /// Optional diagnostics only; absence cannot alter attribution or release. + fn report_blamed_render(&self, _qid: u8) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -242,6 +244,9 @@ pub(super) fn service( } let mut sources = host.fault_sources()?; sources.report(request.generation, super::queue::compute::retirement::stampless_total()); + if let Some(qid) = request.blamed { + host.report_blamed_render(qid); + } let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index af3f5f491430e0..3bd159037e2660 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -231,6 +231,9 @@ impl recovery::Queues for Service<'_> { queue.note_interrupted(stamp) }) } + fn report_blamed_render(&self, qid: u8) { + self.firmware.queues.report_render_recovery(qid, &self.firmware.port); + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs index 3a5b042fff6834..20f537ab69598c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -6,6 +6,20 @@ use crate::g17::runtime::Registry; use kernel::prelude::*; impl Registry { + pub(in crate::g17::runtime) fn report_render_recovery(&self, qid: u8, port: &crate::g17::port::Port) { + for entry in self.render.entries.iter().flatten().filter(|entry| { + entry.reservation.ids.iter().any(|id| id.qid() == qid) + }) { + if let Some(pair) = &entry.pair { + let qids = pair.qids(); + pair.report_recovery((|| Ok([port.stamp(qids[0])?, port.stamp(qids[1])?]))()); + } else { + pr_warn!("G17: recovery render queue={} unavailable borrowed={} constructing={}\n", + qid, entry.borrowed_active, entry.constructing); + } + } + } + pub(in crate::g17::runtime) fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { self.render.entries.iter().flatten() .filter_map(|entry| entry.pair.as_ref()) From fd2c52d58875245d118c4850506440cf79040062 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:35:32 +1100 Subject: [PATCH 015/171] soc: apple: drive T6040 GPU memory demand from publication and continuous idle T6040 keeps the GFX supplier powered for the firmware's lifetime, as its PMGR DT documents. Using that supplier as CLPC demand therefore pins DCS 5 / fabric 2 even after the GPU cores gate. Neo's activity-following supplier is not a valid T6040 idle witness. Add a DT-selected CLPC demand consumer. Every actual GPU publication asserts it, including off-lock direct publication. The existing race-checked 16 ms continuous-idle effort clear releases it; quarantined work does not count as execution. CLPC retains its existing 32 ms hold and CPU arbitration. Firmware shutdown and consumer destruction clear the request. An independent refcounted request allocation is safe across provider removal, and a device link orders the supplier. The hot path uses only a leaf spinlock, with no sleeping call, allocation or GPU callback. Missing requested providers defer probe. Expose last requested floor, reason bits and host timestamps alongside actual PMC readback, and expose safe core-power plus last publication time in the read-only G17 firmware_reports view. Generic runtime PM remains distinct from firmware core gating: do not relabel unsupported generic PM as active/suspended. Keep Neo's existing PMGR input and all voltages, floors and delays unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: integration/aurora has no T6040 device tree or CLPC table: the t6040.dtsi and T6040 table changes are left out, and the host GPU demand interface, its probe allocation and its demand input are ported into integration/aurora's clpc.c without the vote-reason diagnostics] (cherry picked from commit 32f7efefcb39734a758d8dbbd3708099f288c067 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../devicetree/bindings/gpu/apple,agx.yaml | 7 + drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/g17/clpc.c | 16 +++ drivers/gpu/drm/asahi/g17/clpc.h | 9 ++ drivers/gpu/drm/asahi/g17/mod.rs | 4 +- drivers/gpu/drm/asahi/g17/port.rs | 19 ++- drivers/gpu/drm/asahi/g17/power.rs | 40 ++++++ drivers/gpu/drm/asahi/g17/reports.rs | 5 + drivers/soc/apple/clpc.c | 129 ++++++++++++++++++ include/linux/soc/apple/clpc.h | 24 ++++ 10 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/clpc.c create mode 100644 drivers/gpu/drm/asahi/g17/clpc.h create mode 100644 include/linux/soc/apple/clpc.h diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index bfed6b4659d8ad..92cdc61b6623ae 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -64,6 +64,13 @@ properties: - const: hw-cal-b - const: globals + apple,clpc: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + Host performance controller receiving GPU submission and idle demand. + GFX supplier power cannot represent GPU work on firmware that keeps + its coprocessor domain powered while the cores are idle. + apple,idle-off-delay-ms: $ref: /schemas/types.yaml#/definitions/uint32 minimum: 2 diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 33d575639d81c4..1041f0ded2ed41 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o +asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c new file mode 100644 index 00000000000000..b8ae1242cd8aad --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include "clpc.h" + +int asahi_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + return apple_clpc_gpu_get(dev, out); +} +void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) +{ + apple_clpc_gpu_set_active(gpu, active); +} +void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu) +{ + apple_clpc_gpu_put(gpu); +} diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h new file mode 100644 index 00000000000000..3fc6f754734547 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _ASAHI_G17_CLPC_H +#define _ASAHI_G17_CLPC_H +struct device; +struct apple_clpc_gpu; +int asahi_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); +void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); +#endif diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 181ea4ab9ab73b..e985dd695273c8 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -645,6 +645,7 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { + self.port.set_effort_full(false); if let Some(secondary) = self.secondary.as_mut() { secondary.stop(); } @@ -695,6 +696,7 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } @@ -1378,7 +1380,7 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new()?, + port: port::Port::new(power::Demand::new(pdev.as_ref().as_raw())?)?, seen_submissions: 0, rings: channel::Rings::new()?, recovery: recovery::State::new(), diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 8d129c49c640cd..4c2c320d4089b0 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -100,6 +100,8 @@ pub(super) struct Port { publications: AtomicU64, /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, + demand: super::power::Demand, + last_submission_ns: AtomicU64, submissions: AtomicU64, } @@ -137,7 +139,7 @@ impl Drop for OuterClaim<'_> { } impl Port { - pub(super) fn new() -> Result> { + pub(super) fn new(demand: super::power::Demand) -> Result> { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), @@ -148,6 +150,8 @@ impl Port { generation: AtomicU64::new(0), publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), + demand, + last_submission_ns: AtomicU64::new(0), submissions: AtomicU64::new(0), }), GFP_KERNEL, @@ -421,6 +425,7 @@ impl Port { /// clear sees this submission or this submission sees the flag withdrawn. pub(super) fn note_submission(&self) -> bool { self.submissions.fetch_add(1, Ordering::SeqCst); + self.note_gpu_activity(); self.effort_full.load(Ordering::SeqCst) } @@ -430,5 +435,17 @@ impl Port { pub(super) fn set_effort_full(&self, full: bool) { self.effort_full.store(full, Ordering::SeqCst); + self.demand.set_active(full); + } + + pub(super) fn note_gpu_activity(&self) { + self.last_submission_ns.store(super::now_ns(), Ordering::Relaxed); + self.demand.set_active(true); + } + + pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "effort_full={} direct_submissions={} last_submission_ns={}", + self.effort_full.load(Ordering::SeqCst), self.submissions(), + self.last_submission_ns.load(Ordering::Relaxed)) } } diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index bbd10b204edd75..075d2829e42435 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -300,3 +300,43 @@ mod tests { assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); } } + +unsafe extern "C" { + fn asahi_clpc_gpu_get(dev: *mut kernel::bindings::device, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// DT-selected memory-demand provider. Its independent allocation outlives a +/// detached provider; a device link orders supplier removal and suspend. +pub(super) struct Demand(*mut core::ffi::c_void); + +// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. +unsafe impl Send for Demand {} +// SAFETY: shared access only changes that locked request, never its lifetime. +unsafe impl Sync for Demand {} + +impl Demand { + pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { + let mut out = core::ptr::null_mut(); + // SAFETY: live platform device and writable result; C returns an owned reference. + let ret = unsafe { asahi_clpc_gpu_get(dev, &mut out) }; + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(Self(out)) + } + } + + pub(super) fn set_active(&self, active: bool) { + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_active(self.0, active) }; + } +} + +impl Drop for Demand { + fn drop(&mut self) { + // SAFETY: releases exactly the reference returned by get, clearing demand first. + unsafe { asahi_clpc_gpu_put(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 7e1afef70cad2d..62ed791322ad99 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -57,6 +57,11 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; + let core_power = firmware.bridge.regs.g17_core_power(); + writeln!(f, "power_management=firmware core_power={:#x} cores_powered={} idle_hold_ms={}", + core_power, core_power & 0xf != 0, + super::power::IDLE_HOLD_MS.load(core::sync::atomic::Ordering::Relaxed))?; + firmware.port.write_power(f)?; firmware.rings.write_reports(f) } } diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 382dc669b5870b..92413280785661 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -21,6 +21,12 @@ */ #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -86,6 +92,15 @@ struct apple_clpc_vote { unsigned long last_demand; }; +/* Independent lifetime: a late consumer put never dereferences a removed provider. */ +struct apple_clpc_gpu { + struct kref ref; + spinlock_t lock; + bool active; + u64 updated_ns; + u64 sequence; +}; + struct apple_clpc { struct device *dev; struct device *pmc; @@ -109,6 +124,7 @@ struct apple_clpc { unsigned long gpu_read_errors; struct dentry *debugfs; struct regmap *gpu_pmgr; + struct apple_clpc_gpu *gpu; u32 gpu_offset; /* Graphics powered at the last sample, since when, and that sample's time */ bool gpu_on; @@ -126,6 +142,92 @@ struct apple_clpc { unsigned long gpu_floor_returns; }; +static void apple_clpc_gpu_free(struct kref *ref) +{ + kfree(container_of(ref, struct apple_clpc_gpu, ref)); +} + +void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) +{ + unsigned long flags; + + if (!gpu) + return; + spin_lock_irqsave(&gpu->lock, flags); + gpu->active = active; + gpu->updated_ns = ktime_get_mono_fast_ns(); + gpu->sequence++; + spin_unlock_irqrestore(&gpu->lock, flags); +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_set_active); + +void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) +{ + if (gpu) { + apple_clpc_gpu_set_active(gpu, false); + kref_put(&gpu->ref, apple_clpc_gpu_free); + } +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_put); + +int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + struct device_node *np; + struct platform_device *provider; + struct apple_clpc *clpc; + struct device_link *link; + int ret = -EPROBE_DEFER; + + *out = NULL; + np = of_parse_phandle(dev->of_node, "apple,clpc", 0); + if (!np) + return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; + if (!of_device_is_compatible(np, "apple,t6040-clpc")) { + of_node_put(np); + return -EINVAL; + } + provider = of_find_device_by_node(np); + of_node_put(np); + if (!provider) + return ret; + link = device_link_add(dev, &provider->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + put_device(&provider->dev); + return -ENOMEM; + } + device_lock(&provider->dev); + clpc = device_is_bound(&provider->dev) ? dev_get_drvdata(&provider->dev) : NULL; + if (clpc && clpc->gpu && refcount_read(&clpc->gpu->ref.refcount) == 1) { + kref_get(&clpc->gpu->ref); + *out = clpc->gpu; + ret = 0; + } + device_unlock(&provider->dev); + put_device(&provider->dev); + return ret; +} +EXPORT_SYMBOL_GPL(apple_clpc_gpu_get); + +static bool apple_clpc_gpu_active(struct apple_clpc_gpu *gpu, u64 *updated, u64 *sequence) +{ + unsigned long flags; + bool active; + + spin_lock_irqsave(&gpu->lock, flags); + active = gpu->active; + *updated = gpu->updated_ns; + *sequence = gpu->sequence; + spin_unlock_irqrestore(&gpu->lock, flags); + return active; +} + +static void apple_clpc_gpu_owner_put(void *data) +{ + struct apple_clpc_gpu *gpu = data; + + kref_put(&gpu->ref, apple_clpc_gpu_free); +} + /* cpufreq policies can appear after probe; pick up new ones as they do. */ static void apple_clpc_find_clusters(struct apple_clpc *clpc) { @@ -339,6 +441,13 @@ static void apple_clpc_sample(struct apple_clpc *clpc) for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) demand[rail] = max(demand[rail], cl->request[rail]); } + if (clpc->gpu) { + u64 updated, sequence; + + if (apple_clpc_gpu_active(clpc->gpu, &updated, &sequence)) + for (rail = 0; rail < APPLE_CLPC_RAILS; rail++) + demand[rail] = clpc->hw->map[rail][APPLE_CLPC_LEVELS - 1]; + } if (clpc->gpu_pmgr && apple_clpc_gpu_streaming(clpc, now)) { bool released = apple_clpc_gpu_limited(clpc, now); @@ -389,6 +498,13 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) clpc->hw->rail[rail]), clpc->hw->map[rail][0], clpc->hw->map[rail][1], clpc->hw->map[rail][2], clpc->hw->map[rail][3]); + if (clpc->gpu) { + u64 updated, sequence; + bool active = apple_clpc_gpu_active(clpc->gpu, &updated, &sequence); + + seq_printf(s, "host_gpu: active %u request_ns %llu sequence %llu\n", + active, updated, sequence); + } seq_printf(s, "samples: %lu update errors: %lu gpu read errors: %lu\n", clpc->samples, clpc->update_errors, clpc->gpu_read_errors); if (clpc->gpu_pmgr) { @@ -467,6 +583,19 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->dev = dev; clpc->pmc = dev->parent; clpc->hw = of_device_get_match_data(dev); + /* T6040 firmware keeps GFX powered for its whole lifetime. A separate + * host publication/idle input, not that supplier state, drives demand. + */ + if (of_device_is_compatible(dev->of_node, "apple,t6040-clpc")) { + clpc->gpu = kzalloc_obj(*clpc->gpu); + if (!clpc->gpu) + return -ENOMEM; + kref_init(&clpc->gpu->ref); + spin_lock_init(&clpc->gpu->lock); + ret = devm_add_action_or_reset(dev, apple_clpc_gpu_owner_put, clpc->gpu); + if (ret) + return ret; + } /* The property is optional: without it, CPU demand alone sets the floors. */ if (clpc->hw->gpu_demand) gpu = of_parse_phandle(dev->of_node, "apple,gpu-power-state", 0); diff --git a/include/linux/soc/apple/clpc.h b/include/linux/soc/apple/clpc.h new file mode 100644 index 00000000000000..37b2cfce5594a9 --- /dev/null +++ b/include/linux/soc/apple/clpc.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _LINUX_SOC_APPLE_CLPC_H +#define _LINUX_SOC_APPLE_CLPC_H + +#include +#include +#include + +struct apple_clpc_gpu; + +#if IS_REACHABLE(CONFIG_APPLE_CLPC) +int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); +void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); +void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu); +#else +static inline int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +{ + *out = NULL; + return device_property_present(dev, "apple,clpc") ? -ENODEV : 0; +} +static inline void apple_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active) {} +static inline void apple_clpc_gpu_put(struct apple_clpc_gpu *gpu) {} +#endif +#endif From 77e47cddbf9d357370063999b54dcb17f9622483 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:37:42 +1100 Subject: [PATCH 016/171] drm/asahi: pass an opaque device across the CLPC FFI boundary Keep the C wrapper responsible for the device layout, as the existing sysfs wrappers do. Rust bindings include private configuration-dependent scheduler members and cannot be named in a WERROR-safe extern signature. The live platform device and owned request lifetime remain unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 61a2fec62a709db5b435a9a41ff51270362d4a80 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/clpc.c | 2 +- drivers/gpu/drm/asahi/g17/clpc.h | 2 +- drivers/gpu/drm/asahi/g17/power.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/clpc.c b/drivers/gpu/drm/asahi/g17/clpc.c index b8ae1242cd8aad..58adaba19910f2 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.c +++ b/drivers/gpu/drm/asahi/g17/clpc.c @@ -2,7 +2,7 @@ #include #include "clpc.h" -int asahi_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) +int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out) { return apple_clpc_gpu_get(dev, out); } diff --git a/drivers/gpu/drm/asahi/g17/clpc.h b/drivers/gpu/drm/asahi/g17/clpc.h index 3fc6f754734547..372820ade7efff 100644 --- a/drivers/gpu/drm/asahi/g17/clpc.h +++ b/drivers/gpu/drm/asahi/g17/clpc.h @@ -3,7 +3,7 @@ #define _ASAHI_G17_CLPC_H struct device; struct apple_clpc_gpu; -int asahi_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out); +int asahi_clpc_gpu_get(void *dev, struct apple_clpc_gpu **out); void asahi_clpc_gpu_set_active(struct apple_clpc_gpu *gpu, bool active); void asahi_clpc_gpu_put(struct apple_clpc_gpu *gpu); #endif diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 075d2829e42435..a2e293df91510f 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -302,7 +302,7 @@ mod tests { } unsafe extern "C" { - fn asahi_clpc_gpu_get(dev: *mut kernel::bindings::device, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); } @@ -320,7 +320,7 @@ impl Demand { pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { let mut out = core::ptr::null_mut(); // SAFETY: live platform device and writable result; C returns an owned reference. - let ret = unsafe { asahi_clpc_gpu_get(dev, &mut out) }; + let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { From 9ddc2c0335074a533fb2450e5297e7627c193ba0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:39:16 +1100 Subject: [PATCH 017/171] drm/asahi: expose firmware-managed GPU core power as a read-only view Provide a concise debugfs power view with the existing always-accessible core power witness, host hold state, idle policy, effort, direct-publication count and last publication time. Read under the live firmware owner without waking the GPU. The same values accompany raw firmware reports. Generic Linux runtime PM does not manage this firmware's autonomous core gating and may correctly report unsupported. Report the actual firmware power witness without enabling a fictitious runtime-suspend implementation. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 460e4606950d5358128cb09f5b131a476a37be42 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/reports.rs | 25 ++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index e985dd695273c8..f9399bbc3bba26 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1118,6 +1118,7 @@ pub(crate) struct Gpu { shared: Arc, _capacity: Pin>>, _reports: Pin>>, + _power: Pin>>, } impl Gpu { @@ -1231,6 +1232,10 @@ impl Gpu { directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), GFP_KERNEL, )?; + let power = KBox::pin_init( + directory.read_only_file(c"power", reports::PowerView(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1241,6 +1246,7 @@ impl Gpu { shared, _capacity: capacity, _reports: reports, + _power: power, }, GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 62ed791322ad99..b67512f7eec6bc 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -57,15 +57,30 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; - let core_power = firmware.bridge.regs.g17_core_power(); - writeln!(f, "power_management=firmware core_power={:#x} cores_powered={} idle_hold_ms={}", - core_power, core_power & 0xf != 0, - super::power::IDLE_HOLD_MS.load(core::sync::atomic::Ordering::Relaxed))?; - firmware.port.write_power(f)?; + write_power(firmware, f)?; firmware.rings.write_reports(f) } } +fn write_power(firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let core_power = firmware.bridge.regs.g17_core_power(); + writeln!(f, "power_management=firmware sampled_ns={} core_power={:#x} cores_powered={} host_hold={} idle_hold_ms={}", + super::now_ns(), core_power, core_power & 0xf != 0, firmware.hold.active(), + super::power::IDLE_HOLD_MS.load(core::sync::atomic::Ordering::Relaxed))?; + firmware.port.write_power(f) +} + +/// Actual firmware-managed core gating, separate from generic Linux runtime PM. +pub(super) struct PowerView(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for PowerView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + write_power(firmware, f) + } +} + #[kernel::macros::kunit_tests(asahi_g17_reports)] mod tests { use super::*; From 01626e9fa62dedcd11ddb3bef14601c9ea693472 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 17:48:41 +1100 Subject: [PATCH 018/171] drm/asahi: retain memory demand until both firmware processors stop Clear the owned CLPC request after the processor stop sequence, so a slow shutdown does not begin the memory-floor idle hold while firmware can still consume work. Late request destruction remains independently safe and clears the request again without accessing provider storage. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit fa12ae88cc55fbda94349566ca606f78482a6d24 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f9399bbc3bba26..c2cea7bd07db9a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -645,11 +645,11 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.port.set_effort_full(false); if let Some(secondary) = self.secondary.as_mut() { secondary.stop(); } self.primary.stop(); + self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. From 8c4c25026d7e869b471881e7f9ec7ff56a9b62e3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:35:35 +1100 Subject: [PATCH 019/171] drm/asahi: freeze render progress before acknowledging G17 recovery Resolve slot attribution while firmware is halted in state 1, alongside its key and monitor snapshot. Previously classify() read pass timestamps after the 3->0 acknowledgement, when resumed work could change them and make a later-started peer appear responsible for the original fault. Keep the captured attribution through the handshake. Unknown 25G83 slot keys still preserve firmware blame; no new retirement or release witness is introduced. This fixes the shared sampling race, not the unqualified T6040 reason-2 first faults. KUnit drives the real service() handshake and rejects a pass read after the first acknowledgement. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 8bce44c04ac01e54b8425c881ba9cf3155fd23e5 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 109 +++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 9eb14fe521d3ee..8e46bd467fe1c8 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -247,6 +247,11 @@ pub(super) fn service( if let Some(qid) = request.blamed { host.report_blamed_render(qid); } + // The keys, monitor and pass-start evidence must describe the same halted + // instant. Firmware may update pass timestamps as soon as state 3 is + // acknowledged; sampling them during classification can blame a bystander. + let attribution = sources.attribute(request.blamed, + |qid, stamp| host.render_pass_started(qid, stamp)); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; @@ -277,7 +282,7 @@ pub(super) fn service( host.recovery_mut().pending = None; let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); - let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + let (cancelled, blame) = classify(host, entries, request.blamed, attribution); if listed != 0 { pr_warn!( "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", @@ -331,10 +336,9 @@ fn read_entries(host: &mut H) -> Entries { /// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed /// render that had not started while others had (or with no culprit at all) is innocent, and its /// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { +fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; - let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} Attribution::Queues(qids) => guilty_qids = qids, @@ -394,3 +398,102 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }); (cancelled & !(guilty_qids & !collateral) & spared, blame) } + + +#[kernel::macros::kunit_tests(asahi_g17_recovery_snapshot)] +mod tests { + use super::*; + use core::cell::Cell; + + struct Halted { + recovery: State, + state: Cell, + reads: Cell, + known_keys: bool, + blamed: bool, + } + + impl Halted { + fn new(known_keys: bool) -> Self { + let mut recovery = State::new(); + assert!(recovery.request(0, 38).unwrap()); + Self { recovery, state: Cell::new(1), reads: Cell::new(0), + known_keys, blamed: false } + } + } + + impl Memory for Halted { + fn recovery_state(&self) -> Result { Ok(self.state.get()) } + fn host_recovery(&self) -> Result { Ok(0) } + fn fault_sources(&self) -> Result { + assert_eq!(self.state.get(), 1); + let mut sources = Sources::sample(3, |slot| { + Some((if self.known_keys { ((38 + slot as u64) << 40) | 1 } else { 0 }, 0)) + }); + sources.reason = Some(2); + Ok(sources) + } + fn transition(&self, expected: u32, next: u32) -> Result { + assert_eq!(self.state.get(), expected); + self.state.set(next); + Ok(()) + } + fn clear_timestamps(&self) -> Result { panic!("active render scheduler") } + fn entry(&self, _index: usize) -> Result<(u32, u32)> { + assert_eq!(self.state.get(), 3); + Ok((0, 0)) + } + fn wait_tick(&self) { + assert_eq!(self.state.get(), 2); + self.state.set(3); + } + } + + impl Queues for Halted { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, qid: u8, _stamp: u64) -> Option { + // This is the regression oracle: state 0/2/3 is too late. A + // resumed peer may have started after the fault we are attributing. + assert_eq!(self.state.get(), 1); + self.reads.set(self.reads.get() + 1); + Some(qid == 39) + } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, _qid: u8) -> Option { None } + fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } + fn settle_blamed_render(&mut self, qid: u8, _error: Error) { + assert_eq!(qid, 38); + self.blamed = true; + } + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + fn announce_withheld(&mut self) {} + } + + impl Host for Halted { + fn recovery(&self) -> &State { &self.recovery } + fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } + } + + #[test] + fn pass_evidence_is_frozen_before_the_first_acknowledgement() { + let mut host = Halted::new(true); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 2); + assert_eq!(host.state.get(), 0); + assert_eq!(host.recovery.generation(), 1); + assert!(!host.blamed); + } + + #[test] + fn unknown_25g83_keys_still_preserve_firmware_blame() { + let mut host = Halted::new(false); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 0); + assert!(host.blamed); + assert_eq!(host.state.get(), 0); + } +} From 5b9d99ad1e768aa87bb256f2c810b11e2ed3b69d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:37:11 +1100 Subject: [PATCH 020/171] drm/asahi: record halted tiler capacity alongside G17 recovery progress Record the declared TVB state, block inventory and high-water values, plus the blamed pair's bounded render geometry/GTP layout and pending parameter-buffer growth. All reads run in the existing state-1 snapshot; there is no allocation, MMIO access or high-water reset. These observations distinguish a full tiler, a delayed grow request and an unstamped unstarted pass in the large point/barrier CTS cases. They are diagnostics, not a timeout extension or a claim that reason 2 means TVB exhaustion. Keep raw captures in private qualification evidence. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 0d753834db34dc82c6f4e7198cc72d48e8e87673 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer.rs | 16 ++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/render.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/runtime/render/grow.rs | 4 ++-- .../gpu/drm/asahi/g17/runtime/render/recovery.rs | 2 ++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 8f3a03fbb8dba3..fd6460db2d1fbc 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -677,6 +677,22 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + /// Read-only diagnostic while firmware is halted. In particular this + /// does not reset the high-water counter as automatic_target() does. + pub(crate) fn report_recovery(&self) { + let read = |offset| -> Result { + Ok(self.graph.word(offset)?.load(Ordering::Relaxed)) + }; + pr_warn!("G17: recovery tvb id={} blocks={} cursors={:?} pages={:?} capacity={:?} producer={:?} consumer={:?} max_pages={:?} reset={:?} initialized={}\n", + self.buffer_id, self.blocks(), self.cursors(), + read(core::mem::offset_of!(fw::State, pages)), + read(core::mem::offset_of!(fw::State, block_capacity)), + read(core::mem::offset_of!(fw::State, producer)), + read(core::mem::offset_of!(fw::State, consumer)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + self.initialized); + } pub(crate) fn state_va(&self) -> u64 { self.graph.gpu_va() } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fe72c71129b03c..f16e682b1e8fc5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1039,7 +1039,13 @@ impl Pair { pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", self.slot, self.qids(), self.active.len(), self.prepared.is_some(), self.quarantined, stamps, self.free_list.counters()); + self.memory.manager.report_recovery(); for active in self.active.iter().take(4) { + if let Validated::Render { pass, .. } = &active.packet.command { + pr_warn!("G17: recovery render geometry ordinal={} width={} height={} layers={} layout={:?}\n", + active.ticket.ordinal, pass.width, pass.height, pass.layers, + target::Layout::new(pass, self.clusters, self.variant.programs)); + } pr_warn!("G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", active.ticket.ordinal, active.outer_published, active.packet.context.id(), active.packet.order.sequence, observation::snapshot(&self.memory, active.ticket.ordinal)); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 20569af84bc7bf..277534328406ee 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -14,8 +14,8 @@ use kernel::prelude::*; const RETRY_LIMIT: u16 = 1000; pub(super) struct Grow { - requests: KVec, - active: Option, + pub(super) requests: KVec, + pub(super) active: Option, pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, release_attempts: u16, } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs index 20f537ab69598c..7a6d987591c764 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -7,6 +7,8 @@ use kernel::prelude::*; impl Registry { pub(in crate::g17::runtime) fn report_render_recovery(&self, qid: u8, port: &crate::g17::port::Port) { + pr_warn!("G17: recovery grow pending={} active={:?} reply={:?}\n", + self.render.grow.requests.len(), self.render.grow.active, self.render.grow.reply); for entry in self.render.entries.iter().flatten().filter(|entry| { entry.reservation.ids.iter().any(|id| id.qid() == qid) }) { From bd329341487d1bbea3689c4db19638d004ec455a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:37:12 +1100 Subject: [PATCH 021/171] drm/asahi: trace G17 input conversion and scheduler hop boundaries Add static-key input-resolution and scheduler lifecycle traces. Join these to existing publish/complete and dma_fence/sched events using the engine/context/sequence and fence identity to separate conversion, inline setup, dependency/capacity waits, scheduler dispatch and firmware execution in multi-queue signal hops. Counters are read only when each tracepoint is enabled. The trace path does not poll a fence, allocate, request GPU power or alter scheduling. These are measurement boundaries; no unmeasured latency saving is claimed. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 84f47efb93f38852c0686537e0e1ff298b7de8d4 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/job.rs | 16 ++++++-- drivers/gpu/drm/asahi/g17/queue/mod.rs | 6 +++ drivers/gpu/drm/asahi/g17/trace.c | 15 +++++++ drivers/gpu/drm/asahi/g17/trace.h | 56 ++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 30 ++++++++++++++ drivers/gpu/drm/asahi/g17/trace_api.h | 4 ++ 6 files changed, 124 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 29908f214bdc53..b6c61e3bba847d 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -363,10 +363,17 @@ impl sched::JobImpl for Job { } fn prepare(job: &mut Self) -> Option { - job.backend.prepare(&job.packet) + super::trace::schedule(&job.packet, 5, 0); + let wait = job.backend.prepare(&job.packet); + super::trace::schedule(&job.packet, 6, i32::from(wait.is_some())); + wait } fn run(job: &mut Self) -> Result> { - job.backend.publish(job.packet.clone()).map(Some) + super::trace::schedule(&job.packet, 3, 0); + let result = job.backend.publish(job.packet.clone()).map(Some); + super::trace::schedule(&job.packet, 4, + result.as_ref().err().map_or(0, |error| error.to_errno())); + result } fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep @@ -430,6 +437,7 @@ impl Scheduler { early: bool, publication: Option, ) -> Result { + super::trace::schedule(&packet, 0, dependencies.len() as i32); let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); // Only commands which actually write timestamps become host-prefix @@ -462,7 +470,9 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if !(early && self.backend.try_early(&packet)) { + let inline = early && self.backend.try_early(&packet); + super::trace::schedule(&packet, if inline { 1 } else { 2 }, 0); + if !inline { if packet.completion.needs_source_ready() { packet.completion.set_source_ready(ready.clone()); } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 16891524e25a06..d22c0f617f374a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -212,6 +212,9 @@ impl Queue { } // Only convert inputs when no independent host-prefix wait could keep // the retained source bookkeeping queued behind an external signal. + if engine == Engine::Compute { + super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); + } let sources = if engine == Engine::Compute && !inputs.is_empty() && !destinations.has_outputs() && self.frontiers[0].iter().all(|(sequence, fences)| @@ -225,6 +228,9 @@ impl Queue { failure.as_ref().is_none_or(|(sequence, _)| !Order::contains(order.wait_through[index], *sequence))) { self.backend.input_sources(inputs) } else { None }; + if engine == Engine::Compute { + super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); + } // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index 59f0094d129e9e..62307838bb36e4 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -70,3 +70,18 @@ void asahi_g17_trace_report(u8 role, u8 ring, const u32 *words) { trace_asahi_g17_report(role, ring, words, __arch_counter_get_cntvct()); } + +void asahi_g17_trace_inputs(u32 ctx, u64 seq, u8 phase, u32 inputs, u32 sources, + u32 waits, bool converted) +{ + if (trace_asahi_g17_inputs_enabled()) + trace_asahi_g17_inputs(ctx, seq, phase, inputs, sources, waits, + converted, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_schedule(u8 engine, u32 ctx, u64 seq, u8 phase, s32 detail) +{ + if (trace_asahi_g17_schedule_enabled()) + trace_asahi_g17_schedule(engine, ctx, seq, phase, detail, + __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index 36e1b445f1b775..c8cecbcc7f400b 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -158,6 +158,62 @@ TRACE_EVENT(asahi_g17_event, TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) ); +/* Input conversion: phase 0 entry, 1 result. converted=0 also covers no live input. */ +TRACE_EVENT(asahi_g17_inputs, + TP_PROTO(u32 ctx, u64 seq, u8 phase, u32 inputs, u32 sources, u32 waits, + bool converted, u64 counter), + TP_ARGS(ctx, seq, phase, inputs, sources, waits, converted, counter), + TP_STRUCT__entry( + __field(u32, ctx) + __field(u64, seq) + __field(u8, phase) + __field(u32, inputs) + __field(u32, sources) + __field(u32, waits) + __field(bool, converted) + __field(u64, counter) + ), + TP_fast_assign( + __entry->ctx = ctx; + __entry->seq = seq; + __entry->phase = phase; + __entry->inputs = inputs; + __entry->sources = sources; + __entry->waits = waits; + __entry->converted = converted; + __entry->counter = counter; + ), + TP_printk("ctx=%u seq=%llu phase=%u inputs=%u sources=%u waits=%u converted=%u counter=%llu", + __entry->ctx, __entry->seq, __entry->phase, __entry->inputs, + __entry->sources, __entry->waits, __entry->converted, __entry->counter) +); + +/* phase: 0 enqueue, 1 inline published, 2 scheduler fallback, 3 run begin, + * 4 run end, 5 prepare begin, 6 prepare end. detail: dependencies, error or wait. */ +TRACE_EVENT(asahi_g17_schedule, + TP_PROTO(u8 engine, u32 ctx, u64 seq, u8 phase, s32 detail, u64 counter), + TP_ARGS(engine, ctx, seq, phase, detail, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u32, ctx) + __field(u64, seq) + __field(u8, phase) + __field(s32, detail) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->phase = phase; + __entry->detail = detail; + __entry->counter = counter; + ), + TP_printk("engine=%u ctx=%u seq=%llu phase=%u detail=%d counter=%llu", + __entry->engine, __entry->ctx, __entry->seq, __entry->phase, + __entry->detail, __entry->counter) +); + /* Raw 18-word records: trace=0, statistics=1, event=2. No unit inference. */ TRACE_EVENT(asahi_g17_report, TP_PROTO(u8 role, u8 ring, const u32 *words, u64 counter), diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index 1a99b41a80753b..78d88db4718c6b 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -106,3 +106,33 @@ pub(super) fn report(role: u8, ring: u8, words: &[u32; super::fw::channels::EVEN // SAFETY: C synchronously copies the complete initialized 18-word array. unsafe { asahi_g17_trace_report(role, ring, words.as_ptr()) }; } + + +unsafe extern "C" { + fn asahi_g17_trace_inputs(ctx: u32, seq: u64, phase: u8, inputs: u32, + sources: u32, waits: u32, converted: bool); + fn asahi_g17_trace_schedule(engine: u8, ctx: u32, seq: u64, phase: u8, detail: i32); +} + +/// Input conversion boundaries. No fence status is polled for tracing. +pub(super) fn inputs(ctx: u32, seq: u64, phase: u8, inputs: usize, + sources: Option<&super::sync_input::Sources>) +{ + let (count, waits) = sources.map_or((0, 0), + |s| (s.sources.len(), s.publication_waits().len())); + // SAFETY: Bounded UAPI counts and plain values only. C reads the timer only + // behind this event's static key; no GPU registers or pointers are exposed. + unsafe { asahi_g17_trace_inputs(ctx, seq, phase, inputs as u32, + count as u32, waits as u32, sources.is_some()) }; +} + +/// Host scheduling boundaries, joined to publish/complete by engine/context/sequence. +pub(super) fn schedule(packet: &super::job::Packet, phase: u8, detail: i32) { + let engine = match packet.command { + super::command::Validated::Render { .. } => RENDER, + super::command::Validated::Compute { .. } => COMPUTE, + }; + // SAFETY: Plain values only; counter sampling is static-key gated in C. + unsafe { asahi_g17_trace_schedule(engine, packet.context.id(), + packet.order.sequence, phase, detail) }; +} diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h index be5eff4c8161bf..cd4d360c55595a 100644 --- a/drivers/gpu/drm/asahi/g17/trace_api.h +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -22,6 +22,10 @@ void asahi_g17_trace_message(u8 kind, u64 value); void asahi_g17_trace_event(u8 phase); void asahi_g17_trace_teardown(u8 kind, u64 a, u64 b, u64 c); +void asahi_g17_trace_inputs(u32 ctx, u64 seq, u8 phase, u32 inputs, u32 sources, + u32 waits, bool converted); +void asahi_g17_trace_schedule(u8 engine, u32 ctx, u64 seq, u8 phase, s32 detail); + bool asahi_g17_trace_report_enabled(void); void asahi_g17_trace_report(u8 role, u8 ring, const u32 *words); From 0dd07fd2da65b1276a5d670e4ba50d0aba40fb75 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 27 Oct 2026 18:52:18 +1100 Subject: [PATCH 022/171] drm/asahi: bound the G17 tail cache separately from 25G83 auxiliary storage 25G83 packs its existing GTP sub-buffers after the tail-pointer cache. One of them is 64 times the tile map. The inherited 32 MiB TPC limit was applied to the combined object, so the valid 10240-square target used by CTS point_size_10000 is rejected with ERANGE even though the cache and tile record individually fit their original limits. Apply TPC_MAX to the aligned cache, and retain TARGET_RECORD_MAX before allocating or growing the object. Both bounds cap the derived auxiliary extent. Keep the encoder's buffer sizing, address checks, allocation accounting, retained generations and memory-pressure handling unchanged. No firmware field or numeric resource limit increases. The same preflight serves fresh and grown targets; Neo has no GTP tail and retains its policy. KUnit covers the CTS geometry for both utile shapes, oversized core and layered tile-record rejection, and unchanged Neo cache capacity. Hardware must confirm the point case; this does not diagnose the separate 256-square 1 MiB vertex/barrier fault or decode firmware reason 2. Link: https://github.com/KhronosGroup/VK-GL-CTS/blob/vulkan-cts-1.4.6.2/external/vulkancts/modules/vulkan/rasterization/vktRasterizationTests.cpp#L9384 Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit aecb91d831daac37c5260c2a2931eb4ce2cb31ad in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer/target.rs | 71 ++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index 64c88da8eca149..42a2c04b40112d 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -139,6 +139,16 @@ impl Layout { }) } pub(crate) fn tpc_capacity(&self) -> Result { + // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object + // also contains GTP sub-buffers, notably 64 times the tile map. Applying + // the cache limit to their combined extent rejects valid large targets. + // Keep the original cache and tile-record limits; they also bound the + // derived auxiliary extent before any backing allocation or replacement. + if aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN) > cfg::TPC_MAX + || self.record_stride()? > cfg::TARGET_RECORD_MAX + { + return Err(ERANGE); + } Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { @@ -188,7 +198,7 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + if stride > cfg::TARGET_RECORD_MAX { return Err(ERANGE); } let records = Object::new(dev, vm, stride * DEPTH)?; @@ -272,9 +282,6 @@ impl Target { let stride = layout.record_stride()?; let mut grew = false; if tpc > self.tpc.backing.size() { - if tpc > cfg::TPC_MAX { - return Err(ERANGE); - } Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; grew = true; } @@ -350,3 +357,59 @@ impl Target { } } } + + +#[kernel::macros::kunit_tests(asahi_g17_target_capacity)] +mod tests { + use super::*; + + fn pass(size: u32, utile: u8) -> RenderPass { + // SAFETY: RenderPass and its nested surfaces/programs contain only + // integers and bools, for which the all-zero representation is valid. + let mut pass: RenderPass = unsafe { core::mem::zeroed() }; + pass.width = size; + pass.height = size; + pass.layers = 1; + pass.utile_width = utile; + pass.utile_height = utile; + pass + } + + #[test] + fn t6040_auxiliary_tiler_storage_is_not_tail_pointer_cache() { + // CTS point_size_10000 renders to 10240 squared. Its GTP sub-buffer + // is required by the existing 25G83 program, not a larger cache. + for utile in [16, 32] { + let layout = Layout::new(&pass(10240, utile), 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.tpc_size < cfg::TPC_MAX); + assert!(layout.record_stride().unwrap() < cfg::TARGET_RECORD_MAX); + assert!(layout.tpc_end > cfg::TPC_MAX); + assert!(layout.tpc_capacity().unwrap() >= layout.tpc_end); + assert_eq!(layout.gtp[0], Some(0)); + assert!(layout.gtp[1].unwrap() >= layout.tpc_size); + } + } + + #[test] + fn core_and_tile_record_limits_still_reject_oversized_targets() { + let mut layout = Layout::new(&pass(10240, 32), 2, RegisterPrograms::V25G83).unwrap(); + layout.tpc_size = cfg::TPC_MAX + 1; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + let mut layered = pass(16384, 16); + layered.layers = 2; + let layout = Layout::new(&layered, 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.record_stride().unwrap() > cfg::TARGET_RECORD_MAX); + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } + + #[test] + fn neo_keeps_the_same_cache_extent_and_limit() { + let mut layout = Layout::new(&pass(10240, 32), 1, RegisterPrograms::G17P).unwrap(); + assert!(layout.gtp.iter().all(Option::is_none)); + assert_eq!(layout.tpc_capacity().unwrap(), + aligned(layout.tpc_size, mmu::UAT_PGSZ).unwrap().max(cfg::TPC_MIN)); + layout.tpc_size = cfg::TPC_MAX + 1; + layout.tpc_end = layout.tpc_size; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } +} From 7c43e0699638bb9db64fbcbfd31d77e2e8711bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Bal=C3=A1zsy?= Date: Mon, 26 Oct 2026 17:22:40 +0100 Subject: [PATCH 023/171] drm/asahi: admit a third loaded T6040 25G83 image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A J616s with 26.6.2 boot firmware (25G83, iBoot mBoot-18000.161.10) can load a third normalized T6040 TEXT identity: TEXT 0x60000, DATA 0x138000, the same entropy offsets, sha256 d725264d4831b95a5b8a375abf2fb5cdfbd6d7cbc7f1baa2cc8b37ee5d6e5e98. Without it the GPU probe stops with "GPU firmware image not admitted" and the device has no render node. Add the digest to the 26.6.2 T6040 image list next to the two existing identities and to the KUnit test that exercises every listed image through the lookup used by identify(). Signed-off-by: František Balázsy (cherry picked from commit d42eadabb9c4e4547768c427aae2e46a1a5a1443 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/fwid.rs | 7 ++++++- drivers/gpu/drm/asahi/hw/t6040.rs | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index bb1fd3c36f6233..be81eac5dfb743 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -213,7 +213,7 @@ mod tests { } #[test] - fn t6040_accepts_both_loaded_images() { + fn t6040_accepts_all_loaded_images() { let crate::g17::fw::variant::Firmware::Identified(builds) = crate::hw::t6040::CONFIG.firmware else { @@ -231,6 +231,11 @@ mod tests { 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, ], + [ + 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, 0x2f, + 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, 0x37, 0xee, + 0x5d, 0x6e, 0x5e, 0x98, + ], ]; for (i, digest) in digests.into_iter().enumerate() { let found = measured(build.images, 0x138000, [0x59839, 0x5991c], digest).unwrap(); diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index fa3e51efacadcb..11ed37a7c1ef80 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -85,6 +85,17 @@ pub(crate) const CONFIG: Config = Config { 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, ], }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, + 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, + 0x37, 0xee, 0x5d, 0x6e, 0x5e, 0x98, + ], + }, ], variant: &fw::variant::V25G83, }]), From 2a6f92cbc79f9635f60fe17ad3fea3d48f5f1ac3 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 26 Oct 2026 22:02:48 -0300 Subject: [PATCH 024/171] drm/asahi: g17: never report a busy fragment queue on 25G83 The 25G83 firmware reads the tiling descriptor's fragment_busy word (TA+0x7a4) only in its out-of-memory lane, at the first out-of-memory stop of a pass. When it is non-zero the firmware compares the pass-seq byte of the last completed fragment kick on that queue with the lane's copy of TA+0x7b0. On a mismatch it resumes the stopped TA with the resume-after-grow register write but without adding memory, and retires the lane: the tiler then stops silently with no further out-of-memory stop and no completion, and the forward-progress watchdog fires (reason 0). We set fragment_busy whenever the previous fragment kick had not completed at publish, which is common in a pipelined stream. macOS always writes 0 there on this firmware. Add Variant::fragment_busy and clear it for V25G83, so the out-of-memory lane takes the path macOS takes. Renders that fit their tiler heap never reach that lane, so they are unaffected. V26A428 keeps its current behaviour. The firmware behaviour above is from a static disassembly of the 25G83 firmware and the macOS AGXG16X kext. In our testing on a J616s (T6040, 25G83), with the tiler heap at its tile-count size so ordinary heavy passes must grow, this alone took dEQP-VK.memory.pipeline_barrier.graphics.all_device.1048576_vertex_buffer_stride_2 from 7 firmware recoveries over three runs to 4; the next change removes the rest. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 763b4dd1e7c1d1a5e1e330b974d20c481da51d0f in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/variant.rs | 7 +++++++ drivers/gpu/drm/asahi/g17/queue/render.rs | 8 ++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 78951f6d10a7e2..38a14f4bebca77 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -57,6 +57,9 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Tiling descriptors report whether the fragment queue still had earlier kicks + /// outstanding (`fragment_busy`). + pub(crate) fragment_busy: bool, /// Render item rings carry lead commands: InitBufferManager before the first tiling pass /// of a buffer manager, and FragmentBarrier before every fragment descriptor. pub(crate) lead_commands: bool, @@ -99,6 +102,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, }; @@ -124,6 +128,9 @@ pub(crate) const V25G83: Variant = Variant { few_primitives: false, // A stalled ray traversal wedges the GPU without a firmware recovery. ray_traversal: false, + // When a tiling pass runs out of memory with this set, the firmware resumes it without + // adding memory and the tiler stops silently; macOS always writes 0. + fragment_busy: false, lead_commands: true, programs: super::render::RegisterPrograms::V25G83, }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index f16e682b1e8fc5..768b5440468a73 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -587,8 +587,12 @@ impl Pair { let stamps = host.stamps(qids)?; fence(Ordering::Acquire); let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory - .completion_links(ordinal, kicks, payload, !covered[1])?; + self.memory.completion_links( + ordinal, + kicks, + payload, + self.variant.fragment_busy && !covered[1], + )?; let leads = match self.variant.lead_commands { true => { let init_manager = !self.memory.manager.initialized(); From 9cf3377123e325c1c03f53382f46f453f3bde075 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 26 Oct 2026 22:03:56 -0300 Subject: [PATCH 025/171] drm/asahi: g17: drop the 25G83 fragment barrier Commit 0d9efa515 ("drm/asahi: g17: render lead commands for the 25G83 firmware") led every 25G83 fragment descriptor with a FragmentBarrier (tag 4) that waits for the paired tiling stamp. macOS does not send that item: its fragment submission stores item tags 1, 0xc, 0xe, 0xf and 0x10 only, and keeps fragment-after-tiling ordering in the kick entry's dependency 0, which we already set. The barrier deadlocks a pass that runs out of tiler memory. The firmware's item-ring loop stops at an item whose handler returns 0, and the tag-4 handler returns 0 until the tiling stamp is reached, so the fragment kick's AddKicks is only processed after its tiling pass completes. A partial render needs that fragment kick announced while the tiling pass is stopped (the firmware releases it in out-of-memory state 7 by clearing the kick entry's dependency 0), so the tiling pass waits for the partial render and the partial render waits for the tiling pass. Remove FragmentBarrier and write only InitBufferManager as a lead command. V26A428 has no lead commands and is unaffected. The firmware behaviour above is from a static disassembly of the 25G83 firmware and the macOS AGXG16X kext. On a J616s (T6040, 25G83), with the tiler heap left at its tile-count size so ordinary heavy passes must grow, this and the previous change take dEQP-VK.memory.pipeline_barrier.graphics.all_device.1048576_vertex_buffer_stride_2 from 7 firmware recoveries over three runs to none. A 1/2000 vk-default slice is 769 pass / 0 fail / 0 crash, and the desktop and GL clients render as before. Fixes: 0d9efa515 ("drm/asahi: g17: render lead commands for the 25G83 firmware") Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit ad14f2da457be86a30e212852f23d8e18166f6d7 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/queue.rs | 59 ------------------- drivers/gpu/drm/asahi/g17/fw/variant.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 2 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 36 +++-------- 4 files changed, 10 insertions(+), 89 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 5708a2251a062d..66e7d07db48a52 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -96,8 +96,6 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, - /// [`FragmentBarrier`] (25G83 firmware). - FragmentBarrier = 4, /// [`InitBufferManager`] (25G83 firmware). InitBufferManager = 5, } @@ -569,63 +567,6 @@ impl QueueConfig { } } -/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work -/// waits for the paired tiling stamp, in addition to the kick-entry dependency. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FragmentBarrier { - /// [`CommandTag::FragmentBarrier`]. - pub(crate) tag: u32, - /// GPU address of the tiling queue's completion stamp word. - pub(crate) stamp_va: u64, - /// Timestamp of the tiling kick. - pub(crate) tiling_kick: u64, - /// QID of the tiling queue. - pub(crate) tiling_qid: u32, - /// Tiling stamp value waited for. - pub(crate) wait: u32, - pub(crate) unk_1c: [u8; 8], - /// Event of the wait: the tiling QID. - pub(crate) event: u32, - /// Fragment stamp value. - pub(crate) stamp_self: u32, - /// Current object ID of the fragment stage. - pub(crate) fragment_id: u64, - pub(crate) unk_34: u32, - pub(crate) unk_38: [u8; 0x10], -} - -static_assert!(size_of::() == 0x48); -static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); -static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); -static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); - -impl FragmentBarrier { - /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, - /// signals `stamp_value` at `stamp_va`. - pub(crate) fn new( - stamp_va: u64, - tiling_kick: KickTimestamp, - tiling_qid: u32, - stamp_value: u32, - fragment_id: u64, - ) -> Self { - Self { - tag: CommandTag::FragmentBarrier as u32, - stamp_va, - tiling_kick: tiling_kick.get(), - tiling_qid, - wait: stamp_value, - unk_1c: [0; 8], - event: tiling_qid, - stamp_self: stamp_value, - fragment_id, - unk_34: 1, - unk_38: [0; 0x10], - } - } -} - /// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling /// descriptor that uses a parameter buffer. The block count is read from the manager. #[repr(C, packed)] diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 38a14f4bebca77..39c8cfcd77a2a9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -61,7 +61,7 @@ pub(crate) struct Variant { /// outstanding (`fragment_busy`). pub(crate) fragment_busy: bool, /// Render item rings carry lead commands: InitBufferManager before the first tiling pass - /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + /// of a buffer manager. pub(crate) lead_commands: bool, /// Register programs of the render descriptors. pub(crate) programs: super::render::RegisterPrograms, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 768b5440468a73..6fb4e233de8d60 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -596,7 +596,7 @@ impl Pair { let leads = match self.variant.lead_commands { true => { let init_manager = !self.memory.manager.initialized(); - self.memory.write_leads(ordinal, kicks[0], init_manager)? + self.memory.write_leads(ordinal, init_manager)? } false => [false; 2], }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 884537b99b6d83..90a11ca977684e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,8 +31,8 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; -/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the -/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +/// The lead command of a stage (InitBM before tiling), after the stage's configuration record +/// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. @@ -534,7 +534,6 @@ const _: () = { assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); assert!(size_of::() <= LEAD); - assert!(LEAD + size_of::() <= CONFIG_STRIDE); assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; @@ -648,14 +647,8 @@ impl Memory { } /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the - /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages - /// have one. - pub(super) fn write_leads( - &mut self, - ordinal: u64, - tiling_kick: fw::kick::KickTimestamp, - init_manager: bool, - ) -> Result<[bool; 2]> { + /// tiling descriptor when `init_manager`. Returns which stages have one. + pub(super) fn write_leads(&mut self, ordinal: u64, init_manager: bool) -> Result<[bool; 2]> { if !self.owns(ordinal) { return Err(EIO); } @@ -664,23 +657,10 @@ impl Memory { .descriptors .pointer(Graph::offset(0, ordinal), size_of::())? .cast::(); - let fragment = self - .graph - .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? - .cast::(); - // SAFETY: Both views were checked during preparation and the ordinal still owns them; - // only integer fields are read. - let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + // SAFETY: The view was checked during preparation and the ordinal still owns it; only + // integer fields are read. + let ta = unsafe { &*ta }; let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; - let barrier = queue::FragmentBarrier::new( - ta.stamp_va, - tiling_kick, - ta.qid, - ta.stamp_value, - fragment_id, - ); - self.graph.config.write(CONFIG[1] + slot, barrier)?; if init_manager { let init = queue::InitBufferManager::new( ta.context_id, @@ -690,7 +670,7 @@ impl Memory { ); self.graph.config.write(CONFIG[0] + slot, init)?; } - Ok([init_manager, true]) + Ok([init_manager, false]) } } From 926df5c70e81c121a873c2da7c78632acb412088 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 09:34:19 -0400 Subject: [PATCH 026/171] drm/asahi: retire a G17 render group on its pass record when its stamps stall A render pair is observed only once both queues' completion stamps cover its front group. On the J700, about every other boot under a CTS load, the stamps of one pair stop one kick short of a group the firmware has executed: the pass record holds its tiler start and fragment end, both rings are consumed, and later groups on the pair execute too, but no stamp store, completion notification or later event moves the stamps. The pair is never observed again, and after the scheduler watchdog and its renewals (about 30 s) the client's render fails with ETIMEDOUT and it sees a lost device. Give render the bound compute already has for its records: - When an observation scan finds a front group executed and consumed while the stamps lag it, it latches the time. The scan before a render watchdog attribution also latches, so a pair whose notification never came is found within one watchdog period. - A latched pair stays in the 1 ms rescan without a notification. After STAMP_FALLBACK_NS (50 ms) it is observed and the group retires on its pass record. - When the watchdog of such a front group itself expires, the group retires on its record instead of failing; failing would have given up on the stamps as well. The first and every 64th such retirement are logged with a lifetime count. (cherry picked from commit fa91d949c3dd837816fd5e0c07a88efa905d90c2 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 67 ++++++++++++- .../drm/asahi/g17/queue/render/retirement.rs | 96 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 17 +++- .../drm/asahi/g17/runtime/render/observe.rs | 14 ++- .../drm/asahi/g17/runtime/render/settle.rs | 14 +++ 5 files changed, 203 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 6fb4e233de8d60..2254118f181401 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -142,6 +142,8 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, + /// First observation of this group executed and consumed while its stamps lagged. + stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -816,6 +818,7 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, + stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -1056,6 +1059,56 @@ impl Pair { } } + /// Latches when the front group executed and the firmware consumed its ring entries while + /// `stamps` still lag it, and reports whether that has held for + /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it + /// on its pass record instead of waiting for stamps no later event will move. + pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result { + if self.quarantined || self.prepared.is_some() || state != 0 { + return Ok(false); + } + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) { + front.stamp_wait_since = None; + return Ok(false); + } + let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; + if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { + front.stamp_wait_since = None; + return Ok(false); + } + let since = *front.stamp_wait_since.get_or_insert(now); + Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS) + } + + /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair + /// without a completion notification, which may never come for such a group. + pub(crate) fn stamp_waiting(&self) -> bool { + self.active.front().is_some_and(|front| front.stamp_wait_since.is_some()) + } + + /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed + /// it and consumed its ring entries while the stamps lag, it retires on its pass record now + /// instead of failing. Failing would give up on the stamps as well. + pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc, stamps: [u32; 2], state: u32) -> Result { + if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) { + return Ok(false); + } + if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { + return Ok(false); + } + if let Some(front) = self.active.front_mut() { + front.stamp_wait_since = Some(0); + } + Ok(true) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } @@ -1269,6 +1322,9 @@ impl Pair { { return Err(EIO); } + let stampless = front.stamp_wait_since.is_some_and(|since| { + crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS + }); let mut observer = Observer { memory: &self.memory, qids, @@ -1278,7 +1334,13 @@ impl Pair { if observed.ticket != front.ticket { return Err(EIO); } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) + } else { + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } + } else if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) } else { retirement::poll(&mut front.ticket, &mut observer) }; @@ -1293,6 +1355,9 @@ impl Pair { }; let ordinal = front.ticket.ordinal; let context_update = front.context_update; + if stampless { + retirement::note_stampless(u16::from(self.slot), ordinal); + } self.note_retirement(ordinal, context_update); let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 7996970ee369ff..9332d11a7f4b22 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -155,6 +155,49 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result { + fence(Ordering::Acquire); + let snapshot = memory.snapshot(ticket.ordinal)?; + let Ok(observation) = observe(ticket, snapshot) else { + return Ok(Poll::Retry); + }; + if !observation.complete { + return Ok(Poll::Pending); + } + let [start, end] = memory.pass(ticket.ordinal)?; + Ok(Poll::Complete([start, start, start, end])) +} + /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -382,4 +425,57 @@ mod executed_tests { assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); } + + /// The firmware state of a stalled pair as read on hardware: the front group executed and + /// its two ring entries per queue were consumed, but both stamps hold the previous kick. + struct Stalled { + snapshot: Snapshot, + stamps: [u32; 2], + } + impl Memory for Stalled { + fn snapshot(&mut self, _ordinal: u64) -> Result { + Ok(self.snapshot) + } + fn stamps(&mut self) -> Result<[u32; 2]> { + Ok(self.stamps) + } + fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { + Ok(self.snapshot.pass) + } + } + + fn stalled_ticket() -> Ticket { + let window = Window::new(1097, 2, 1280).unwrap(); + let previous = Cursors { done: 1097, read: 1097, write: 1097 }; + Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } + } + + fn stalled(pass: [u64; 2]) -> Stalled { + let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; + Stalled { + snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, + stamps: [406 << 8; 2], + } + } + + #[test] + fn lagging_stamps_keep_an_executed_group_pending() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stampless_poll_retires_an_executed_consumed_group() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); + } + + #[test] + fn stampless_poll_waits_for_the_fragment_end() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 0]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4e7eaa2d8d2468..6657f83e9e738a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -798,6 +798,19 @@ impl crate::g17::Firmware { { continue; } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; + } self.retire_render_slot(slot, None, deferred)?; self.queues.render.collect_retired(slot)?; self.settle_render_terminal(slot, deferred)?; @@ -1057,10 +1070,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && crate::g17::queue::render::completion_noted( + && (crate::g17::queue::render::completion_noted( entry.reservation.ids.map(kick::Id::qid), entry.event_masks, - ) + ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) }) } pub(in crate::g17) fn defer_render_timeout( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 51682b43728c05..57543be54e8356 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -10,7 +10,7 @@ use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { /// The first slot from `start` whose pair has news to observe: published work in flight, the /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. - fn next_render_observation(&self, start: usize) -> Result> { + fn next_render_observation(&mut self, start: usize) -> Result> { if self.recovery.pending() { return Ok(None); } @@ -27,7 +27,8 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || !crate::g17::queue::render::completion_noted(qids, entry.event_masks) + || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) + && !pair.stamp_waiting()) { continue; } @@ -39,6 +40,15 @@ impl crate::g17::Firmware { if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { return Ok(Some(slot)); } + // The stamps gate this pair out. If its front executed and was consumed anyway, the + // stamps may never move: after the bound, observe it and retire it on its record. + let pair = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { + return Ok(Some(slot)); + } } Ok(None) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 24c724cb32470b..71f0bfc0526968 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -79,6 +79,20 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if error == ETIMEDOUT { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + pair.expire_stamp_wait(packet, stamps, state)?; + } + } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event) From 7f9213abf80afe0b51b002ea1d5a19a16c5984ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 14:23:35 -0400 Subject: [PATCH 027/171] drm/asahi: g17: drop the completion thread's frequency boost while the GPU is the bottleneck Every G17 completion arrives through the primary coprocessor's mailbox, whose receive thread is an RT thread. With the RT default minimum utilization, schedutil raises its CPU cluster to the top frequency for each completion. While the GPU itself is the bottleneck that burst gains nothing. On J700, during GPU-bound work the P-cluster sits at its top frequency in about half of 1 s samples although the CPUs are about 3% busy, and once the SoC reaches its thermal limit (within about 40 s of heavy work) that power comes out of the GPU's clock. Have the feed detector also recognize when the GPU is the bottleneck: at least eight passes in a row whose refill gaps stay under a quarter of their busy time, the complement of its host-refill-bound test, lingering 8 ms like that state. While it holds, the port asks for a minimum utilization of 256 for the receive thread, and for the scheduler's default otherwise. The port re-evaluates at every publication and completion through a lock-free RTKit handle, so after a pause the publication that ends the state restores the default before its completion arrives. There is no new work item or timer. Lowering the boost whenever the feed is not streaming is not enough. The completion that ends a short pause then runs slower too: idle.wake with 2 ms gaps took 5% longer even at a floor of 512. A floor of 0 instead of 256 costs single-submit heavy compute about one completion per 250 ms. Measured on J700 with the panel off, in one boot after a 45 s heat-up, the old and new behaviour strictly alternated five times (gpubench, microseconds per submit, mean of the per-round medians; lower is better): render.heavy fill 2160p d1 2817 -> 2723 -3.3% render.heavy fill 2160p d16 2655 -> 2577 -2.9% render.heavy fill 1080p d1 802 -> 780 -2.7% compute.heavy alu d16 1199 -> 1179 -1.7% compute.heavy alu d1 1374 -> 1375 (one window in five) idle.wake 2 ms / 20 ms 191 / 588 -> 191 / 590 (medians) render.light d1/d32, compute.light d1/d8, compute.light queues d2, sync.signal binary queues within run-to-run noise On the heavy rows the P-cluster's mean frequency drops from 2.5 to 1.0 GHz and system power by 0.4-0.5 W. A KUnit suite covers the detector's three outcomes. (cherry picked from commit d266c703c13f099068284d622d397292241a0224 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/feed.rs | 73 ++++++++++++++++++++++++++++++- drivers/gpu/drm/asahi/g17/mod.rs | 15 ++++++- drivers/gpu/drm/asahi/g17/port.rs | 56 +++++++++++++++++++++--- 3 files changed, 137 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 760f352046192b..bd1228a82e26f2 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,6 +44,8 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, + /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). + bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -57,6 +59,7 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), + bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -108,17 +111,26 @@ impl Detector { Err(_) => 0, } }; - if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + if streak < STREAK_ON { + return; + } + if idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } + } else { + self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } + + fn bound(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) + } } #[cfg(CONFIG_CPU_FREQ)] @@ -416,6 +428,17 @@ impl Feed { } } + /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps + /// under a quarter of their busy time, and the feed does not see host-refill-bound work. + /// Lingers like streaming, so a pause of more than a few milliseconds ends it. + pub(crate) fn gpu_bound(&self) -> bool { + if !cfg!(CONFIG_CPU_FREQ) { + return false; + } + let now = Self::now(); + self.detector.bound(now) && !self.detector.wanted(now) + } + /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -498,3 +521,51 @@ impl PinnedDrop for Feed { self.shutdown(); } } + +#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] +mod detector_tests { + use super::*; + + /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. + fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { + let mut start = 1000; + let mut now = 1_000_000; + for _ in 0..passes { + detector.note(start, start + busy, now); + start += busy + gap; + now += 1_000_000; + } + now + } + + #[test] + fn back_to_back_long_passes_are_gpu_bound() { + let detector = Detector::new(); + // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. + let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); + assert!(detector.bound(now)); + assert!(!detector.wanted(now)); + assert!(!detector.bound(now + LINGER_NS)); + } + + #[test] + fn refill_gaps_stream_instead() { + let detector = Detector::new(); + // 20 us passes with 60 us refill gaps: bound by host refills. + let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); + assert!(detector.wanted(now)); + assert!(!detector.bound(now)); + } + + #[test] + fn paced_or_short_runs_are_neither() { + let detector = Detector::new(); + // Each pass starts after a pause of at least the refill limit. + let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); + assert!(!detector.bound(now) && !detector.wanted(now)); + // Fewer back-to-back passes than the streak needs. + let detector = Detector::new(); + let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); + assert!(!detector.bound(now) && !detector.wanted(now)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c2cea7bd07db9a..2a1f19a0b63c88 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -440,6 +440,16 @@ impl Mailbox { fn send(&self, endpoint: u8, message: u64) -> Result { self.with(|rtkit| rtkit.send_message(endpoint, message)) } + + /// A handle on the receive thread's minimum utilization, usable from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this mailbox is dropped. + unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { + // SAFETY: The caller's obligation is the handle's. + self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) + } } /// A running firmware coprocessor. @@ -1386,7 +1396,10 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new(power::Demand::new(pdev.as_ref().as_raw())?)?, + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, seen_submissions: 0, rings: channel::Rings::new()?, recovery: recovery::State::new(), diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 4c2c320d4089b0..6d0e87c14b9477 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -22,11 +22,20 @@ use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; use kernel::{ new_mutex, new_spinlock, prelude::*, + soc::apple::rtkit, sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, }; const PRIORITIES: usize = Policy::PRIORITIES as usize; +/// Minimum utilization of the primary coprocessor's receive thread, which handles every +/// completion, while the GPU itself is the bottleneck (see `Feed::gpu_bound`). Otherwise the +/// thread keeps the scheduler's RT default, which takes its CPU cluster to the top frequency for +/// each completion: the completion may be on the critical path, of a waiter, of an idle GPU or of +/// work bound by host refills. While the GPU is the bottleneck the burst gains nothing, and at the +/// SoC's thermal limit the power it takes comes out of the GPU's clock. +const RX_UTIL_MIN_GPU_BOUND: u32 = 256; + /// One priority class's compute work ring: its slot array and the cursors of its two firmware /// consumers and of the host producer. #[derive(Clone, Copy)] @@ -89,6 +98,12 @@ pub(super) struct Port { rings: SpinLock<[u32; PRIORITIES]>, #[pin] qos: SpinLock, + /// Sets the completion receive thread's minimum utilization, from `feed`'s view of the GPU. + rx: SetOnce, + feed: Arc, + /// The request last made, `None` before the first. + #[pin] + rx_applied: SpinLock>>, /// Serializes every render publication, with or without the device mutex: the tiling /// and fragment work rings have one host producer at a time, and same-VM attachment /// overlaps are ordered by publication order (see hazard.rs). @@ -139,13 +154,19 @@ impl Drop for OuterClaim<'_> { } impl Port { - pub(super) fn new(demand: super::power::Demand) -> Result> { + pub(super) fn new( + demand: super::power::Demand, + feed: Arc, + ) -> Result> { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), regs: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + rx: SetOnce::new(), + feed, + rx_applied <- new_spinlock!(None, "g17::Port::rx_applied"), render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), @@ -160,7 +181,10 @@ impl Port { /// Called once while the session is built, before any publisher can see the port. pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { - if self.views.populate(views) && self.regs.populate(regs) { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port, and + // with it every use of the handle. + let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; + if self.views.populate(views) && self.regs.populate(regs) && self.rx.populate(rx) { Ok(()) } else { Err(EBUSY) @@ -361,7 +385,9 @@ impl Port { policy: Policy, ) -> Result { let view = self.qos_view()?; - self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?; + self.rx_update(); + Ok(publication) } /// Publishes the two owners of a render command, fragment then tiling. @@ -372,7 +398,10 @@ impl Port { policy: Policy, ) -> Result<[qos::Publication; 2]> { let view = self.qos_view()?; - self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + let publications = + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?; + self.rx_update(); + Ok(publications) } pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { @@ -382,7 +411,24 @@ impl Port { pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { let view = self.qos_view()?; - self.qos.lock().complete(&view, owners) + self.qos.lock().complete(&view, owners)?; + self.rx_update(); + Ok(()) + } + + /// Requests the receive thread's minimum utilization for the feed's current view of the GPU. + /// Called at every publication and completion: the feed's state lingers for a few + /// milliseconds, so after a pause the publication that ends it restores the default before + /// its completion arrives. + fn rx_update(&self) { + let Some(rx) = self.rx.as_ref() else { + return; + }; + let want = self.feed.gpu_bound().then_some(RX_UTIL_MIN_GPU_BOUND); + let mut applied = self.rx_applied.lock(); + if *applied != Some(want) && rx.set(want).is_ok() { + *applied = Some(want); + } } /// Rings the primary coprocessor's doorbell. From 622f15333f41f263039f155c75589e1ed377e572 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 29 Oct 2026 00:34:17 -0400 Subject: [PATCH 028/171] drm/asahi: g17: say what the stampless render retirement relies on A render group whose completion stamps stall retires on its pass record after STAMP_FALLBACK_NS. The comment said the stamps follow the cache maintenance after a pass, but not that this path has no such evidence: whether the firmware completes the maintenance before it advances the done cursors is not known. Say that a group retired this way relies on the 50 ms bound for its counts, stores and exported timestamps to be visible. Comment only. Signed-off-by: DjDeveloperr (cherry picked from commit 4176f537a7cbfefe1540d6e2d832fdb535939cc7 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render/retirement.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 9332d11a7f4b22..a62d291ea3d921 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -163,6 +163,11 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result { fence(Ordering::Acquire); let snapshot = memory.snapshot(ticket.ordinal)?; From 9b2bbe01f77804fabb9358126e6ad459bf730ede Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 29 Oct 2026 07:39:19 -0400 Subject: [PATCH 029/171] drm/asahi: g17: count the stamp fallback from the drain of the command's node The firmware stores the completion stamps of an ioctl's commands only once every kick counted on the ioctl's work-state node has completed: a node's completion word reaches the host's submitted count, and only then do the compute and render stamps move. A command whose record or pass record is present therefore waits for its stamp for as long as the rest of its ioctl runs. A command buffer of a dozen render passes and their copies, or a compute command beside a render that waits on another queue, keeps the stamps of its early members back for tens to hundreds of milliseconds. The 50 ms stampless fallback measured from the record, so it retired those members on their records while the firmware was still running their node: their fences, their VM pins and their render leases were released before the post-maintenance stamp the retirement path otherwise requires. On the Neo, one CTS synchronization2 pass retired about 9.8k compute commands this way. Each of them had kicks of its own node still running (the node's completion word below the host count), and the stamp covered it 4 to 300 ms later (median 28 ms). Count the bound from the drain instead. Once it has passed, a compute command or a render group whose stamps lag asks its node whether every counted kick completed; if not, the stamps are not late and the wait starts again. A drained node whose stamps still lag keeps the existing fallback, so a group whose stamps stop one kick short still retires on its pass record, and the watchdog's expiry still retires an executed group without waiting for the node. A node that cannot be read keeps the old bound. Signed-off-by: DjDeveloperr (cherry picked from commit effb9e00034d83ceb6ac483ee448826d87bc4526 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/context.rs | 23 ++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 17 +++++- .../drm/asahi/g17/queue/compute/retirement.rs | 56 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 31 ++++++++-- .../drm/asahi/g17/queue/render/retirement.rs | 56 +++++++++++++++++-- 5 files changed, 167 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 0262ea10853983..8a6d9b572dec09 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -82,6 +82,19 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } + /// Whether the firmware completed every kick counted so far on node `word`. + fn drained(&self, word: u32) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let completed = self + .object + .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? + .load(Ordering::Relaxed); + fence(Ordering::Acquire); + Ok(completed >= self.counts[slot]) + } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -476,6 +489,16 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } + /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores + /// the completion stamps of the node's commands only then, so a command whose record is + /// present still waits for its stamp while other commands of its ioctl run. + pub(crate) fn drained(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + self.context.work.lock().drained(word) + } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0ec5b1797c011a..7ff828b1e36023 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -964,6 +964,7 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { + let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -983,7 +984,18 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = retirement::record_aged(front.observed_at, now_ns()); + let record_aged = !retirement::stamp_covers(stamp, ticket.kick) + && match retirement::stamp_wait(front.observed_at, now_ns(), || { + // A node that cannot be read keeps the bound from the record. + front.work.drained().unwrap_or(true) + }) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + restart = Some(now); + false + } + retirement::StampWait::Aged => true, + }; let timestamps = retirement::poll( &ticket, front.observed, @@ -1039,6 +1051,9 @@ impl Queue { } } Ok(None) | Err(EAGAIN) => { + if restart.is_some() { + self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; + } self.retire_pending = self.front_finished(); break; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 6b942dcd86a3e9..d7f821bbce68d1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -72,15 +72,16 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. +/// before it retires on the record alone, counted from when its ioctl's work-state node drained. /// /// The firmware stores the stamp after the cache maintenance that follows the record, so the /// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. While other commands of the same submit ioctl are still running, the stamp -/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms -/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as -/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail -/// its VM. +/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on +/// the ioctl's work-state node has completed: while other commands of the same submit ioctl +/// still run, the stamp is not late, however long they run (a command buffer of a dozen render +/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the +/// node drains. Only a stamp still missing well past that is treated as never stored. The bound +/// stays far below the scheduler timeout, so such a command cannot fail its VM. pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; // A command that waits out the fallback must still retire well inside the scheduler timeout. @@ -92,6 +93,33 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) } +/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StampWait { + /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. + Waiting, + /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not + /// late, and the wait counts again from the given host time. + Restart(u64), + /// The node drained and the stamp is still missing after the bound: retire on the record. + Aged, +} + +/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and +/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the +/// command's work-state node completed; it is asked only once the bound has passed. +pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) + -> StampWait +{ + if !record_aged(observed_at, now) { + StampWait::Waiting + } else if drained() { + StampWait::Aged + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); pub(crate) fn stampless_total() -> u64 { @@ -538,6 +566,22 @@ mod tests { assert_eq!(result, Ok(Some([0x1111, 0x2222]))); } + #[test] + fn a_stamp_waits_for_the_rest_of_its_ioctl() { + let at = 1_000_000; + let late = at + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); + assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), + StampWait::Waiting); + // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. + assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // The node drained and the stamp is still missing after the bound. + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); + assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); + } + #[test] fn an_aged_record_still_waits_for_consumed_items() { let record = Some(Observation { timestamps: [10, 20] }); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2254118f181401..540f7cc76e8e75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1084,7 +1084,30 @@ impl Pair { return Ok(false); } let since = *front.stamp_wait_since.get_or_insert(now); - Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS) + Ok(Self::stamp_wait_expired(front, since, now)) + } + + /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks + /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair + /// stays latched for the rescan. + fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { + let drained = || { + // A node that cannot be read keeps the bound from the latch. + front + .packet + .completion + .work_state() + .and_then(|work| work.drained()) + .unwrap_or(true) + }; + match retirement::stamp_wait(since, now, drained) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + front.stamp_wait_since = Some(now); + false + } + retirement::StampWait::Expired => true, + } } /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair @@ -1322,9 +1345,9 @@ impl Pair { { return Err(EIO); } - let stampless = front.stamp_wait_since.is_some_and(|since| { - crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS - }); + let stampless = front + .stamp_wait_since + .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); let mut observer = Observer { memory: &self.memory, qids, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index a62d291ea3d921..4377bbfe3e311d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -156,13 +156,17 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result bool) -> StampWait { + if now.saturating_sub(since) < STAMP_FALLBACK_NS { + StampWait::Waiting + } else if since == 0 || drained() { + StampWait::Expired + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); /// Counts a render group that retired on its pass record because its completion stamps never @@ -484,4 +514,20 @@ mod executed_tests { let mut memory = stalled([10, 0]); assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); } + + #[test] + fn stamps_wait_for_the_rest_of_the_ioctl() { + let since = 1_000_000; + let late = since + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); + // Other kicks of the group's ioctl still run: the bound restarts. + assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // A drained node with lagging stamps is the stall the bound exists for. + assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); + // The watchdog's last resort does not wait for the node. + assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), + StampWait::Expired); + } } From 02015c451dd10b9893d1a3502a35e83e87a0f4dc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:14:21 +1100 Subject: [PATCH 030/171] drm/asahi: Format the client usage accounting paths Apply rustfmt to existing files that the per-client engine accounting change touches. Keep the mechanical formatting separate from the accounting and firmware timestamp changes for review. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: took aurora-silicon's version of g17/mod.rs to resolve a conflict; the module order in asahi_core.rs keeps integration/aurora's G15 adoption module and has no G16] (cherry picked from commit 137cfcf98ebd5aa432856b8743f96a9df2762a28 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/asahi_core.rs | 8 +- drivers/gpu/drm/asahi/file.rs | 75 ++++++++++------ drivers/gpu/drm/asahi/g15/mod.rs | 1 - drivers/gpu/drm/asahi/g17/completion.rs | 50 ++++++++--- drivers/gpu/drm/asahi/g17/mod.rs | 112 +++++++++++++++--------- drivers/gpu/drm/asahi/g17/queue/mod.rs | 106 ++++++++++++++-------- drivers/gpu/drm/asahi/queue/mod.rs | 21 ++++- rust/kernel/drm/file.rs | 4 +- 8 files changed, 252 insertions(+), 125 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 1d14ac3692a460..a2662964a5edea 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -12,22 +12,22 @@ mod driver; mod event; mod file; mod float; -mod fwid; mod fw; -mod gem; -mod g17; +mod fwid; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; // Without the G15 backend, only the UAT adoption the T8122 runtime shares with it. #[cfg(all(CONFIG_DRM_ASAHI_UAT_ADOPTION, not(CONFIG_DRM_ASAHI_G15)))] #[path = "g15/adopt.rs"] mod g15; +mod g17; +mod gem; mod gpu; mod hw; mod initdata; +mod m3; mod mem; mod memacct; -mod m3; mod microseq; mod mmu; mod object; diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 0c4c6048408262..12463c486cf16e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, - memacct, gpu, + memacct, mmu, queue, util::{ @@ -132,11 +132,16 @@ pub(crate) struct SyncItem { } impl SyncItem { - fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, - bundled: bool) -> Result - { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; } match data.sync_type { @@ -230,12 +235,16 @@ impl SyncItem { } anchor = Some(i as usize); } - vec.push(Self::parse_one(file, sync, i >= in_count, - allow_cumulative)?, GFP_KERNEL)?; + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; } if let Some(anchor) = anchor { let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], anchor - in_count as usize)?; + &vec[in_count as usize..], + anchor - in_count as usize, + )?; vec[anchor].cumulative = Some(bundle); } else { for sync in &mut vec[in_count as usize..] { @@ -713,22 +722,28 @@ impl File { }; let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; // Large G17 user objects are populated in full by the first VM_BIND. // Do not multiply direct reclaim/compaction latency by their folio // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = gem::user_pages_no_reclaim( - size, - reg_data.gpu.as_any().is::(), - ); + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); let gem = gem::new_object( - device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, )?; if reg_data.gpu.prepare_new_objects() { gem::prepare_new_object(&gem)?; @@ -1306,7 +1321,10 @@ impl File { if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH && !reg_data.gpu.as_any().is::() { - cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); return Err(EINVAL); } // SAFETY: capable() only reads the calling task's credentials (ioctl process context). @@ -1314,7 +1332,10 @@ impl File { && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } && !file.is_current_master() { - cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); return Err(EACCES); } @@ -1425,9 +1446,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, - gpu.as_any().is::())?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index de1dfa898eaae2..6bbdcf037620d9 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -223,7 +223,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 02e7bbc5c33dc8..a232b9c9eb2ccc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,12 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; +use crate::{ + driver::AsahiDrmDevice, + file, + hw::t8140::{dynamic, resources as cfg}, + mmu, +}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -86,7 +91,9 @@ impl Destinations { Ok(Self(destinations)) } - pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } + pub(crate) fn has_outputs(&self) -> bool { + self.0.iter().any(Option::is_some) + } fn iter(&self) -> core::slice::Iter<'_, Option> { self.0.iter() @@ -287,14 +294,17 @@ impl Completion { sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { fence, member, destinations }; + let output = Output { + fence, + member, + destinations, + }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { @@ -334,7 +344,9 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal + .as_ref() + .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -382,11 +394,15 @@ impl Completion { } match self.check_source_inputs() { Ok(()) => self.output.output().member.note_retired(), - Err(error) => { self.note_input_error(error); } + Err(error) => { + self.note_input_error(error); + } } } pub(super) fn check_source_inputs(&self) -> Result { - if let Some(sources) = &self.sources { sources.check()?; } + if let Some(sources) = &self.sources { + sources.check()?; + } Ok(()) } @@ -394,7 +410,11 @@ impl Completion { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) + self.output + .output() + .destinations + .iter() + .any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -597,7 +617,9 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -607,7 +629,9 @@ impl Completion { } fn release_healthy(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -617,7 +641,9 @@ impl Completion { #[track_caller] pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { if result.is_ok() { - if let Err(error) = self.check_source_inputs() { result = Err(error); } + if let Err(error) = self.check_source_inputs() { + result = Err(error); + } } if self.signalled.swap(true, Ordering::AcqRel) { return; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2a1f19a0b63c88..ae5bbd61a3ce4e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -31,31 +31,31 @@ mod completion; pub(crate) mod context; mod data_segment; mod dependency; -#[cfg(CONFIG_DRM_ASAHI_G15)] -pub(crate) use dependency::inherited_error; mod sync_input; pub(crate) mod sync_output; mod trace; +#[cfg(CONFIG_DRM_ASAHI_G15)] +pub(crate) use dependency::inherited_error; +mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -mod ids; pub(crate) mod hazard; +mod ids; mod initdata; mod job; -mod capacity; mod kick; mod object; mod port; pub(crate) mod power; -mod reports; mod preparation; mod qos; mod queue; mod recovery; mod replay; +mod reports; mod runtime; pub(crate) mod status; mod teardown; @@ -75,12 +75,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, + impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, - impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -755,10 +755,12 @@ impl Firmware { } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }) + Ok( + match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + ) } /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A @@ -767,7 +769,10 @@ impl Firmware { const RETRY_NS: u64 = 1_000_000; let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { - match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + match self + .hold + .step(now_ns, in_flight, self.effort.submissions(), hold_ms) + { power::HoldAction::None => return None, power::HoldAction::Recheck(delay) => return Some(delay), power::HoldAction::Assert => { @@ -994,7 +999,6 @@ impl Shared { self.renders.notify(); } - /// The session's port, valid while `lease` is held: shutdown and device loss join every /// lease before they detach the session. fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { @@ -1141,7 +1145,10 @@ impl Gpu { ) -> Result> { let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { Some(calibration::Calibration::read(dev).inspect_err(|_| { - dev_err!(dev.as_ref(), "Missing or invalid T6040 GPU calibration handoff\n"); + dev_err!( + dev.as_ref(), + "Missing or invalid T6040 GPU calibration handoff\n" + ); })?) } else { None @@ -1172,11 +1179,14 @@ impl Gpu { }; let max_frequency_khz = Self::max_frequency_khz(dev)?; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let idle_delay = power::idle_delay( - node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?, - )?; + let idle_delay = + power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); - dev_info!(dev.as_ref(), "GPU idle power-off delay: {} ms\n", idle_delay); + dev_info!( + dev.as_ref(), + "GPU idle power-off delay: {} ms\n", + idle_delay + ); let channels = availability::Availability::new()?; let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; @@ -1282,8 +1292,12 @@ impl Gpu { let version = Self::os_firmware_version(dev); // iBoot strings can be shared by OS releases. Prefer the reported release's // candidate, but admit only an exact loaded image, regardless of that hint. - let preferred = builds.iter().filter(|build| Some(build.os_version) == version); - let others = builds.iter().filter(|build| Some(build.os_version) != version); + let preferred = builds + .iter() + .filter(|build| Some(build.os_version) == version); + let others = builds + .iter() + .filter(|build| Some(build.os_version) != version); for build in Iterator::chain(preferred, others) { if Self::identify_image(dev, build).is_ok() { dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); @@ -1306,13 +1320,10 @@ impl Gpu { let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; let start = text.start(); let end = start.checked_add(text.size()).ok_or(EINVAL)?; - crate::fwid::identify( - dev.as_ref().as_ref(), - start..end, - data.size(), - build.images, - ) - .inspect_err(|e| dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e))?; + crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) + })?; Ok(()) } @@ -1372,13 +1383,21 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let primary = + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) + })?; let secondary = match cfg.secondary_root_offset { Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) - })?, + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( + |e| { + dev_err!( + dev.as_ref(), + "Secondary coprocessor setup failed: {:?}\n", + e + ) + }, + )?, ), None => None, }; @@ -1450,13 +1469,18 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; } let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; + session.port.attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } @@ -1469,7 +1493,12 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; session .uat .activate_render_context(&session._render_global)?; @@ -1590,8 +1619,7 @@ impl gpu::Gpu for Gpu { | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } if variant.ray_traversal { params.features |= @@ -1694,12 +1722,16 @@ impl gpu::Gpu for Gpu { // Backend construction has rolled back all temporary leases. let waited = now_ns().saturating_sub(admission_started); let limit = job::TIMEOUT_MS as u64 * 1_000_000; - if waited >= limit { return Err(EBUSY); } + if waited >= limit { + return Err(EBUSY); + } self.shared.queue_events(); // A returned queue slot or reclaimed context wakes this. - self.shared - .queues - .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; + self.shared.queues.block( + observed, + (limit - waited).div_ceil(1_000_000) as u32, + true, + )?; } result => break result?, } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index d22c0f617f374a..789c4451fd6360 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -104,7 +104,10 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -159,7 +162,11 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; // Without room the entries are released here, as they always could be. - let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { Some(&mut self.finished) } else { None @@ -201,13 +208,19 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + let Validated::Compute { scratch, .. } = &command else { + return Err(EINVAL); + }; self.ensure_compute(scratch.pages())?; } // Only convert inputs when no independent host-prefix wait could keep @@ -215,21 +228,34 @@ impl Queue { if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute && !inputs.is_empty() + let sources = if engine == Engine::Compute + && !inputs.is_empty() && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| - !Order::contains(order.wait_through[0], *sequence) - || fences.status() > 0) - && self.frontiers[1].iter().all(|(sequence, fences)| + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0) - && self.failures.iter().enumerate().all(|(index, failure)| - failure.as_ref().is_none_or(|(sequence, _)| - !Order::contains(order.wait_through[index], *sequence))) - { self.backend.input_sources(inputs) } else { None }; + || fences.status() > 0 + }) + && self.failures.iter().enumerate().all(|(index, failure)| { + failure.as_ref().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], *sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); } // Only earlier render ordinals covered by the explicit logical prefix @@ -237,8 +263,7 @@ impl Queue { let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -253,7 +278,8 @@ impl Queue { self.backend.feed(), sources.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -290,8 +316,10 @@ impl Queue { }) }); let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -329,7 +357,10 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } - if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { continue; } dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -359,10 +390,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -429,8 +461,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -444,22 +478,22 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { self.context.note_render(); } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); let firmware_source = pending.len() == 1 diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 4abdd3e485f726..fc1bafd0720a7b 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,7 +67,9 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { return Err(EINVAL); } + if layouts != [0; 2] { + return Err(EINVAL); + } Ok(()) } @@ -266,7 +268,10 @@ impl sched::JobImpl for QueueJob::ver { Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } + QueueTimeout { + dev: self.dev.clone(), + id: self.id, + } } fn prepare(job: &mut Self) -> Option { @@ -759,13 +764,21 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if render + .ts_frag_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if compute + .ts_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 88709a7f9b36dd..7f2de64c424d6b 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -163,9 +163,7 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. - unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) - }; + unsafe { bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) }; } /// The postclose callback of a `struct drm_file`. From 063a6b56510720882e5a0267ee3cc5d82e887939 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:17:39 +1100 Subject: [PATCH 031/171] drm/asahi: Account per-client engine time in fdinfo Retain a per-file engine ledger in queues and completions, and report vertex, fragment and compute time through the Rust DRM fdinfo callback. Charge successful firmware timestamp intervals once at retirement, before signalling the completion fence. Shared code handles conversion and saturating atomic accumulation across classic and native backends, including the shared T6040/G16S and T8140/G17P execution path. Enable the classic firmware's existing timestamp operations without requiring application timestamp outputs. Native retirement already reads the required cells. Do not misreport independent command timer ticks as GPU execution-clock cycles or infer frequency-weighted utilization. Document the sources, limits and overhead, and register eight host tests of the production accounting code. Evidence: existing public firmware timestamp and retirement interfaces; host arithmetic and concurrency tests. Kernel build and runtime qualification remain pending an authorized build service route. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: integration/aurora has no G16 backend and no tools/asahi/run-host-tests.py: those changes are left out; the docs index adds only asahi-usage-stats] (cherry picked from commit 05215160a86feb9a03ea6d834da97d7c281c4312 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- Documentation/gpu/asahi-usage-stats.rst | 80 ++++++++++++ Documentation/gpu/drivers.rst | 1 + drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/file.rs | 19 ++- drivers/gpu/drm/asahi/fw/job.rs | 4 - drivers/gpu/drm/asahi/g15/mod.rs | 3 +- drivers/gpu/drm/asahi/g15/poll.rs | 27 ++-- drivers/gpu/drm/asahi/g15/queue.rs | 11 +- drivers/gpu/drm/asahi/g17/completion.rs | 13 ++ drivers/gpu/drm/asahi/g17/mod.rs | 2 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 4 + drivers/gpu/drm/asahi/gpu.rs | 3 + drivers/gpu/drm/asahi/queue/compute.rs | 65 +++++----- drivers/gpu/drm/asahi/queue/mod.rs | 3 + drivers/gpu/drm/asahi/queue/render.rs | 127 ++++++++++--------- drivers/gpu/drm/asahi/usage.rs | 161 ++++++++++++++++++++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/drm/file.rs | 17 +++ tools/asahi/usage-tests.rs | 6 + 19 files changed, 450 insertions(+), 98 deletions(-) create mode 100644 Documentation/gpu/asahi-usage-stats.rst create mode 100644 drivers/gpu/drm/asahi/usage.rs create mode 100644 tools/asahi/usage-tests.rs diff --git a/Documentation/gpu/asahi-usage-stats.rst b/Documentation/gpu/asahi-usage-stats.rst new file mode 100644 index 00000000000000..f5e731e77d0c26 --- /dev/null +++ b/Documentation/gpu/asahi-usage-stats.rst @@ -0,0 +1,80 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Asahi client engine time +======================== + +Asahi reports the following optional keys in each DRM file's fdinfo, following +:doc:`drm-usage-stats`:: + + drm-engine-vertex: 123456 ns + drm-engine-fragment: 234567 ns + drm-engine-compute: 345678 ns + +``vertex`` is the tiling/vertex stage of render commands, ``fragment`` is their +fragment stage, and ``compute`` is compute dispatch. These are separate timelines: +vertex and fragment can overlap. There is no separately submitted blit engine in +the driver's command ABI. Copies executed by compute or render are included in +the engine that executes them. + +The counters belong to the open DRM file, rather than to a PID or a queue handle. +All queues and VMs of that file contribute to the same ledger. Duplicated or +inherited file descriptors share counters and must be deduplicated using +``drm-client-id``. Closing a queue or VM does not reset its file's counters, and +completions retain the ledger until their firmware work retires. A new DRM file +starts at zero, even when another file in the same process has submitted work. + +Time source and limits +---------------------- + +Only completed stage intervals are reported. A long-running command contributes +after retirement, so polling during execution can temporarily show no change. +Counters accumulate monotonically, saturating at the largest unsigned 64-bit +value. Conversion rounds down by less than one nanosecond per retired stage. + +G13 and G14 use the firmware's internal start/end timestamp cells. The existing +timestamp microsequence operations run even when the application requests no +timestamp output. Internal buffers were already allocated for each command; +fdinfo never reads an application's timestamp buffer. + +The G15 and native G16 backends reuse the internal timestamp cells already read +by their successful retirement statistics path. The shared G17 execution backend, +including T6040/G16S and T8140/G17P, charges the timestamp pairs delivered by its +once-only completion path. These backends use the firmware's 24 MHz command timer; +the classic backend uses its hardware configuration's timer frequency. + +These are firmware stage envelopes, not hardware shader occupancy measurements. +They can include firmware scheduling, preemption and stalls between the stage's +start and end operations. Intervals are summed without attempting to merge +overlapping commands on the same engine. Missing, zero-start or reversed intervals +are ignored. Failed or cancelled commands do not contribute incomplete intervals; +successful stages that retired before a later failure remain counted. An +unexecuted G17 replay contributes only when it eventually retires successfully. +There is no submit-to-completion wall-time fallback in the current backends. + +``drm-cycles-*`` and ``drm-maxfreq-*`` are intentionally absent. The timestamp +clock is independent of the variable GPU execution clock. Treating command timer +ticks as GPU busy cycles, or multiplying time by a sampled/maximum GPU frequency, +would give incorrect frequency-weighted utilization. Per-client execution-cycle +counters would require a separate hardware measurement source. + +Accounting cost +--------------- + +Each file allocates one three-counter ledger. Queues and pending completions +retain references to it. Retirement adds one relaxed, saturating atomic update +per valid stage, with no new locks or allocations. The native backends reuse +existing timestamp reads. Classic completion callbacks read two internal cells +per stage; commands without application timestamps also execute two existing +timestamp microsequence operations per stage. Reading fdinfo loads three counters +and does not traverse queues, wait for fences or access hardware. + +Host arithmetic tests +--------------------- + +The ledger uses only ``core`` and can be tested directly from a source checkout:: + + rustc --edition 2021 --test drivers/gpu/drm/asahi/usage.rs -o /tmp/asahi-usage-test + /tmp/asahi-usage-test + +Tests cover conversion, missing/reset timestamps, saturation, overlapping stages, +client isolation, out-of-order completions and simultaneous queues sharing a file. diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index 8289fd66c2982b..97f424dd197980 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -7,6 +7,7 @@ GPU Driver Documentation amdgpu/index apple-dcp-backlight + asahi-usage-stats i915 imagination/index intel-display/index diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index a2662964a5edea..7766c20caad5ba 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -39,6 +39,7 @@ mod stats; mod stats_raw; mod sysfs_exports; mod timestamp; +mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 12463c486cf16e..3f388da8aad840 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -280,6 +280,8 @@ pub(crate) struct File { params_size: core::sync::atomic::AtomicU64, /// GPU memory this client created and holds. mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, } /// Convenience type alias for our DRM `File` type. @@ -300,9 +302,10 @@ impl drm::file::DriverFile for File { let id = gpu.ids().file.next(); let user_range = gpu.user_range(); let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) }) } @@ -316,6 +319,17 @@ impl drm::file::DriverFile for File { }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -329,6 +343,7 @@ impl File { id: u64, user_range: Range, mem: Arc, + usage: Arc, ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { @@ -356,6 +371,7 @@ impl File { (*slot).user_range = user_range; addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); Ok(()) }) } @@ -1358,6 +1374,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index e4f2f9225ea050..aaa83355b437d2 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -139,10 +139,6 @@ pub(crate) struct UserTimestamps { } impl UserTimestamps { - pub(crate) fn any(&self) -> bool { - self.start.is_some() || self.end.is_some() - } - pub(crate) fn pointers(&self) -> Result> { Ok(raw::TimestampPointers { start_addr: self diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 6bbdcf037620d9..b4877490cad71a 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -673,9 +673,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 158abab719b52e..1cc1c483ae78a0 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -550,25 +550,38 @@ impl Firmware { let s = &mut self.engine.stats; // Add one stage interval read from internal timestamp cells (start, end). - let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { - let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; - let start = rd(off + 4) << 32 | rd(off); - let end = rd(off + 12) << 32 | rd(off + 8); + let mut stage = |o: &super::object::FwObject, + off: usize, + sum: &mut u64, + engine: crate::usage::Engine| { + let (Ok(start_lo), Ok(start_hi), Ok(end_lo), Ok(end_hi)) = + (o.r32(off), o.r32(off + 4), o.r32(off + 8), o.r32(off + 12)) + else { + return; + }; + let start = u64::from(start_hi) << 32 | u64::from(start_lo); + let end = u64::from(end_hi) << 32 | u64::from(end_lo); if start == 0 || end < start { return; } + f.job.queue.usage.add(engine, start, end, 24_000_000); *sum += end - start; lo = lo.min(start); hi = hi.max(end); }; for k in 0..f.renders as usize { let p = &scrub.passes[f.claims.pass[k] as usize]; - stage(&p.its, 0, &mut s.ta_ticks); - stage(&p.its, 0x10, &mut s.fs_ticks); + stage(&p.its, 0, &mut s.ta_ticks, crate::usage::Engine::Vertex); + stage( + &p.its, + 0x10, + &mut s.fs_ticks, + crate::usage::Engine::Fragment, + ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { let c = &scrub.compute[f.claims.slot[k] as usize]; - stage(&c.its, 0, &mut s.cdm_ticks); + stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { s.span_ticks += hi - lo; diff --git a/drivers/gpu/drm/asahi/g15/queue.rs b/drivers/gpu/drm/asahi/g15/queue.rs index cc87f1ccbe084d..03a8f46c8a0fce 100644 --- a/drivers/gpu/drm/asahi/g15/queue.rs +++ b/drivers/gpu/drm/asahi/g15/queue.rs @@ -42,6 +42,8 @@ pub(crate) struct QueueState { /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. pub(crate) priority: u32, + /// Completed stage time of the DRM file that created this queue. + pub(crate) usage: Arc, /// USC executable window base. pub(crate) usc_base: u64, /// Engine service turn of this queue (least recently served first). @@ -103,7 +105,13 @@ impl Drop for Queue { impl Queue { /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as /// `file.rs` passes it. - pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + pub(crate) fn new( + gpu: &Gpu, + vm: mmu::Vm, + priority: u32, + usc_exec_base: u64, + usage: Arc, + ) -> Result { if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { return Err(EINVAL); } @@ -129,6 +137,7 @@ impl Queue { let state = Arc::new( QueueState { priority, + usage, usc_base: usc_exec_base, turn: AtomicU64::new(0), owned: [const { AtomicU32::new(0) }; 32], diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a232b9c9eb2ccc..e7249cdc3d32cb 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -255,6 +255,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + usage: Arc, sources: Option>, #[pin] vm_job: Mutex>, @@ -292,6 +293,7 @@ impl Completion { host_predecessors: bool, feed: Option>, sources: Option>, + usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); @@ -322,6 +324,7 @@ impl Completion { terminal, compute, feed, + usage, sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), @@ -651,6 +654,16 @@ impl Completion { if result.is_ok() && self.interrupted() { result = Err(ECANCELED); } + if let Ok(ticks) = &result { + use crate::usage::Engine; + let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; + if self.compute.is_some() { + self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); + } else { + self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); + self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); + } + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ae5bbd61a3ce4e..c2962c45f7afc5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1701,6 +1701,7 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); @@ -1742,6 +1743,7 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, + usage, ) { Ok(frontend) => frontend, Err(error) => { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 789c4451fd6360..e593c5879cf741 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -84,6 +84,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -116,6 +117,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -128,6 +130,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -277,6 +280,7 @@ impl Queue { host_predecessors.clone().next().is_some(), self.backend.feed(), sources.clone(), + self.usage.clone(), )?; let publication = completion .publication() diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 84d82bb1880ef2..2ff7ab369c43bd 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -391,6 +391,7 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( @@ -1447,6 +1448,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1464,6 +1466,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index c9f205708e3131..ff9804c5bd10be 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -102,6 +102,10 @@ impl super::QueueInner::ver { GFP_KERNEL, )?; + let usage_timestamps = timestamps.clone(); + let usage = self.usage.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; + let uuid = 0; mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); @@ -150,20 +154,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -174,20 +176,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_comp); builder.add(microseq::FinalizeCompute::ver { @@ -362,6 +362,15 @@ impl super::QueueInner::ver { comp_job.add_cb(comp, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Compute, + raw.start.load(Ordering::Relaxed), + raw.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index fc1bafd0720a7b..b011ac127045ed 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -153,6 +153,7 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, + usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -487,6 +488,7 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -552,6 +554,7 @@ impl Queue::ver { notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, + usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 68c4856ca4200c..4de0e84239a02e 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -511,20 +511,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -535,20 +533,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_frag); builder.add(microseq::FinalizeFragment::ver { @@ -929,9 +925,21 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; frag_job.add_cb(frag, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()); + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Fragment, + raw.frag.start.load(Ordering::Relaxed), + raw.frag.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); @@ -1022,20 +1030,18 @@ impl super::QueueInner::ver { unk_178: (!clustering) as u32, })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -1046,20 +1052,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_vtx); builder.add(microseq::FinalizeVertex::ver { @@ -1368,9 +1372,20 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Vertex, + raw.vtx.start.load(Ordering::Relaxed), + raw.vtx.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs new file mode 100644 index 00000000000000..496c722dba651e --- /dev/null +++ b/drivers/gpu/drm/asahi/usage.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-file engine time, shared by all firmware backends. +//! +//! Queues retain their file's ledger until their last completion. Only authenticated, +//! successful completion timestamps are charged, before signalling the completion fence. +//! These are command timer ticks, not shader clock cycles. Never export them as drm-cycles. + +use core::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy)] +pub(crate) enum Engine { + Vertex, + Fragment, + Compute, +} + +/// One ledger per DRM file, shared by all of its queues and VMs. +pub(crate) struct Client { + ns: [AtomicU64; 3], +} + +impl Client { + pub(crate) const fn new() -> Self { + Self { + ns: [const { AtomicU64::new(0) }; 3], + } + } + + /// Called exactly once per retired stage, while its internal timestamps are retained. + /// Missing, reversed or reset timestamps contribute nothing. Sub-nanosecond fractions + /// are rounded down per stage. Saturation keeps fdinfo monotonic even on overflow. + pub(crate) fn add(&self, engine: Engine, start: u64, end: u64, frequency_hz: u32) { + if start == 0 || end <= start || frequency_hz == 0 { + return; + } + let ns = ticks_to_ns(end - start, frequency_hz); + let counter = &self.ns[engine as usize]; + let mut old = counter.load(Ordering::Relaxed); + loop { + match counter.compare_exchange_weak( + old, + old.saturating_add(ns), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(value) => old = value, + } + } + } + + /// Vertex, fragment and compute, in ns. No cross-engine atomic snapshot is promised. + pub(crate) fn snapshot(&self) -> [u64; 3] { + self.ns.each_ref().map(|ns| ns.load(Ordering::Relaxed)) + } +} + +fn ticks_to_ns(ticks: u64, frequency_hz: u32) -> u64 { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let hz = u64::from(frequency_hz); + // The remainder is smaller than a u32 frequency, so its product fits in u64. + (ticks / hz) + .saturating_mul(NSEC_PER_SEC) + .saturating_add(ticks % hz * NSEC_PER_SEC / hz) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HZ: u32 = 24_000_000; + + #[test] + fn idle_and_invalid_intervals() { + let client = Client::new(); + assert_eq!(client.snapshot(), [0; 3]); + for (start, end, hz) in [ + (0, 100, HZ), + (100, 0, HZ), + (20, 10, HZ), + (1, 1, HZ), + (1, 2, 0), + ] { + client.add(Engine::Compute, start, end, hz); + } + assert_eq!(client.snapshot(), [0; 3]); + } + + #[test] + fn engine_and_client_isolation() { + let a = Client::new(); + let b = Client::new(); + a.add(Engine::Vertex, 10, 24_010, HZ); + a.add(Engine::Fragment, 20, 48_020, HZ); + b.add(Engine::Compute, 30, 72_030, HZ); + assert_eq!(a.snapshot(), [1_000_000, 2_000_000, 0]); + assert_eq!(b.snapshot(), [0, 0, 3_000_000]); + } + + #[test] + fn overlapping_stages_are_independent() { + let client = Client::new(); + client.add(Engine::Vertex, 1, 25, HZ); + client.add(Engine::Fragment, 13, 37, HZ); + assert_eq!(client.snapshot(), [1_000, 1_000, 0]); + } + + #[test] + fn conversion_matches_wide_reference() { + for hz in [1, HZ, 1_000_000_000, u32::MAX] { + for ticks in [0, 1, 23, 24, 25, u32::MAX as u64, u64::MAX / 2, u64::MAX] { + let expected = (u128::from(ticks) * 1_000_000_000 / u128::from(hz)) + .min(u128::from(u64::MAX)) as u64; + assert_eq!(ticks_to_ns(ticks, hz), expected); + } + } + } + + #[test] + fn fractional_ticks_round_down() { + assert_eq!(ticks_to_ns(1, HZ), 41); + assert_eq!(ticks_to_ns(3, HZ), 125); + } + + #[test] + fn out_of_order_completions_accumulate() { + let client = Client::new(); + client.add(Engine::Compute, 100, 124, HZ); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot(), [0, 0, 2_000]); + } + + #[test] + fn saturation_is_monotonic() { + let client = Client::new(); + client.add(Engine::Compute, 1, u64::MAX, 1); + assert_eq!(client.snapshot()[2], u64::MAX); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot()[2], u64::MAX); + } + + #[test] + fn concurrent_queues_share_one_file() { + let client = std::sync::Arc::new(Client::new()); + let threads: std::vec::Vec<_> = (0..8) + .map(|_| { + let client = client.clone(); + std::thread::spawn(move || { + for _ in 0..10_000 { + client.add(Engine::Compute, 1, 25, HZ); + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } + assert_eq!(client.snapshot(), [0, 0, 80_000_000]); + } +} diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index f981774335c7ca..b7632359fc9595 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 7f2de64c424d6b..f152957397221b 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -44,6 +44,11 @@ pub trait DriverFile { fn memory_stats(&self) -> Option { None } + + /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. + /// Engine names must be stable, unique and contain no whitespace or colons. + /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. + fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -136,6 +141,18 @@ impl File { // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the // file, so `postclose_callback()` cannot run concurrently. let file = unsafe { File::::from_raw(raw_file) }; + file.inner().engine_stats(|name, ns| { + // SAFETY: The DRM core owns printer throughout this callback; the format and + // name are NUL-terminated, and the vararg types match %s and %llu. + unsafe { + bindings::drm_printf( + printer, + c"drm-engine-%s:\t%llu ns\n".as_ptr(), + name.as_char_ptr(), + ns as core::ffi::c_ulonglong, + ) + }; + }); let Some(stats) = file.inner().memory_stats() else { return; }; diff --git a/tools/asahi/usage-tests.rs b/tools/asahi/usage-tests.rs new file mode 100644 index 00000000000000..67bbbc58d6276e --- /dev/null +++ b/tools/asahi/usage-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production per-client engine ledger's tests without a kernel build. + +#[path = "../../drivers/gpu/drm/asahi/usage.rs"] +mod usage; From dafa6d0c8c52768b2342c70318d042878156a35f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 02:51:43 +1100 Subject: [PATCH 032/171] drm/asahi: Format the G17 statistics integration paths Apply rustfmt to the existing files touched by the qualified statistics snapshot, keeping formatting separate from the decoder change. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/runtime/events.rs] (cherry picked from commit 2bdb22771cb2c0cb031e3d4ad8e70c5718cff99d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 11 ++--- drivers/gpu/drm/asahi/g17/channel.rs | 7 ++- drivers/gpu/drm/asahi/g17/port.rs | 49 +++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 29 +++++++----- drivers/gpu/drm/asahi/sysfs_exports.rs | 6 ++- 5 files changed, 70 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 80582ece5d2391..1dc776acab746a 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -227,10 +227,7 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); @@ -449,7 +446,11 @@ impl AsahiPlatformDriver { if let Err(error) = crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) { - dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); } Ok(AsahiPlatformDriverData { _device: reg.device().into(), diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 568647ebb444ae..7b99daf4c1fce2 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -85,7 +85,12 @@ impl Rings { } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing, reports: &mut super::reports::Reports) -> Result { + fn consume_report( + init: &InitData, + role: Role, + ring: ReportRing, + reports: &mut super::reports::Reports, + ) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 6d0e87c14b9477..ff95b07fa759ba 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -225,7 +225,11 @@ impl Port { /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns /// the new producer, the retained consumer witness for this slot. - pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + pub(super) fn publish_compute( + &self, + mut claim: OuterClaim<'_>, + record: &WorkSlot, + ) -> Result { if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { return Err(EINVAL); } @@ -242,7 +246,8 @@ impl Port { unsafe { core::ptr::copy_nonoverlapping( core::ptr::from_ref(record).cast::(), - ring.slots.add(producer as usize * core::mem::size_of::()), + ring.slots + .add(producer as usize * core::mem::size_of::()), core::mem::size_of::(), ); } @@ -280,13 +285,16 @@ impl Port { let (fragment, tiling) = self.render_rings(priority)?; let cursors = [fragment.cursors()?, tiling.cursors()?]; let room = |[first, second, producer]: [u32; 3]| { - let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + let room = + |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; room(first).min(room(second)) }; - Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { - priority, - producers: [cursors[0][2], cursors[1][2]], - })) + Ok( + (room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + }), + ) } /// Copies a render pair's fragment, then tiling record into the slots after the producers @@ -312,7 +320,10 @@ impl Port { return Err(EINVAL); } // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. - let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + let slot = unsafe { + ring.slots + .add(producer as usize * core::mem::size_of::()) + }; Ok((slot, (producer + 1) % WORK_RING_SLOTS)) }; let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; @@ -385,7 +396,10 @@ impl Port { policy: Policy, ) -> Result { let view = self.qos_view()?; - let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?; + let publication = self + .qos + .lock() + .publish(&view, owner, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publication) } @@ -399,7 +413,9 @@ impl Port { ) -> Result<[qos::Publication; 2]> { let view = self.qos_view()?; let publications = - self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?; + self.qos + .lock() + .publish_owners(&view, owners, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publications) } @@ -485,13 +501,18 @@ impl Port { } pub(super) fn note_gpu_activity(&self) { - self.last_submission_ns.store(super::now_ns(), Ordering::Relaxed); + self.last_submission_ns + .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "effort_full={} direct_submissions={} last_submission_ns={}", - self.effort_full.load(Ordering::SeqCst), self.submissions(), - self.last_submission_ns.load(Ordering::Relaxed)) + writeln!( + f, + "effort_full={} direct_submissions={} last_submission_ns={}", + self.effort_full.load(Ordering::SeqCst), + self.submissions(), + self.last_submission_ns.load(Ordering::Relaxed) + ) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3bd159037e2660..55d517a60ade8b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -47,7 +47,8 @@ impl event::Transport for Service<'_> { role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = channel::Events::new(role); + let mut events = + channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -232,7 +233,9 @@ impl recovery::Queues for Service<'_> { }) } fn report_blamed_render(&self, qid: u8) { - self.firmware.queues.report_render_recovery(qid, &self.firmware.port); + self.firmware + .queues + .report_render_recovery(qid, &self.firmware.port); } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self @@ -264,11 +267,9 @@ impl Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed( - state, - |owner| port.qos_complete(&[owner]), - &mut |event| deferred.push(event), - )?; + queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { + deferred.push(event) + })?; } Ok(()) } @@ -313,7 +314,10 @@ impl Firmware { }); // Publishers without the device mutex stamp their tickets with this // mirror; they rejoin only after preparation admission reopens. - service.firmware.port.set_generation(service.firmware.recovery.generation()); + service + .firmware + .port + .set_generation(service.firmware.recovery.generation()); let result = result?; if let Some(error) = result.target_error { dev_warn!( @@ -394,7 +398,8 @@ impl crate::g17::Shared { deferred.finish(); crate::g17::trace::event(1); if recover { - self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations + .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -451,8 +456,10 @@ impl crate::g17::Shared { || self.progress.has_waiters() || self.queues.has_waiters() || self.renders.has_waiters() - || now.saturating_sub(self.housekeeping_ns.load(core::sync::atomic::Ordering::Relaxed)) - >= HOUSEKEEPING_NS; + || now.saturating_sub( + self.housekeeping_ns + .load(core::sync::atomic::Ordering::Relaxed), + ) >= HOUSEKEEPING_NS; let mut lingers = None; let mut pools = None; let mut shrinking = false; diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index cee5d52a4e2619..e7f842d8ac6fdd 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -29,7 +29,11 @@ pub(crate) fn register_idle( // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes // it with READ_ONCE/WRITE_ONCE. let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } } /// Register only after initialization succeeds, keeping the snapshot Arc alive. From 7cb914b6ce08c055217bba0d0968da50d7b7fdf1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 05:28:57 +1100 Subject: [PATCH 033/171] drm/asahi: Export qualified G16S and G17P statistics Decode bounded tiler, renderer and device percentages and the logical pstate from complete copied records in one shared G17 path. Merge firmware roles in drain order, bound the utilisation averaging window, and invalidate epochs on restart, suspend/resume, quarantine and observed gating. Keep stable logical states valid without inventing a periodic-report deadline. Read a coherent snapshot under the device mutex without waking the GPU. Leave ambiguous A/B requested frequency, achieved frequency, busy time, measured power and temperature unsupported; preserve the legacy G13/G14 formatter. Add shared KUnit/host SPEC vectors, transport and formatter tests, legacy byte comparisons, negative controls and ABI documentation. Resolve the existing driver Clippy diagnostics with typed conversions and explicit contracts. Enforce bound-device access during SG preparation with a joined binding lease instead of unchecked lifetime casts; exercise removal and rebind with host concurrency controls. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: integration/aurora has no G16 backend and lacks three host test scripts this commit edits: those changes are left out. integration/aurora keeps its own map_physical() and adoption signature for G15, its newest_bind field in VmBoData, its large user BO reclaim policy in gem.rs, and the M3 runtime stop at unbind next to close_binding()] (cherry picked from commit fdcf196afe4a1a2fc6fe56ba03c72d2e136f525b in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../ABI/testing/sysfs-driver-asahi-agx-stats | 57 +- drivers/gpu/drm/asahi/alloc.rs | 18 +- drivers/gpu/drm/asahi/asahi_core.rs | 3 + drivers/gpu/drm/asahi/binding.rs | 130 +++++ drivers/gpu/drm/asahi/driver.rs | 31 +- drivers/gpu/drm/asahi/file.rs | 17 +- drivers/gpu/drm/asahi/float.rs | 4 +- drivers/gpu/drm/asahi/fw/event.rs | 2 +- drivers/gpu/drm/asahi/fw/types.rs | 6 + drivers/gpu/drm/asahi/fwid.rs | 2 +- drivers/gpu/drm/asahi/g15/mod.rs | 6 + drivers/gpu/drm/asahi/g17/channel.rs | 85 ++- drivers/gpu/drm/asahi/g17/fence.rs | 228 ++++++-- drivers/gpu/drm/asahi/g17/initdata.rs | 150 +++-- drivers/gpu/drm/asahi/g17/mod.rs | 27 +- drivers/gpu/drm/asahi/g17/port.rs | 16 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 514 ++++++++++++++---- drivers/gpu/drm/asahi/g17/queue/render.rs | 185 +++++-- .../drm/asahi/g17/queue/render/publication.rs | 8 +- drivers/gpu/drm/asahi/g17/reports.rs | 54 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 4 + drivers/gpu/drm/asahi/g17/stats.rs | 40 ++ drivers/gpu/drm/asahi/g17/telemetry.rs | 246 +++++++++ drivers/gpu/drm/asahi/gem.rs | 8 +- drivers/gpu/drm/asahi/gpu.rs | 16 +- drivers/gpu/drm/asahi/initdata.rs | 44 +- drivers/gpu/drm/asahi/mem.rs | 6 +- drivers/gpu/drm/asahi/microseq.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 172 ++++-- drivers/gpu/drm/asahi/object.rs | 40 +- drivers/gpu/drm/asahi/pgtable.rs | 37 +- drivers/gpu/drm/asahi/queue/compute.rs | 19 +- drivers/gpu/drm/asahi/queue/mod.rs | 23 +- drivers/gpu/drm/asahi/queue/render.rs | 159 +++--- drivers/gpu/drm/asahi/stats.rs | 8 +- drivers/gpu/drm/asahi/sysfs.c | 92 +++- drivers/gpu/drm/asahi/sysfs.h | 22 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 73 ++- drivers/gpu/drm/asahi/util.rs | 11 +- drivers/gpu/drm/asahi/workqueue.rs | 2 +- tools/asahi/binding-lifetime-tests.py | 125 +++++ tools/asahi/g15-probe-init-tests.py | 8 + .../asahi/g17-telemetry-integration-tests.py | 371 +++++++++++++ tools/asahi/g17-telemetry-tests.rs | 6 + 44 files changed, 2525 insertions(+), 552 deletions(-) create mode 100644 drivers/gpu/drm/asahi/binding.rs create mode 100644 drivers/gpu/drm/asahi/g17/stats.rs create mode 100644 drivers/gpu/drm/asahi/g17/telemetry.rs create mode 100644 tools/asahi/binding-lifetime-tests.py create mode 100644 tools/asahi/g17-telemetry-integration-tests.py create mode 100644 tools/asahi/g17-telemetry-tests.rs diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 11a964e6f17d81..10cba6c35673fa 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -64,5 +64,58 @@ Description: `#[repr(C)]` on the Rust side and its offsets are asserted at build time in `sysfs.c`; readers must parse `key value` lines and must not assume binary layout of this file. - G17 firmware has no supported decoder for this snapshot format; - its file reports unsupported rather than fabricated zero counters. + + G16S (T6040/V25G83) and G17P (T8140/V26A428) use a separate, + qualified snapshot. The G13/G14 output above is unchanged. + The following keys replace the legacy keys on these firmwares: + + utilisation_status, pstate_status + Each is valid, stale, or unavailable. Numeric metrics + are printed only while their capability is valid; + otherwise their value is the string unsupported. + + util_tiler_percent, util_renderer_percent, util_device_percent + Unsigned integer percent (0..100), from tag 0x13 + words 3, 4 and 6 respectively (zero-based). Device + utilisation is independent of the overlapping engines; + it is never their sum. Each complete valid packet from + either role contributes one sample to a shared integer + mean, rounded down. Reads do not consume samples. + A packet with any value above 100 invalidates the whole + utilisation capability. Zero is a valid sample. + + pstate Logical selector 0..10 from tag 0x10 word 6. Words 1 + and 2 form a little-endian u64 firmware timestamp. + The first timestamp establishes a baseline. Invalid + selectors and backwards timestamps remove pstate + validity; timestamps are never extended across epochs. + P0 is a logical state, not a core-power measurement. + + requested_frequency_hz, achieved_frequency_hz, busy_ns, + power_mw, temperature_raw + Always unsupported on these firmwares. The statistics + packet does not identify performance table A or B; + a selector is neither a host request nor a measured + frequency. No qualified busy-time duration, power unit + or temperature field is available. + + The host uses a 16 ms age/averaging-window limit for V25G83 and + 32 ms for V26A428 (two nominal periods). These are conservative + host policies, not firmware guarantees about packet duration. + The oldest contributing utilisation sample bounds its window. + A logical pstate has no qualified periodic-report cadence and + remains valid until a replacement or an epoch boundary; its + host receive time is retained in diagnostics, not used to invent + a periodic-report deadline. + Suspend/resume, restart, statistics-ring quarantine, observed + core gating and teardown end validity. After an epoch boundary, + each ring's already queued records are discarded before new + baselines are accepted. A quarantined statistics ring disables + both roles' exports until reprobe; a trace-ring error does not. + + Reads take the device mutex and do not wake the GPU. The + firmware_reports debugfs view retains per-role raw packets, + counts and host monotonic receive times, plus capability status, + policy, timestamp and diagnostic tick deltas. Tick deltas are + never accumulated as GPU busy time. Firmware role streams are + merged in drain order without deduplication. diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index d815fdf1040ac1..10e367b1a15fae 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -162,14 +162,14 @@ impl Drop for GenericAlloc { // the invariant, and greater than debug_len as checked above. unsafe { let p = p.as_ptr().add(self.debug_offset - debug_len); - (p as *mut u32).write(STATE_DEAD); + p.cast::().write(STATE_DEAD); } } } if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = self.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xde, self.size()) }; } } if self.padding != 0 { @@ -178,7 +178,7 @@ impl Drop for GenericAlloc { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts( - (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + p.as_ptr().cast::().cast_const().add(self.size()), self.padding, ) }; @@ -313,8 +313,8 @@ pub(crate) trait Allocator { unsafe { let p = p.as_ptr(); p.write_bytes(0x42, debug_offset - 2 * debug_len); - let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; - let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + let cur = p.add(debug_offset - debug_len).cast::(); + let prev = p.add(debug_offset - 2 * debug_len).cast::(); prev.copy_from(cur, 1); cur.copy_from(&debug, 1); }; @@ -344,7 +344,7 @@ pub(crate) trait Allocator { if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = ret.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xaa, ret.size()) }; } } @@ -354,7 +354,7 @@ pub(crate) trait Allocator { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts_mut( - (p.as_ptr() as *mut u8).add(ret.size()), + p.as_ptr().cast::().add(ret.size()), padding, ) }; @@ -536,7 +536,7 @@ impl Allocator for SimpleAllocator { ); let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; - let p = obj.vmap()?.addr as *mut u8; + let p = obj.vmap()?.addr.cast::(); if debug_enabled(DebugFlags::FillAllocations) { let obj_size = obj.size(); if let Ok(vmap) = obj.vmap() { @@ -990,7 +990,7 @@ impl HeapAllocator { let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { let idx = idx.unwrap_or(inner.backing_objects.len() - 1); let obj = &mut inner.backing_objects[idx]; - let p = obj.0.vmap()?.addr as *mut u8; + let p = obj.0.vmap()?.addr.cast::(); Ok((obj.2, obj.0.size(), p)) })?; assert!(obj_start <= start); diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 7766c20caad5ba..a218270d6fc425 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,6 +4,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; +mod binding; mod buffer; mod channel; mod cleanup; @@ -66,6 +67,8 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, diff --git a/drivers/gpu/drm/asahi/binding.rs b/drivers/gpu/drm/asahi/binding.rs new file mode 100644 index 00000000000000..5e0fac1f29ab29 --- /dev/null +++ b/drivers/gpu/drm/asahi/binding.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scoped DMA-device access during probe and runtime mapping preparation. +//! +//! The platform owner closes admission and joins leases before unbind returns. +//! VMs retain the gate, so an old VM cannot acquire access after removal/rebind. + +use crate::driver::AsahiDrmDevice; +use kernel::{ + device::{Bound, Device}, + new_condvar, new_mutex, + prelude::*, + sync::{aref::ARef, Arc, CondVar, Mutex}, +}; + +kernel::sync::global_lock! { + // SAFETY: Module initialization initializes this lock before registering adapters. + unsafe(uninit) static DEVICES: Mutex>> = KVec::new(); +} + +/// Initialize the registry before a platform adapter can probe. +/// +/// # Safety +/// +/// Called exactly once per module instance, before any other binding operation. +pub(crate) unsafe fn init() { + // SAFETY: The module initializer establishes the single-call ordering above. + unsafe { DEVICES.init() }; +} + +struct State { + live: bool, + users: usize, +} + +#[pin_data] +pub(crate) struct Gate { + key: usize, + dev: ARef, + #[pin] + state: Mutex, + #[pin] + changed: CondVar, +} + +/// The platform binding's owner; Drop also covers errors during probe. +pub(crate) struct Owner(Arc); + +impl Owner { + /// Register while probe supplies the parent bound-device proof. + /// + /// # Safety + /// + /// The platform adapter must call close before unbind returns, independently + /// of this owner's Drop. On probe error the owner must be closed/dropped + /// before the bound probe callback returns. + pub(crate) unsafe fn new(dev: &Device, drm: &AsahiDrmDevice) -> Result { + let gate = Arc::pin_init( + pin_init!(Gate { + key: core::ptr::from_ref(drm) as usize, + dev: dev.into(), + state <- new_mutex!(State { live: true, users: 0 }, "asahi::binding::state"), + changed <- new_condvar!("asahi::binding::changed"), + }), + GFP_KERNEL, + )?; + DEVICES.lock().push(gate.clone(), GFP_KERNEL)?; + Ok(Self(gate)) + } + + /// Called from platform unbind before devres releases the parent's state. + pub(crate) fn close(&self) { + DEVICES.lock().retain(|gate| !Arc::ptr_eq(gate, &self.0)); + let mut state = self.0.state.lock(); + state.live = false; + while state.users != 0 { + self.0.changed.wait(&mut state); + } + } +} + +impl Drop for Owner { + fn drop(&mut self) { + self.close(); + } +} + +/// Look up the binding for this DRM instance, not just its reusable parent address. +pub(crate) fn lookup(drm: &AsahiDrmDevice) -> Result> { + let key = core::ptr::from_ref(drm) as usize; + DEVICES + .lock() + .iter() + .find(|gate| gate.key == key) + .cloned() + .ok_or(ENODEV) +} + +impl Gate { + /// Covers only SG-table acquisition/segment preparation, not firmware waits. + pub(crate) fn enter(&self) -> Result> { + let mut state = self.state.lock(); + if !state.live { + return Err(ENODEV); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease(self)) + } +} + +pub(crate) struct Lease<'a>(&'a Gate); + +impl Lease<'_> { + pub(crate) fn device(&self) -> &Device { + // SAFETY: Owner starts during bound probe, rejects new leases on close, + // and joins existing leases before unbind returns. This borrow cannot + // outlive its lease; the ARef also retains the device allocation. + unsafe { self.0.dev.as_bound() } + } +} + +impl Drop for Lease<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.users -= 1; + if state.users == 0 { + self.0.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 1dc776acab746a..e2ecd5193bd625 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -47,6 +47,7 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. @@ -237,6 +238,7 @@ impl platform::Driver for AsahiPlatformDriver { // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } + this.close_binding(); } /// Device probe function. @@ -256,6 +258,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -272,6 +275,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -327,6 +333,7 @@ impl platform::Driver for AsahiPlatformDriver { )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -351,6 +358,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -360,6 +370,7 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -419,6 +430,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -434,9 +448,14 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); @@ -453,6 +472,7 @@ impl AsahiPlatformDriver { ); } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), @@ -465,8 +485,15 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + self.binding.close(); // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 3f388da8aad840..b135a558eeee32 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -215,15 +215,15 @@ impl SyncItem { let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid @@ -345,6 +345,9 @@ impl File { mem: Arc, usage: Arc, ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); @@ -466,13 +469,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; file.inner() @@ -1343,9 +1346,9 @@ impl File { ); return Err(EINVAL); } - // SAFETY: capable() only reads the calling task's credentials (ioctl process context). if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } && !file.is_current_master() { cls_pr_debug!( diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 998ddb9ed9b2d9..51cac02624d3d1 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -145,12 +145,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() + F32U::from_i64(i64::from(v)).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() + F32U::from_u64(u64::from(v)).pack() } } }; diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs index 52bc456f58707d..e26bc6ceb34485 100644 --- a/drivers/gpu/drm/asahi/fw/event.rs +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -73,7 +73,7 @@ pub(crate) mod raw { // We could use fetch_add, but the non-LSE atomic // sequence Rust produces confuses the hypervisor. let v = self.0.load(Ordering::Relaxed); - self.0.store(v + (amount as u64), Ordering::Relaxed); + self.0.store(v + u64::from(amount), Ordering::Relaxed); } } diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs index f55d6cec6b8ca3..11def551029f5b 100644 --- a/drivers/gpu/drm/asahi/fw/types.rs +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -117,6 +117,8 @@ macro_rules! default_zeroed { (<$($lt:lifetime),*>, $type:ty) => { impl<$($lt),*> Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -126,6 +128,8 @@ macro_rules! default_zeroed { ($type:ty) => { impl Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -144,6 +148,7 @@ unsafe impl Zeroable for Pad {} impl Default for Pad { fn default() -> Self { + // SAFETY: The wrapper contains only an array of bytes. unsafe { core::mem::zeroed() } } } @@ -170,6 +175,7 @@ unsafe impl Zeroable for Array {} impl Default for Array { fn default() -> Self { + // SAFETY: Every element admits the all-zero bit pattern through T: Zeroable. unsafe { core::mem::zeroed() } } } diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index be81eac5dfb743..34c65a0d8a0979 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -50,7 +50,7 @@ impl fmt::Display for Hex<'_> { pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { // SAFETY: memremap() of a boot-loader range creates a temporary mapping; it has no other // side effects. - let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as usize) }; if map.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index b4877490cad71a..d55b6385f6214f 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -184,6 +184,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 7b99daf4c1fce2..7b4c9de0e0fd96 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -46,18 +46,59 @@ impl ReportRing { pub(crate) struct Rings { quarantined: [[bool; 2]; 2], reports: super::reports::Reports, + telemetry: super::telemetry::Telemetry, + max_age_ns: u64, + suspended: bool, + discard_statistics: [bool; 2], + statistics_epoch: u64, } impl Rings { - pub(crate) fn new() -> Result { + pub(crate) fn new(graph: super::fw::variant::InitGraph) -> Result { Ok(Self { quarantined: [[false; 2]; 2], reports: super::reports::Reports::new()?, + telemetry: super::telemetry::Telemetry::default(), + max_age_ns: super::stats::max_age_ns(graph), + suspended: false, + discard_statistics: [false; 2], + statistics_epoch: 0, }) } + pub(super) fn invalidate_statistics(&mut self) { + self.telemetry.invalidate(); + // The packets have no epoch marker. Discard each role's already + // queued records before accepting a new baseline after the boundary. + self.discard_statistics = [true; 2]; + } + + pub(super) fn suspend_statistics(&mut self, suspended: bool) { + self.suspended = suspended; + self.invalidate_statistics(); + } + + pub(super) fn statistics_suspended(&self) -> bool { + self.suspended + } + + pub(super) fn observe_statistics_power(&mut self, powered: bool, epoch: u64) { + if !powered || epoch != self.statistics_epoch { + self.invalidate_statistics(); + } + self.statistics_epoch = epoch; + } + + pub(super) fn read_statistics(&self, now_ns: u64) -> super::telemetry::Readout { + self.telemetry.read(now_ns, self.max_age_ns) + } + pub(super) fn write_reports(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - self.reports.write(f) + self.reports.write(f)?; + let view = self.read_statistics(super::now_ns()); + writeln!(f, "qualified max_age_ns={} util_status={} util_host_ns={} util_samples={} pstate_status={} pstate_host_ns={} pstate_timestamp={} delta_ticks={:?}", + self.max_age_ns, view.util_status, view.util_host_ns, view.util_samples, + view.pstate_status, view.pstate_host_ns, view.pstate_timestamp, self.telemetry.delta_ticks) } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { @@ -69,12 +110,29 @@ impl Rings { } fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { - let quarantined = &mut self.quarantined[role as usize][ring as usize]; - if *quarantined { + if self.quarantined[role as usize][ring as usize] { return; } - if let Err(error) = Self::consume_report(init, role, ring, &mut self.reports) { - *quarantined = true; + let statistics = matches!(ring, ReportRing::Statistics); + let accept = !self.suspended + && !self.discard_statistics[role as usize] + && !self + .quarantined + .iter() + .any(|rings| rings[ReportRing::Statistics as usize]); + if let Err(error) = Self::consume_report( + init, + role, + ring, + &mut self.reports, + &mut self.telemetry, + self.max_age_ns, + accept, + ) { + self.quarantined[role as usize][ring as usize] = true; + if statistics { + self.invalidate_statistics(); + } dev_err!( dev.as_ref(), "G17 {:?} {:?} ring stopped: {:?}\n", @@ -82,6 +140,8 @@ impl Rings { ring, error ); + } else if statistics && !self.suspended { + self.discard_statistics[role as usize] = false; } } @@ -90,6 +150,9 @@ impl Rings { role: Role, ring: ReportRing, reports: &mut super::reports::Reports, + telemetry: &mut super::telemetry::Telemetry, + max_age_ns: u64, + accept_statistics: bool, ) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; @@ -104,8 +167,8 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - // Copy all words before returning the slot. Interpretation is a - // separate firmware qualification step, never a scheduling input. + // Decode only the complete copy, before returning the slot. + // Statistics never affect scheduling or power control. let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; @@ -113,7 +176,11 @@ impl Rings { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { - reports.record(role, words, super::now_ns()); + let host_ns = super::now_ns(); + reports.record(role, words, host_ns); + if accept_statistics { + telemetry.record(&words, host_ns, max_age_ns); + } } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 8245e8bca71e65..81d1d6fe43960d 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -68,7 +68,9 @@ static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); #[vtable] impl FenceOps for ExportFence { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } @@ -76,7 +78,13 @@ impl FenceOps for ExportFence { #[inline(never)] fn export_fallback(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; + let fence = contexts.new_fence( + 0, + ExportFence { + source: None, + scoped: false, + }, + )?; // SAFETY: Unique, initialized and still private. Only this constructor // publishes ExportFence objects; its payload is immutable after sharing. let ops = unsafe { @@ -86,8 +94,8 @@ fn export_fallback(contexts: &FenceContexts) -> Result> let previous = EXPORT_OPS.load(Ordering::Acquire); if previous == 0 { match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => {}, - Err(previous) if previous == ops => {}, + Ok(_) => {} + Err(previous) if previous == ops => {} Err(_) => return Err(EIO), } } else if previous != ops { @@ -96,21 +104,32 @@ fn export_fallback(contexts: &FenceContexts) -> Result> Ok(fence) } -/// Caller retains the leaf fence throughout lookup. Exact registered operations -/// identify the repr(C) FenceObject payload, not a driver or timeline name. -pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) - -> Option> -{ +/// Find the submission behind a leaf fence with the exact registered operations. +/// Those operations identify the repr(C) FenceObject payload, not its name. +/// +/// # Safety +/// +/// `raw` must be non-null and point to an initialized dma_fence. The caller +/// must retain a fence reference throughout this lookup. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) -> Option> { let expected = EXPORT_OPS.load(Ordering::Acquire); - if expected == 0 { return None; } + if expected == 0 { + return None; + } // SAFETY: The caller owns a live fence reference. ExportFence has a unique // timeline callback and nontrivial payload drop. No cast precedes the exact // registered vtable check. FenceObject begins with its dma_fence. let object = unsafe { - if (*raw).ops as usize != expected { return None; } + if (*raw).ops as usize != expected { + return None; + } &*(raw as *const FenceObject) }; - object.source.as_ref().filter(|source| source.firmware_source).cloned() + object + .source + .as_ref() + .filter(|source| source.firmware_source) + .cloned() } struct State { @@ -144,33 +163,50 @@ impl Submission { Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { - Arc::pin_init(pin_init!(Self { - state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), - remaining: AtomicU8::new(1), - error: AtomicI32::new(0), - owners: AtomicU8::new(1), - firmware_source, - retired: AtomicBool::new(false), - }), GFP_KERNEL) + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + retired: AtomicBool::new(false), + }), + GFP_KERNEL, + ) } pub(super) fn source_status(&self) -> Result { - let complete = self.remaining.load(Ordering::Acquire) == 0 - || self.retired.load(Ordering::Acquire); + let complete = + self.remaining.load(Ordering::Acquire) == 0 || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); - if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + if error != 0 { + return Err(super::dependency::inherited_error(Error::from_errno(error))); + } Ok(complete) } pub(super) fn same_vm(&self, status: &Arc) -> bool { - self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + self.state + .lock() + .work + .as_ref() + .is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { + self.state.lock().ready.clone() } - pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } fn set_ready(&self, ready: Fence) { - if !self.firmware_source { return; } + if !self.firmware_source { + return; + } let mut state = self.state.lock(); - if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + if state.work.is_some() && state.ready.is_none() { + state.ready = Some(ready); + } } fn note_error(&self, error: Error) -> bool { - self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + self.error + .compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() } fn release_state(&self) { let old = { @@ -182,15 +218,26 @@ impl Submission { fn owner_done(&self) { // Every hardware owner retains its Completion/Member through exact // retirement or stop. Exported fence FDs must not retain failed pins. - if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { + self.release_state(); + } } fn member_done(&self, fence: &UserFence, result: Result) { - if let Err(error) = result { self.note_error(error); } + if let Err(error) = result { + self.note_error(error); + } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { fence.set_error(Error::from_errno(error)); } - else { self.release_state(); } - if fence.scoped { signal_exported(fence); } else { fence.signal(); } + if error != 0 { + fence.set_error(Error::from_errno(error)); + } else { + self.release_state(); + } + if fence.scoped { + signal_exported(fence); + } else { + fence.signal(); + } } } } @@ -200,16 +247,28 @@ pub(crate) struct Aggregate(UserFence); impl Aggregate { fn source(&self) -> &Arc { // Only Outputs::publish constructs Aggregate, after setting the payload. - self.0.source.as_ref().expect("published aggregate has submission state") + self.0 + .source + .as_ref() + .expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { + Fence::from_fence(&self.0) } - fn fence(&self) -> Fence { Fence::from_fence(&self.0) } pub(crate) fn member(&self) -> Member { self.source().remaining.fetch_add(1, Ordering::Relaxed); self.source().owners.fetch_add(1, Ordering::Relaxed); - Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + Member { + aggregate: self.clone(), + complete: AtomicBool::new(false), + } } fn finish_enqueue(&self, result: Result) { - if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + if let Err(error) = result { + self.source() + .error + .store(error.to_errno(), Ordering::Release); + } self.source().member_done(&self.0, result); self.source().owner_done(); } @@ -220,22 +279,38 @@ pub(crate) struct Member { complete: AtomicBool, } impl Member { - pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } + pub(super) fn needs_source_ready(&self) -> bool { + self.aggregate.source().firmware_source + } pub(crate) fn work_state(&self) -> Result> { - self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) + self.aggregate + .source() + .state + .lock() + .work + .clone() + .ok_or(ECANCELED) } pub(super) fn is_source(&self, source: &Arc) -> bool { self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) } - pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } - pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.aggregate.source().set_ready(ready); + } + pub(super) fn note_error(&self, error: Error) -> bool { + self.aggregate.source().note_error(error) + } pub(super) fn note_retired(&self) { let source = self.aggregate.source(); - if source.firmware_source { source.retired.store(true, Ordering::Release); } + if source.firmware_source { + source.retired.store(true, Ordering::Release); + } } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.aggregate.source().member_done(&self.aggregate.0, result); + self.aggregate + .source() + .member_done(&self.aggregate.0, result); } } } @@ -278,7 +353,10 @@ mod tests { let member = aggregate.member(); aggregate.finish_enqueue(Ok(())); member.complete(Err(EIO)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + EIO.to_errno() + ); assert!(source.state.lock().ready.is_some()); drop(member); // Keep both the exported fence and source state alive, as userspace can. @@ -291,7 +369,10 @@ mod tests { let aggregate = aggregate(&contexts()); drop(aggregate.member()); aggregate.finish_enqueue(Err(ENOMEM)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + ENOMEM.to_errno() + ); } #[test] @@ -318,7 +399,9 @@ mod tests { } assert!(super::super::job::fence_status(&aggregate.fence()) > 0); assert_eq!(source.remaining.load(Ordering::Acquire), 0); - for member in &mut members { drop(member.take()); } + for member in &mut members { + drop(member.take()); + } assert_eq!(source.owners.load(Ordering::Acquire), 0); } } @@ -333,7 +416,9 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { sync.syncobj.replace_fence(Some(fence)); } + } else { + sync.syncobj.replace_fence(Some(fence)); + } } } @@ -346,10 +431,15 @@ pub(crate) struct Outputs { cumulative: Option, } impl Outputs { - pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, - input_count: usize, allocation_failure: &Fence) -> Result - { - if input_count > syncs.len() { return Err(EINVAL); } + pub(crate) fn new( + contexts: &FenceContexts, + mut syncs: KVec, + input_count: usize, + allocation_failure: &Fence, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); let fallback = match export_fallback(contexts) { Ok(fence) => fence, @@ -362,11 +452,23 @@ impl Outputs { return Err(error); } }; - Ok(Self { fallback: Some(fallback), syncs, input_count, - submission: None, result: Err(ECANCELED), cumulative }) - } - pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } - pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + Ok(Self { + fallback: Some(fallback), + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + cumulative, + }) + } + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + pub(crate) fn publish( + &mut self, + context: Arc, + firmware_source: bool, + ) -> Result { let source = Submission::new(context, firmware_source)?; let mut fence = self.fallback.take().ok_or(EINVAL)?; fence.source = Some(source); @@ -381,7 +483,10 @@ impl Outputs { } Ok(submission) } - pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } } impl Drop for Outputs { fn drop(&mut self) { @@ -393,11 +498,16 @@ impl Drop for Outputs { return; } if let Some(fallback) = self.fallback.take() { - if let Err(error) = self.result { fallback.set_error(error); } + if let Err(error) = self.result { + fallback.set_error(error); + } if let Some(bundle) = &mut self.cumulative { bundle.fail(&Fence::from_fence(&fallback)); } else { - install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&fallback), + ); } fallback.signal(); } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index c8763f40871d16..578bb7533f3c92 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -41,7 +41,11 @@ pub(crate) struct ComputeCompletion<'a> { /// Records in the compute completion ring that the host scans. pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { let slots = abi::COMPLETION_RING_SIZE / 0x40; - if slots < 0x800 { slots } else { 0x800 } + if slots < 0x800 { + slots + } else { + 0x800 + } }; /// Timestamp bits of a completion record's start and end words. @@ -203,7 +207,13 @@ struct BuildEnv<'a> { impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn fixed( + &self, + offset: u64, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -215,7 +225,13 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn dynamic( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -320,7 +336,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { + pub(super) fn initialize_compute_pool_blocks( + &self, + slot: u16, + page_list_va: u64, + blocks: usize, + ) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -682,23 +703,21 @@ impl InitData { /// Raw views of the memory publishers share without the device mutex: every work ring, /// the completion stamps, the QoS table and the recovery state word. They stay valid for /// this session. - pub(super) fn port_views( - &self, - mailbox: *const super::Mailbox, - ) -> Result { - use super::fw::{channels, queue::{DataMaster, Policy}}; + pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { + use super::fw::{ + channels, + queue::{DataMaster, Policy}, + }; let ring_of = |priority: u8, engine: DataMaster| -> Result { let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; - let word = |offset: usize| -> Result<*const AtomicU32> { - Ok(state.word(base + offset)?) - }; + let word = + |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( abi::work_ring_offset(channel), - channels::WORK_RING_SLOTS as usize - * core::mem::size_of::(), + channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ word(abi::ChannelState::CONSUMER)?, @@ -707,13 +726,14 @@ impl InitData { producer: word(abi::ChannelState::PRODUCER)?, }) }; - let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { - let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; - for (priority, view) in views.iter_mut().enumerate().skip(1) { - *view = ring_of(priority as u8, engine)?; - } - Ok(views) - }; + let rings = + |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; Ok(super::port::Views { compute: rings(DataMaster::Compute)?, tiling: rings(DataMaster::Tiling)?, @@ -775,13 +795,13 @@ impl InitData { } fence(Ordering::Acquire); let mut newest = None; - for slot in 0..COMPUTE_COMPLETION_SLOTS { + for (slot, seen_control) in seen.iter_mut().enumerate() { let completion = ComputeCompletion { object, offset: slot * 0x40, }; let control = completion.read(0)?; - seen[slot] = control; + *seen_control = control; if control != 0 { let end = completion.read(5)? & COMPLETION_TIME_MASK; if newest.is_none_or(|(_, latest)| end > latest) { @@ -814,6 +834,10 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. + #[expect( + clippy::too_many_arguments, + reason = "The firmware graph builder keeps its independent input objects explicit." + )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -891,7 +915,8 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = + self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -975,7 +1000,11 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + let pages = self.add(KernelObject::backing( + env.dev, + low::RENDER_PAGE_LIST_SIZE, + WB, + )?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -983,10 +1012,18 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing( + env.dev, + low::RENDER_RUN_LIST_SIZE, + WB, + )?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing( + env.dev, + abi::FREE_LIST_BLOCK_SIZE, + WB, + )?)?; } self.render_pages = pages; self.render_runs = runs; @@ -994,7 +1031,12 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; + self.alias( + first_block + block, + env.render, + low::render_block(block), + GS, + )?; } Ok(first_block) } @@ -1031,12 +1073,8 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; + let ready = + self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1427,15 +1465,15 @@ impl InitData { /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). pub(super) fn init_accepted(&self) -> Result { - Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + Ok(self + .status_word(Role::Primary, 0x14)? + .load(Ordering::Acquire) + == 1) } pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn status_read_words( @@ -1443,11 +1481,8 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1540,13 +1575,15 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset + .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1559,7 +1596,8 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? + let mask = self + .object(self.cluster)? .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { @@ -1567,13 +1605,19 @@ impl super::recovery::Memory for InitData { // address is unknown. Zero keeps ownership unknown and cannot override // firmware blame, but preserves the progress evidence for diagnostics. let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; + let monitor = self + .recovery_view_words::<6>( + abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, + ) + .ok() + .map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); + sources.reason = self + .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok() + .map(|words| words[0]); Ok(sources) } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c2962c45f7afc5..2a0a9c0a632224 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -57,8 +57,10 @@ mod recovery; mod replay; mod reports; mod runtime; +pub(crate) mod stats; pub(crate) mod status; mod teardown; +pub(crate) mod telemetry; mod timeout; mod validation; @@ -734,6 +736,16 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. Submissions published /// without the device mutex count as activity, and a clear that races one is abandoned. + fn observe_statistics_power(&mut self) { + if self.rings.statistics_suspended() { + return; + } + self.rings.observe_statistics_power( + self.bridge.regs.g17_core_power() & 0xf != 0, + self.port.statistics_epoch(), + ); + } + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { let submissions = self.port.submissions(); let submitted = submissions != self.seen_submissions; @@ -1130,6 +1142,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + stats: Arc, _capacity: Pin>>, _reports: Pin>>, _power: Pin>>, @@ -1256,6 +1269,13 @@ impl Gpu { directory.read_only_file(c"power", reports::PowerView(shared.clone())), GFP_KERNEL, )?; + let stats = Arc::new( + crate::stats::StatsSnapshot { + g17_reader: Some(stats::Reader(shared.clone())), + ..Default::default() + }, + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1264,6 +1284,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + stats, _capacity: capacity, _reports: reports, _power: power, @@ -1420,7 +1441,7 @@ impl Gpu { shared.feed.clone(), )?, seen_submissions: 0, - rings: channel::Rings::new()?, + rings: channel::Rings::new(shared.variant.graph)?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), @@ -1591,7 +1612,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - None + Some(self.stats.clone()) } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1722,7 +1743,7 @@ impl gpu::Gpu for Gpu { // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. let waited = now_ns().saturating_sub(admission_started); - let limit = job::TIMEOUT_MS as u64 * 1_000_000; + let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; if waited >= limit { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index ff95b07fa759ba..6f9d1c7d956c05 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -116,6 +116,7 @@ pub(super) struct Port { /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, demand: super::power::Demand, + statistics_epoch: AtomicU64, last_submission_ns: AtomicU64, submissions: AtomicU64, } @@ -172,6 +173,7 @@ impl Port { publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), demand, + statistics_epoch: AtomicU64::new(0), last_submission_ns: AtomicU64::new(0), submissions: AtomicU64::new(0), }), @@ -500,7 +502,21 @@ impl Port { self.demand.set_active(full); } + pub(super) fn statistics_epoch(&self) -> u64 { + self.statistics_epoch.load(Ordering::Acquire) + } + pub(super) fn note_gpu_activity(&self) { + // Observe gating before activity can wake the cores. A lockless + // publisher cannot clear the decoder under the device mutex; this + // epoch makes both the next drain and the next reader discard it. + if self + .regs + .as_ref() + .is_some_and(|regs| regs.g17_core_power() & 0xf == 0) + { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } self.last_submission_ns .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 7ff828b1e36023..368e95f144c240 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -35,7 +35,8 @@ const NATIVE_WAIT_STRIDE: usize = 0x60; // A submission can name every physical compute channel. Keep the complete // wait list inside its own cached slot, including the last retained slot. const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET - + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) + .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; @@ -280,7 +281,10 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + producer: self + .graph + .word(CURSORS + self.pointers.producer)? + .load(Ordering::Relaxed), consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) @@ -332,7 +336,9 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + self.graph + .dword(ITEMS + index * 8)? + .store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); self.graph @@ -685,7 +691,10 @@ impl Queue { /// never runs, or the failed command itself, whose ring entry it does not consume (a ray /// traversal's, on Neo). Only a kill of the queue's context gives them back. pub(crate) fn fault_pending(&self) -> bool { - self.fault_stamp.is_some() && self.quarantined && self.awaiting_witness && self.exit.is_none() + self.fault_stamp.is_some() + && self.quarantined + && self.awaiting_witness + && self.exit.is_none() } /// The stamp that proves retirement: the stored one, or a newer reported fault's. @@ -724,7 +733,9 @@ impl Queue { } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self + .context() + .is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { @@ -801,7 +812,10 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), + qos: self + .binding + .as_ref() + .map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -818,7 +832,8 @@ impl Queue { } pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + self.pool_release_pending() + && crate::g17::timeout::background_stalled(self.release_sent_ns, now) } /// A closed owner can still retain its last commands or their completion @@ -827,7 +842,12 @@ impl Queue { pub(crate) fn release_settling(&self) -> bool { // release_owner keeps the old owner while its last commands retire. // Waiting only once owner is None misses precisely that interval. - release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + release_pending( + self.released, + self.quarantined, + self.active.len, + self.retire_pending, + ) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -1029,8 +1049,13 @@ impl Queue { let front = self.active.front().ok_or(EIO)?; if let Some(ticket) = front.ticket { crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, - ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + crate::g17::trace::complete( + crate::g17::trace::COMPUTE, + ticket.qid, + ticket.ordinal, + &front.packet, + [timestamps[0], timestamps[1]], + ); } self.retire_pending = true; defer(Deferred::Retired( @@ -1251,7 +1276,9 @@ impl Queue { /// Validate the complete unpublished item budget before changing descriptors, /// work counts or queue producers. Leave one item empty across ring wrap. fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { return Err(ERANGE); } @@ -1260,14 +1287,18 @@ fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); return Err(EAGAIN); } - Ok(()) + Ok(()) } /// Retained commands are capacity, not a failed VM. Nothing has been written /// when this check runs, so the outer publisher can wait for retirement under /// its original admission deadline, just as it does for a full firmware ring. fn retained_capacity( - quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, + quarantined: bool, + released: bool, + active: usize, + replays: usize, + replay_front: bool, ) -> Result { if quarantined || released { return Err(EBUSY); @@ -1293,8 +1324,13 @@ mod admission_tests { let waits = crate::g17::kick::COMPUTE_CHANNELS; for slot in 0..RECORD_SLOTS { for index in 0..waits { - let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + let start = record_start( + slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + CACHED_RECORD_STRIDE, + ) + .unwrap(); assert!(start >= slot * CACHED_RECORD_STRIDE); assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); } @@ -1305,13 +1341,27 @@ mod admission_tests { fn record_window_cannot_overwrite_the_next_slot() { // The nineteenth wait used to cross the old slot boundary silently. let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; - assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert_eq!( + record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), + Err(EINVAL) + ); assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); - assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), - Ok(CACHED_RECORD_STRIDE - 1)); + assert_eq!( + record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1) + ); } #[test] @@ -1320,11 +1370,17 @@ mod admission_tests { for sequence in 0..IN_FLIGHT { active.push(sequence).unwrap(); } - assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Err(EAGAIN) + ); assert_eq!(active.front(), Some(&0)); assert_eq!(active.len, IN_FLIGHT); assert_eq!(active.pop(), Some(0)); - assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Ok(()) + ); assert_eq!(active.front(), Some(&1)); } @@ -1332,28 +1388,114 @@ mod admission_tests { fn replay_front_and_terminal_ownership_remain_distinct() { assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, IN_FLIGHT, 1, true), + Err(EAGAIN) + ); for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + assert_eq!( + retained_capacity(quarantined, released, 0, 1, true), + Err(EBUSY) + ); } } #[test] fn item_refusal_is_retryable_and_preserves_the_empty_slot() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 10, + consumer: 10, + count: n + }, + true + ), + Ok(()) + ); } #[test] fn bad_item_cursors_are_not_backpressure() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); - assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 0, + count: 0 + }, + false + ), + Err(EIO) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n, + consumer: 0, + count: n + }, + false + ), + Err(ERANGE) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: n, + count: n + }, + true + ), + Err(ERANGE) + ); } #[test] @@ -1389,7 +1531,9 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { return false; } + if self.growing { + return false; + } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1454,8 +1598,12 @@ impl Queue { } context.mark_published(); context.note_compute_binding(true); - crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), - if same_vm { 2 } else { 1 }); + crate::g17::trace::bind( + crate::g17::trace::COMPUTE, + self.qid(), + context.id(), + if same_vm { 2 } else { 1 }, + ); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1523,7 +1671,8 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated + && self.pool.populated() && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } @@ -1619,9 +1768,8 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); self.release_sent_ns = Some(crate::g17::now_ns()); @@ -1636,7 +1784,9 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { return Ok(None); } + if self.growing { + return Ok(None); + } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1645,11 +1795,18 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let alias = self + .binding + .as_mut() + .and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let descriptors = self + .binding + .as_mut() + .and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1664,7 +1821,9 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { return Err(EAGAIN); } + if !self.can_rearm() { + return Err(EAGAIN); + } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1678,27 +1837,50 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { return Err(EAGAIN); } - if self.quarantined || self.released || self.exit.is_some() - || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + pub(crate) fn begin_growth( + &mut self, + init: &crate::g17::initdata::InitData, + blocks: usize, + ) -> Result { + if self.growing { + return Err(EAGAIN); + } + if self.quarantined + || self.released + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + || self.pool_phase != PoolPhase::Populated + { + return Err(EAGAIN); + } if self.installed { - let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; + } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth( + &mut self, + init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, blocks: usize) -> Result { - if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + alias: &mut Option, + blocks: usize, + ) -> Result { + if !self.growing || blocks < self.growth_blocks { + return Err(EAGAIN); + } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1706,7 +1888,9 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { return Err(EINVAL); } + if alias.is_none() { + return Err(EINVAL); + } let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. @@ -1718,11 +1902,19 @@ impl Queue { self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + Ok(Vacated { + _alias: old_alias, + _pages: pages.take(), + _descriptors: None, + _ring: None, + }) } pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + self.growing.then_some(Growth { + qid: self.qid(), + phase: self.pool_phase, + }) } /// Transfer an abandoned builder's release back to the event worker. Do not @@ -1885,7 +2077,9 @@ fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Res if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { return Err(EINVAL); } - slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) + slot.checked_mul(stride) + .and_then(|start| start.checked_add(offset)) + .ok_or(EINVAL) } impl Graph { @@ -1957,8 +2151,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -2023,8 +2216,12 @@ impl Queue { let first = !self.installed && self.ordinal == 0; // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. - if first && !native.is_empty() { return Err(EAGAIN); } - if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } + if first && !native.is_empty() { + return Err(EAGAIN); + } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { + return Err(EINVAL); + } let entry_signal = first || dependencies .iter() @@ -2081,7 +2278,9 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let cached = self + .config_cache + .and_then(|(config, at)| (at == epoch).then_some(config)); let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; @@ -2155,10 +2354,16 @@ impl Queue { let cursors = self.graph.cursors()?; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { - let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), - packet.order.sequence); - let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, record)?; + let record = wait.bytes( + crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence, + ); + let address = self.graph.record( + record_slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + record, + )?; wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; @@ -2207,8 +2412,11 @@ impl Queue { packet.mark_published(); visible = true; }) - })().or_else(|error| { - if !visible { self.pool.cancel_unpublished(target)?; } + })() + .or_else(|error| { + if !visible { + self.pool.cancel_unpublished(target)?; + } Err(error) })?; visible = true; @@ -2336,8 +2544,13 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity(self.quarantined, self.released, self.active.len, - self.replays.len, replay_front)?; + retained_capacity( + self.quarantined, + self.released, + self.active.len, + self.replays.len, + replay_front, + )?; if !packet.context.is_current() || !self .binding @@ -2425,17 +2638,30 @@ impl Queue { u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + Ok(cursors) if cursors.count != 0 => writeln!( + f, + "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, + cursors.count - 1 + ), _ => writeln!(f, "items_unavailable=1"), }?; - writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), - u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + writeln!( + f, + "compute_settling qid={} release={} pool_release={}", + self.qid(), + u8::from(self.release_settling()), + u8::from(self.pool_release_pending()) + )?; if let Some(binding) = self.binding.as_ref() { - binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + binding + .context + .write_capacity(f, "compute", usize::from(self.qid()))?; } if let Some(previous) = self.previous.as_ref() { - previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + previous + .context + .write_capacity(f, "compute_previous", usize::from(self.qid()))?; } Ok(()) } @@ -2544,10 +2770,18 @@ impl Queue { /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { - self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready - && !self.growing && !self.quarantined && !self.awaiting_witness - && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 - && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + self.claims == 0 + && self.installed + && self.owner.is_some() + && self.retirement_ready + && !self.growing + && !self.quarantined + && !self.awaiting_witness + && self.exit.is_none() + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.pool_phase == PoolPhase::Populated && !self.source_hint.has_readers(&self.source_readers) } @@ -2560,7 +2794,8 @@ impl Queue { impl Queue { pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + self.pool_phase == PoolPhase::Vacant + && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } } @@ -2571,19 +2806,35 @@ impl Queue { /// caller validates the current preparation epoch and cancels only a true /// result's never-published QID; installed rings and identities stay intact. pub(crate) fn rollback_unpublished( - &mut self, owner: u64, context: &Arc, + &mut self, + owner: u64, + context: &Arc, ) -> Result<(bool, Option)> { - if self.owner != Some(owner) || !self.unpublished_binding - || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.retire_pending || self.failure_status_pending + if self.owner != Some(owner) + || !self.unpublished_binding + || self + .binding + .as_ref() + .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing + || self.quarantined + || self.awaiting_witness + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.retire_pending + || self.failure_status_pending || self.pool_phase != PoolPhase::Populated - || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + || self + .previous + .as_ref() + .is_some_and(|previous| previous.publication.is_some()) { return Ok((false, None)); } - if !self.installed { return Ok((true, None)); } + if !self.installed { + return Ok((true, None)); + } let mut retired = None; if let Some(previous) = self.previous.as_ref() { self.graph.set_owner(&previous.context)?; @@ -2604,37 +2855,61 @@ impl Queue { } } - impl Queue { /// Same immutable singleton identity and current physical epoch. - pub(crate) fn has_input_source(&self, source: &Arc, - status: &Arc, epoch: u64) -> bool - { - !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| - active.ticket.is_some_and(|ticket| ticket.epoch == epoch) - && Arc::ptr_eq(active.packet.completion.status(), status) - && active.packet.completion.is_input_source(source)) - } - - pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, - status: &Arc, epoch: u64) -> Result> - { + pub(crate) fn has_input_source( + &self, + source: &Arc, + status: &Arc, + epoch: u64, + ) -> bool { + !self.quarantined + && self.replays.len == 0 + && self.active.iter().any(|active| { + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source) + }) + } + + pub(crate) fn source_dependency( + &mut self, + source: &crate::g17::sync_input::Source, + status: &Arc, + epoch: u64, + ) -> Result> { for active in self.active.iter() { - if !active.packet.completion.is_input_source(&source.submission) { continue; } - if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } - if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + if !active.packet.completion.is_input_source(&source.submission) { + continue; + } + if !Arc::ptr_eq(active.packet.completion.status(), status) { + return Err(EINVAL); + } + if self.quarantined || self.replays.len != 0 { + return Err(EAGAIN); + } let ticket = active.ticket.ok_or(EAGAIN)?; - if ticket.epoch != epoch { return Err(EAGAIN); } + if ticket.epoch != epoch { + return Err(EAGAIN); + } self.valid_ticket(active, &ticket)?; - if active.packet.cancel_requested() { return Err(ECANCELED); } + if active.packet.cancel_requested() { + return Err(ECANCELED); + } source.submission.source_status()?; - let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), - crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let dependency = crate::g17::fw::kick::KickDependency::new( + self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, + ) + .ok_or(EIO)?; let hold = self.source_hint.hold(&self.source_readers)?; - return Ok(Some((active.published, dependency, + return Ok(Some(( + active.published, + dependency, self.graph.support.gpu_va() + STAMP as u64, - u32::from(self.qid()), hold))); + u32::from(self.qid()), + hold, + ))); } Ok(None) } @@ -2650,3 +2925,12 @@ impl Queue { } } } + +/// Publication, dependency, stamp address, queue and the retained source hold. +pub(crate) type SourceDependency = ( + u64, + crate::g17::fw::kick::KickDependency, + u64, + u32, + crate::g17::sync_input::Hold, +); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 540f7cc76e8e75..2436cfd33ed35b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -248,8 +248,7 @@ impl Pair { let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = - kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -336,7 +335,11 @@ impl Pair { self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self + .replay + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { return Err(EIO); } self.replay.pop(); @@ -363,14 +366,20 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) + || self + .replay + .front() + .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, attachments, ray_query, .. + pass, + attachments, + ray_query, + .. } = &packet.command else { return Err(EINVAL); @@ -465,11 +474,9 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = self.memory.write_descriptors( - &args, - attachments, - packet.context.id().try_into()?, - ); + let result = + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -669,7 +676,9 @@ impl Pair { .filter(|_| self.recovery_generation == epoch); let present = [0, 1].map(|stage| { let mut config = configs[stage]; - let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + let reload = kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)); if config.completion_seed_valid != 0 && self.installed[stage] && held.is_some_and(|held| held[stage] == reload) @@ -687,7 +696,10 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, ray_query, .. } = &packet.command else { + let Validated::Render { + pass, ray_query, .. + } = &packet.command + else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -790,10 +802,7 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); @@ -948,7 +957,11 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + if self + .registered + .is_some_and(|class| class != policy.priority()) + && self.in_flight() + { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -986,8 +999,12 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; - crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, - packet); + crate::g17::trace::publish( + crate::g17::trace::RENDER, + self.qids()[1], + self.ordinal as u32, + packet, + ); host.note_submission() })(); if let Err(error) = publish { @@ -1053,9 +1070,14 @@ impl Pair { active.ticket.ordinal, pass.width, pass.height, pass.layers, target::Layout::new(pass, self.clusters, self.variant.programs)); } - pr_warn!("G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", - active.ticket.ordinal, active.outer_published, active.packet.context.id(), - active.packet.order.sequence, observation::snapshot(&self.memory, active.ticket.ordinal)); + pr_warn!( + "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, + active.outer_published, + active.packet.context.id(), + active.packet.order.sequence, + observation::snapshot(&self.memory, active.ticket.ordinal) + ); } } @@ -1063,7 +1085,12 @@ impl Pair { /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it /// on its pass record instead of waiting for stamps no later event will move. - pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result { + pub(crate) fn note_stamp_wait( + &mut self, + stamps: [u32; 2], + state: u32, + now: u64, + ) -> Result { if self.quarantined || self.prepared.is_some() || state != 0 { return Ok(false); } @@ -1074,7 +1101,10 @@ impl Pair { return Ok(false); } let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) { + if stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { front.stamp_wait_since = None; return Ok(false); } @@ -1113,14 +1143,25 @@ impl Pair { /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair /// without a completion notification, which may never come for such a group. pub(crate) fn stamp_waiting(&self) -> bool { - self.active.front().is_some_and(|front| front.stamp_wait_since.is_some()) + self.active + .front() + .is_some_and(|front| front.stamp_wait_since.is_some()) } /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed /// it and consumed its ring entries while the stamps lag, it retires on its pass record now /// instead of failing. Failing would give up on the stamps as well. - pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc, stamps: [u32; 2], state: u32) -> Result { - if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) { + pub(crate) fn expire_stamp_wait( + &mut self, + packet: &Arc, + stamps: [u32; 2], + state: u32, + ) -> Result { + if !self + .active + .front() + .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) + { return Ok(false); } if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { @@ -1183,7 +1224,9 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory.manager.note_required(required, crate::g17::now_ns()); + self.memory + .manager + .note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1246,7 +1289,10 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + let ordinal = self + .active + .iter() + .map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1278,7 +1324,11 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { + if self + .active + .iter() + .any(|active| active.packet.completion.spared()) + { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1394,8 +1444,13 @@ impl Pair { front.flist_completed = true; } crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, - &front.packet, [timestamps[0], timestamps[3]]); + crate::g17::trace::complete( + crate::g17::trace::RENDER, + qids[1], + ordinal as u32, + &front.packet, + [timestamps[0], timestamps[3]], + ); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1455,7 +1510,13 @@ impl Pair { // A torn sample is no proof; a later poll or the timeout looks again. let witness = |executed| { matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + retirement::abandoned( + &active.ticket, + snapshot, + stamps, + state, + executed + ), Ok(true) ) }; @@ -1464,7 +1525,11 @@ impl Pair { // another queue waited on never: the wait ended when firmware retired it. let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) || active.packet.completion.has_firmware_consumer(); - Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) + Ok(( + unexecuted && !lost, + !unexecuted && witness(true), + unexecuted && lost, + )) })() .unwrap_or_else(|error: Error| { failure = Some(error); @@ -1728,7 +1793,8 @@ impl Pair { /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer /// walks that context's work list, so the commands it never unlinked can be released. fn context_killed(&self) -> bool { - crate::g17::queue::compute::exit::State::read(&self.memory.context).is_ok_and(|state| state.killed != 0) + crate::g17::queue::compute::exit::State::read(&self.memory.context) + .is_ok_and(|state| state.killed != 0) } /// The pair's context may be killed although its commands were not retired: they all ran @@ -1867,15 +1933,13 @@ impl Pair { // own even with successors queued, so the pair is not quarantined into a sticky EIO. It // fails with ECANCELED, which spared work never reports as device loss; consecutive // abandoned spared successors are retired the same way. - let front_spared = self.active.front().is_some_and(|a| { - Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() - }); + let front_spared = self + .active + .front() + .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); if !self.quarantined && (self.active.len() == 1 || front_spared) { let mut retired = false; - loop { - let Some(active) = self.active.front() else { - break; - }; + while let Some(active) = self.active.front() { let spared = active.packet.completion.spared(); if retired && !spared { break; @@ -2146,12 +2210,25 @@ impl Pair { } /// The render prefix of `packet` on this pair, as the registry collects it /// for an owner whose only pair this is. - pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + pub(crate) fn own_dependencies( + &self, + owner: u64, + packet: &Arc, + ) -> Result { let mut dependencies = RenderDependencies::EMPTY; let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; + self.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + false, + )?; for dependency in frontier.as_slice() { - *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } if dependencies.render_count != 0 @@ -2165,7 +2242,10 @@ impl Pair { &mut tiling, )?; for dependency in tiling.as_slice() { - *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.tiling_count += 1; } } @@ -2178,8 +2258,12 @@ impl Pair { /// not left the pair yet. pub(crate) fn switch_in_flight(&self) -> Option { let (from, first) = self.switch?; - (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) - .then_some(from) + (self.replay.len() != 0 + || self + .active + .iter() + .any(|active| active.ticket.ordinal == first)) + .then_some(from) } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { @@ -2344,7 +2428,9 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { self.graph_started = false; } + if foreign { + self.graph_started = false; + } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -2395,4 +2481,3 @@ mod completion_noted_tests { assert!(completion_noted([3, 68], both)); } } - diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index c1985c72f6ec1f..d65c40c7cfadee 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -41,6 +41,10 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { + #[expect( + clippy::too_many_arguments, + reason = "The publication lane describes independent fields of the firmware wire record." + )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -94,7 +98,9 @@ impl<'a> Lane<'a> { push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph + .queues + .word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index b67512f7eec6bc..246aa5f02edbfa 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -4,7 +4,10 @@ //! treating another firmware's pstate report as an achieved GPU frequency. //! Writers and readers hold the device mutex; reads never power the GPU up. -use super::{fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, Role}; +use super::{ + fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + Role, +}; use kernel::prelude::*; #[derive(Clone, Copy, Default)] @@ -40,10 +43,25 @@ impl Reports { } pub(super) fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified")?; - for (index, sample) in self.samples.iter().enumerate().filter(|(_, s)| s.count != 0) { - writeln!(f, "role={} tag={:#x} count={} host_ns={} words={:08x?}", - index / 32, index % 32, sample.count, sample.host_ns, sample.words)?; + writeln!( + f, + "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified" + )?; + for (index, sample) in self + .samples + .iter() + .enumerate() + .filter(|(_, s)| s.count != 0) + { + writeln!( + f, + "role={} tag={:#x} count={} host_ns={} words={:08x?}", + index / 32, + index % 32, + sample.count, + sample.host_ns, + sample.words + )?; } Ok(()) } @@ -54,15 +72,29 @@ pub(super) struct View(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for View { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; - writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return writeln!(f, "online=0"); + }; + firmware.observe_statistics_power(); + writeln!( + f, + "online=1 graph={:?} sampled_ns={}", + self.0.variant.graph, + super::now_ns() + )?; write_power(firmware, f)?; firmware.rings.write_reports(f) } } -fn write_power(firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { +fn write_power( + firmware: &super::Firmware, + f: &mut kernel::fmt::Formatter<'_>, +) -> kernel::fmt::Result { + if firmware.rings.statistics_suspended() { + return writeln!(f, "power_management=firmware suspended=1"); + } let core_power = firmware.bridge.regs.g17_core_power(); writeln!(f, "power_management=firmware sampled_ns={} core_power={:#x} cores_powered={} host_hold={} idle_hold_ms={}", super::now_ns(), core_power, core_power & 0xf != 0, firmware.hold.active(), @@ -76,7 +108,9 @@ pub(super) struct PowerView(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for PowerView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + let Some(firmware) = state.as_deref() else { + return writeln!(f, "online=0"); + }; write_power(firmware, f) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 55d517a60ade8b..3e779bba4fbbfb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -38,9 +38,11 @@ impl event::Transport for Service<'_> { } fn statistics(&mut self, role: Role) { let firmware = &mut *self.firmware; + firmware.observe_statistics_power(); firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + firmware.observe_statistics_power(); } fn drain( &mut self, @@ -298,6 +300,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + self.rings.invalidate_statistics(); if self.recovery.restart_limit(super::now_ns()) { pr_warn!("G17: restart storm limit reached\n"); return Err(ETIMEDOUT); @@ -318,6 +321,7 @@ impl Firmware { .firmware .port .set_generation(service.firmware.recovery.generation()); + service.firmware.rings.invalidate_statistics(); let result = result?; if let Some(error) = result.target_error { dev_warn!( diff --git a/drivers/gpu/drm/asahi/g17/stats.rs b/drivers/gpu/drm/asahi/g17/stats.rs new file mode 100644 index 00000000000000..4541ab8d402202 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/stats.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A sysfs reader owns the session, never a pointer into detachable Firmware. + +use super::{fw::variant::InitGraph, telemetry::Readout, Shared}; +use core::sync::atomic::Ordering; +use kernel::sync::Arc; + +pub(crate) struct Reader(pub(super) Arc); + +impl Reader { + pub(crate) fn read(&self) -> Readout { + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return Readout::default(); + }; + if self.0.crashed.load(Ordering::Acquire) || firmware.recovery.pending() { + firmware.rings.invalidate_statistics(); + return firmware.rings.read_statistics(super::now_ns()); + } + firmware.observe_statistics_power(); + firmware.rings.read_statistics(super::now_ns()) + } + + pub(crate) fn suspend(&self, suspended: bool) { + if let Some(firmware) = self.0.state.lock().as_deref_mut() { + firmware.rings.suspend_statistics(suspended); + } + } +} + +/// Two nominal periods is a conservative host age/window policy. The firmware +/// does not guarantee a packet duration or a pstate report cadence. Logical +/// pstate validity is controlled by records and epoch boundaries, not this age. +pub(super) const fn max_age_ns(graph: InitGraph) -> u64 { + match graph { + InitGraph::V25G83 => 16_000_000, + InitGraph::G17P => 32_000_000, + } +} diff --git a/drivers/gpu/drm/asahi/g17/telemetry.rs b/drivers/gpu/drm/asahi/g17/telemetry.rs new file mode 100644 index 00000000000000..a048665dcab5cd --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/telemetry.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Qualified V25G83/V26A428 statistics, decoded from complete copied records. +//! No record measures busy time, power, temperature or achieved frequency. +//! A/B table selection is absent, so neither nominal candidate is exported as +//! a requested or achieved frequency. All state is protected by the device mutex. + +pub(super) const UNAVAILABLE: u32 = 0; +pub(super) const VALID: u32 = 1; +pub(super) const STALE: u32 = 2; + +/// C read callback result; independent capabilities never imply one another. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub(crate) struct Readout { + pub(crate) util_status: u32, + pub(crate) pstate_status: u32, + pub(crate) tiler: u32, + pub(crate) renderer: u32, + pub(crate) device: u32, + pub(crate) pstate: u32, + pub(crate) util_host_ns: u64, + pub(crate) pstate_host_ns: u64, + pub(crate) util_samples: u64, + pub(crate) pstate_timestamp: u64, +} + +const _: () = { + assert!(core::mem::offset_of!(Readout, util_host_ns) == 24); + assert!(core::mem::size_of::() == 56); +}; + +#[derive(Default)] +pub(super) struct Telemetry { + view: Readout, + sums: [u64; 3], + window_ns: u64, + timestamp: Option, + /// Diagnostic interval only, never used as busy time. + pub(super) delta_ticks: Option, +} + +impl Telemetry { + /// End an epoch. Preserve the reason that formerly valid data is stale, + /// but never carry an average or a timestamp baseline into a new epoch. + pub(super) fn invalidate(&mut self) { + if self.view.util_status != UNAVAILABLE { + self.view.util_status = STALE; + } + if self.view.pstate_status != UNAVAILABLE { + self.view.pstate_status = STALE; + } + self.view.util_samples = 0; + self.timestamp = None; + self.delta_ticks = None; + } + + /// `max_age_ns` is a host policy (two nominal periods), not a packet duration. + /// Every role contributes one equally weighted sample to the same bounded + /// collection window. Reads do not consume samples or change the mean. + pub(super) fn record(&mut self, words: &[u32; 18], host_ns: u64, max_age_ns: u64) { + match words[0] { + 0x13 => { + let values = [words[3], words[4], words[6]]; + if values.iter().any(|&value| value > 100) { + self.view.util_status = UNAVAILABLE; + self.view.util_samples = 0; + return; + } + if self.view.util_samples == 0 + || host_ns.saturating_sub(self.window_ns) >= max_age_ns + || host_ns < self.window_ns + || self.view.util_samples >= u64::MAX / 100 + { + self.sums = [0; 3]; + self.view.util_samples = 0; + self.window_ns = host_ns; + } + for (sum, value) in self.sums.iter_mut().zip(values) { + *sum += u64::from(value); + } + self.view.util_samples += 1; + self.view.util_host_ns = host_ns; + self.view.util_status = VALID; + self.view.tiler = (self.sums[0] / self.view.util_samples) as u32; + self.view.renderer = (self.sums[1] / self.view.util_samples) as u32; + self.view.device = (self.sums[2] / self.view.util_samples) as u32; + } + 0x10 => { + let timestamp = u64::from(words[1]) | (u64::from(words[2]) << 32); + if self.timestamp.is_some_and(|last| timestamp < last) { + self.invalidate(); + self.view.pstate_status = UNAVAILABLE; + return; + } + self.delta_ticks = self.timestamp.map(|last| timestamp - last); + self.timestamp = Some(timestamp); + self.view.pstate_host_ns = host_ns; + self.view.pstate_timestamp = timestamp; + self.view.pstate_status = if words[6] <= 10 { VALID } else { UNAVAILABLE }; + self.view.pstate = words[6]; + } + _ => {} + } + } + + pub(super) fn read(&self, now_ns: u64, max_age_ns: u64) -> Readout { + let mut view = self.view; + let expired = |received| now_ns < received || now_ns - received >= max_age_ns; + // The oldest contributing sample bounds the whole averaging window. + if view.util_status == VALID && expired(self.window_ns) { + view.util_status = STALE; + } + // A pstate report describes a state transition, with no qualified + // cadence. Its validity ends on replacement or epoch invalidation, + // rather than an invented periodic-report deadline. + view + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_telemetry))] +#[cfg(any(test, CONFIG_KUNIT))] +mod tests { + use super::*; + + const AGE: u64 = 16_000_000; + const U1: [u32; 18] = [ + 0x13, 0, 0, 25, 75, 0xdeadbeef, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U2: [u32; 18] = [ + 0x13, 0, 0, 35, 85, 0x12345678, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U3: [u32; 18] = [0x13, 0, 0, 101, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P1: [u32; 18] = [0x10, 0x10, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2A: [u32; 18] = [0x10, 0x1000, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2B: [u32; 18] = [0x10, 0x2500, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P3: [u32; 18] = [ + 0x10, 0x3000, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + #[test] + fn spec_u1_u2_u3_both_firmware_policies() { + for age in [AGE, 2 * AGE] { + let mut stats = Telemetry::default(); + assert_eq!(stats.read(0, age), Readout::default()); + stats.record(&U1, 100, age); + let view = stats.read(100, age); + assert_eq!((view.tiler, view.renderer, view.device), (25, 75, 50)); + assert_eq!(view.util_status, VALID); + stats.record(&U2, 101, age); // May be the other role; no deduplication. + let view = stats.read(101, age); + assert_eq!((view.tiler, view.renderer, view.device), (30, 80, 57)); + assert_eq!(view.util_samples, 2); + stats.record(&U3, 102, age); + assert_eq!(stats.read(102, age).util_status, UNAVAILABLE); + stats.record(&U2, 103, age); + assert_eq!(stats.read(103, age).device, 65); + } + } + + #[test] + fn spec_p1_p2_p3_and_regression() { + let mut stats = Telemetry::default(); + stats.record(&P1, 100, AGE); + let view = stats.read(100, AGE); + assert_eq!(view.pstate, 6); + assert_eq!(view.pstate_status, VALID); + assert_eq!(view.pstate_timestamp, 0x1_0000_0010); + assert_eq!(stats.delta_ticks, None); // First sample is only a baseline. + stats.invalidate(); + stats.record(&P2A, 101, AGE); + stats.record(&P2B, 102, AGE); + assert_eq!(stats.delta_ticks, Some(5376)); + assert_eq!( + stats.delta_ticks.unwrap() * 1_000_000_000 / 24_000_000, + 224_000 + ); + stats.record(&P3, 103, AGE); + assert_eq!(stats.read(103, AGE).pstate_status, UNAVAILABLE); + stats.record(&U1, 104, AGE); + stats.record(&P2A, 105, AGE); // Backwards, including across role drain order. + assert_eq!(stats.read(105, AGE).pstate_status, UNAVAILABLE); + assert_eq!(stats.read(105, AGE).util_status, STALE); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 106, AGE); // New epoch establishes a new baseline. + assert_eq!(stats.read(106, AGE).pstate_status, VALID); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 107, AGE); // Equal timestamps are non-regressing. + assert_eq!(stats.delta_ticks, Some(0)); + } + + #[test] + fn age_epoch_and_zero_are_independent() { + let mut stats = Telemetry::default(); + stats.record(&U1, 100, AGE); + stats.record(&P1, 200, AGE); + assert_eq!(stats.read(99, AGE).util_status, STALE); + assert_eq!(stats.read(100 + AGE - 1, AGE).util_status, VALID); + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, VALID); + assert_eq!(stats.read(200 + AGE, AGE).pstate_status, VALID); + assert_eq!(stats.read(u64::MAX, AGE).pstate_status, VALID); + stats.record(&U2, 100 + AGE, AGE); + assert_eq!(stats.read(100 + AGE, AGE).device, 65); // Old window was discarded. + stats.invalidate(); // Suspend, restart, quarantine, gate or teardown. + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, STALE); + let mut zero = [0; 18]; + zero[0] = 0x13; + stats.record(&zero, 101 + AGE, AGE); + let view = stats.read(101 + AGE, AGE); + assert_eq!(view.util_status, VALID); + assert_eq!((view.tiler, view.renderer, view.device), (0, 0, 0)); + zero[0] = 0x10; + stats.record(&zero, 102 + AGE, AGE); + assert_eq!(stats.read(102 + AGE, AGE).pstate, 0); + assert_eq!(stats.read(102 + AGE, AGE).pstate_status, VALID); + } + + #[test] + fn invalid_components_and_selectors_never_clamp() { + for word in [3, 4, 6] { + let mut stats = Telemetry::default(); + let mut words = U1; + words[word] = u32::MAX; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).util_status, UNAVAILABLE); + } + for selector in [11, 15, 16, u32::MAX] { + let mut stats = Telemetry::default(); + let mut words = P1; + words[6] = selector; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).pstate_status, UNAVAILABLE); + } + let mut stats = Telemetry::default(); + for tag in [0x11, 0x12, 0x18, 0x1d, u32::MAX] { + let mut words = U1; + words[0] = tag; + stats.record(&words, 1, AGE); + } + assert_eq!(stats.read(1, AGE), Readout::default()); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 72238159d04752..38d499b0cf4dad 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,9 +41,7 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, - memacct, - mmu, + file, memacct, mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -133,9 +131,6 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -466,7 +461,6 @@ pub(crate) fn prepare_new_object(gem: &Object) -> Result { Ok(()) } - #[cfg(CONFIG_DRM_ASAHI_G15)] extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 2ff7ab369c43bd..1cabe24d20d687 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -323,7 +323,7 @@ pub(crate) trait Gpu: Send + Sync { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } if self.soft_faults() && self.supports_soft_faults() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -1132,7 +1132,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1182,20 +1182,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 4c8a4e0fcc370f..b66a110c733190 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -137,11 +137,11 @@ impl<'a> InitDataBuilder::ver<'a> { continue; } - let f_khz = (ps.freq_hz / 1000) as u64; - let v_max = ps.max_volt_mv() as u64; + let f_khz = u64::from(ps.freq_hz / 1000); + let v_max = u64::from(ps.max_volt_mv()); t1.push( - (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + (1000000000 * u64::from(curve_cfg.t1_coef) / (f_khz * v_max)) .try_into() .unwrap(), GFP_KERNEL, @@ -149,7 +149,7 @@ impl<'a> InitDataBuilder::ver<'a> { for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { t3[j].push( - (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + (u64::from(t3_coef) * 1000000100 * u64::from(*scale) / (f_khz * v_max * 6)) .try_into() .unwrap(), GFP_KERNEL, @@ -423,7 +423,7 @@ impl<'a> InitDataBuilder::ver<'a> { #[ver(G >= G14X)] fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), #[ver(V < V13_0B4)] - fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + fast_die0_sensor_present: U64(u64::from(cfg.fast_die0_sensor_present)), unk_163c: 1, unk_3644: 0, hws1 <- Self::hw_shared1(cfg), @@ -487,10 +487,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataA::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_a.len() { @@ -502,8 +504,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_a.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_a).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); break; @@ -670,10 +674,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataB::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_b.len() { @@ -685,8 +691,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_b.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { - if sla[i] != dyncfg.hw_data_b[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_b).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); break; @@ -824,10 +832,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_globals.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::Globals::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_globals.len() { @@ -839,8 +849,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_globals.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { - if sla[i] != dyncfg.hw_globals[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_globals).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); break; @@ -971,12 +983,12 @@ impl<'a> InitDataBuilder::ver<'a> { num_entries: usize, ) -> raw::UatLevelInfo { raw::UatLevelInfo { - index_shift: index_shift as _, + index_shift: index_shift as u8, unk_1: 14, unk_2: 14, unk_3: 8, unk_4: 0x4000, - num_entries: num_entries as _, + num_entries: num_entries as u16, unk_8: U64(1), unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), index_mask: U64(((num_entries - 1) << index_shift) as u64), diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 5f0ab2b4aea4f5..79071cbc434a04 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -71,7 +71,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) + x = in(reg) (u64::from(asid) << 48) ); } } @@ -85,7 +85,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -147,7 +147,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= (exp as u64) << 44; + val |= u64::from(exp) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs index cbdb5de62e9218..42cd40b8220f6d 100644 --- a/drivers/gpu/drm/asahi/microseq.rs +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -45,7 +45,7 @@ impl Builder { pub(crate) fn add(&mut self, op: T) -> Result { let off = self.ops.len(); let p: *const T = &op; - let p: *const u8 = p as *const u8; + let p: *const u8 = p.cast::(); // SAFETY: Microseq operations always have no padding bytes, so it is safe to // access them as a byte slice. let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 79dc1d50755d8b..50e06d3138d5ed 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -108,8 +108,6 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; @@ -1034,6 +1032,7 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1263,14 +1262,17 @@ impl TimestampVmap { /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused /// when the destination is prepared, and ignored here. pub(crate) fn write(&self, offset: usize, value: u64) { - if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { return; } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. let _guard = self.0.owner().timestamp_lock.lock(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { // SAFETY: All eight bytes were checked above; the vmap pins the backing. @@ -1493,7 +1495,7 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); uat_inner.handoff_lock(); @@ -1736,7 +1738,7 @@ impl UatShared { /// Writes back context-table slot `idx` after it changed, where this SoC needs it. fn publish_slot(&self, idx: usize) { if self.quirks.clean_tables { - let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); pgtable::clean_poc(slot, size_of::()); pgtable::poc_barrier(); } @@ -1745,13 +1747,20 @@ impl UatShared { /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. @@ -1939,7 +1948,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -2046,10 +2057,7 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -2057,7 +2065,10 @@ fn prefetch_sgt_segments( segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { segments, _lease: sgt.into_lease() }) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) } /// Reuse cached segments without allocating another copy for each bind. The bind lease was @@ -2169,7 +2180,9 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2393,7 +2406,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, ) } @@ -2408,7 +2427,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, ) } @@ -2436,18 +2461,25 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; @@ -2518,16 +2550,21 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2601,7 +2638,7 @@ impl Vm { gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; + let dev: &device::Device = device.as_ref().as_ref(); let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm @@ -2622,6 +2659,8 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; @@ -2694,6 +2733,8 @@ impl Vm { size: usize, prot: Prot, ) -> Result { + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { @@ -2798,6 +2839,8 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); @@ -3042,7 +3085,7 @@ impl Uat { let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = @@ -3104,7 +3147,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { @@ -3388,9 +3431,9 @@ impl Uat { { return Err(EIO); } - for index in UAT_USER_CTX_START..UAT_NUM_CTX { - if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 { return Err(EIO); } @@ -3491,14 +3534,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -3511,11 +3559,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -3603,7 +3656,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); let shared_quirks = quirks; @@ -3732,7 +3785,10 @@ impl Uat { inner.handoff().init(firmware, quirks.handoff_slot)?; if quirks.clean_tables { - pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); pgtable::poc_barrier(); } @@ -3760,7 +3816,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); pgtable::poc_barrier(); } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index a66a82d92a5830..7c930df5410622 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,11 +298,15 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>(); let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -337,10 +341,14 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>>(); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { + if !core::ptr::eq(p.cast::>(), raw) { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -349,7 +357,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner, @@ -400,14 +408,14 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = &*ret.inner as *const T; + let q = core::ptr::from_ref::(&*ret.inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) @@ -437,7 +445,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -449,7 +457,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } } } @@ -475,8 +483,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) + .field("inner", &format_args!("{:#X?}", self.inner)) + .field("alloc", &format_args!("{:?}", self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 7244f12a43d398..215c19bf49ef08 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -153,9 +153,8 @@ impl ReservedTables { pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { let start = res.start(); // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = unsafe { - kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? - }; + let map = + unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; self.starts.push(start, GFP_KERNEL)?; self.regions.push(map, GFP_KERNEL)?; Ok(()) @@ -671,7 +670,7 @@ impl UatPageTable { let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; let entry = |p: *mut u8| -> Result> { - let uptep = p as *const _ as *const Pte; + let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is // aligned so it is safe to deref as an AtomicU64. let upte = unsafe { &*uptep }; @@ -679,7 +678,10 @@ impl UatPageTable { // Allocate if requested if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + mod_pr_debug!( + "UATPageTable::with_pages: alloc PT at {:#x}\n", + pt_page.phys() + ); if clean { // A non-snooping walker must never follow a parent to a // table whose zeroes are still in the CPU caches. @@ -755,7 +757,7 @@ impl UatPageTable { iova ); let leaves = |p: *mut u8| -> Result { - let ptep = p as *const _ as *const Pte; + let ptep = p.cast_const().cast::(); // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; @@ -769,7 +771,13 @@ impl UatPageTable { // SAFETY: Page table addresses are either allocated by us, or // firmware-managed tables in RAM or in a reserved region. unsafe { - with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + with_table( + reserved.as_deref(), + phys, + idx * PTE_SIZE, + count * PTE_SIZE, + leaves, + ) }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -806,7 +814,10 @@ impl UatPageTable { } if let Some((parent_phys, parent_index)) = parent { let oas_mask = self.oas_mask; - let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + let in_reserved = self + .reserved + .as_ref() + .is_some_and(|r| r.contains(parent_phys)); // SAFETY: Only looks the page up; nothing is accessed through it. if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { return Err(EIO); @@ -832,7 +843,15 @@ impl UatPageTable { // its child descriptor is detached. A changed descriptor indicates // an ownership violation; fail closed instead of freeing a table // that may still be reachable. - unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; + unsafe { + with_table( + reserved, + parent_phys, + parent_index * PTE_SIZE, + PTE_SIZE, + detach, + ) + }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index ff9804c5bd10be..a89fcaf0b8dcb2 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -35,6 +35,10 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; #[versions(AGX)] impl super::QueueInner::ver { /// Submit work to a compute queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_compute( &self, gpu: &gpu::GpuManager::ver, @@ -54,9 +58,10 @@ impl super::QueueInner::ver { return Err(EINVAL); } - let mut user_timestamps: fw::job::UserTimestamps = Default::default(); - user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; - user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + let user_timestamps = fw::job::UserTimestamps { + start: common::get_timestamp_object(objects, cmdbuf.ts.start)?, + end: common::get_timestamp_object(objects, cmdbuf.ts.end)?, + }; // This sequence number increases per new client/VM? assigned to some slot, // but it's unclear *which* slot... @@ -214,7 +219,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_5c_g14: U64(0), restart_branch_offset: off, - has_attachments: (attachments.count > 0) as u32, + has_attachments: u32::from(attachments.count > 0), #[ver(V >= V13_0B4)] unk_64: Default::default(), #[ver(V >= V13_0B4)] @@ -316,8 +321,8 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffff, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: cmdbuf.sampler_count, + sampler_max: cmdbuf.sampler_count + 1, }), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, @@ -328,7 +333,7 @@ impl super::QueueInner::ver { stamp_value: ev_comp.value.next(), stamp_slot: ev_comp.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid, event_seq: ev_comp.event_seq as u32, }), diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index b011ac127045ed..ff41ae065c20af 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -364,7 +364,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - return guard.registration_data_with(|reg_data| { + guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -439,7 +439,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }); + }) } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -632,8 +632,11 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -644,8 +647,10 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i as usize] = microseq::Attachment { + attachments.list[i] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -764,7 +769,7 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; let payload_size = header.size as usize; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; if render @@ -789,7 +794,7 @@ impl Queue for Queue::ver { } nr_commands += 1; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -881,7 +886,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { + let queue_job = match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -911,7 +916,7 @@ impl Queue for Queue::ver { } } - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 4de0e84239a02e..e02411e2417c09 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -47,9 +47,9 @@ impl super::QueueInner::ver { cmdbuf: &uapi::drm_asahi_cmd_render, num_clusters: u32, ) -> Result { - let width: u32 = cmdbuf.width_px as u32; - let height: u32 = cmdbuf.height_px as u32; - let layers: u32 = cmdbuf.layers as u32; + let width: u32 = u32::from(cmdbuf.width_px); + let height: u32 = u32::from(cmdbuf.height_px); + let layers: u32 = u32::from(cmdbuf.layers); if layers == 0 || layers > 2048 { cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); @@ -68,8 +68,8 @@ impl super::QueueInner::ver { let tile_width = 32u32; let tile_height = 32u32; - let utile_width = cmdbuf.utile_width_px as u32; - let utile_height = cmdbuf.utile_height_px as u32; + let utile_width = u32::from(cmdbuf.utile_width_px); + let utile_height = u32::from(cmdbuf.utile_height_px); match (utile_width, utile_height) { (32, 32) | (32, 16) | (16, 16) => (), @@ -153,7 +153,7 @@ impl super::QueueInner::ver { tpc_size, meta1_layer_stride, #[ver(G < G14X)] - meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + meta1_blocks: meta1_layer_stride * u32::from(cmdbuf.layers), #[ver(G >= G14X)] meta1_blocks: meta1_layer_stride, layermeta_size: if layers > 1 { 0x100 } else { 0 }, @@ -182,6 +182,10 @@ impl super::QueueInner::ver { } /// Submit work to a render queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_render( &self, gpu: &gpu::GpuManager::ver, @@ -199,7 +203,7 @@ impl super::QueueInner::ver { & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING - | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) != 0 { cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); @@ -235,8 +239,7 @@ impl super::QueueInner::ver { let mut clustering = nclusters > 1; if debug_enabled(debug::DebugFlags::DisableClustering) - || cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + || cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING != 0 { clustering = false; @@ -260,7 +263,7 @@ impl super::QueueInner::ver { // but it's unclear *which* slot... let slot_client_seq: u8 = (self.id & 0xff) as u8; - let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + let tile_info = Self::get_tiling_params(cmdbuf, if clustering { nclusters } else { 1 })?; let buffer = &self.buffer; let notifier = self.notifier.clone(); @@ -375,9 +378,7 @@ impl super::QueueInner::ver { if cmdbuf.layers > 1 { tile_config |= 1; } - if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0 - { + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0 { tile_config |= 0x10000; } @@ -398,19 +399,19 @@ impl super::QueueInner::ver { // Calculate the number of 2KiB blocks to allocate per utile. This is // just a bit of dimensional analysis. let pixels_per_utile: u32 = - (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + u32::from(cmdbuf.utile_width_px) * u32::from(cmdbuf.utile_height_px); let samples_per_utile: u32 = pixels_per_utile << samples_log2; - let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let utile_size_bytes: u32 = u32::from(cmdbuf.sample_size_B) * samples_per_utile; let block_size_bytes: u32 = 2048; let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); #[ver(G >= G14X)] let frg_tilecfg = 0x0000000_00036011 - | (((tile_info.tiles_x - 1) as u64) << 44) - | (((tile_info.tiles_y - 1) as u64) << 53) + | (u64::from(tile_info.tiles_x - 1) << 44) + | (u64::from(tile_info.tiles_y - 1) << 53) | (if unk1 { 0 } else { 0x20_00000000 }) | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) - | ((utile_config as u64 & 0xf000) << 28); + | ((u64::from(utile_config) & 0xf000) << 28); // TODO: check #[ver(V >= V13_0B4)] @@ -428,9 +429,9 @@ impl super::QueueInner::ver { let frg_unk_140 = 0x8c60; let frg_unk_158 = 0x1c; #[ver(G >= G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals); #[ver(G < G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals) | 0x400; let reload_zlsctrl = cmdbuf.zls_ctrl; let iogpu_unk54 = 0x3a0012006b0003; let iogpu_unk56 = 1; @@ -441,12 +442,12 @@ impl super::QueueInner::ver { #[ver(G >= G14X)] let vtx_unk_f0 = 0x1c; #[ver(G < G14)] - let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_f0 = 0x1c + u64::from(align(tile_info.meta1_blocks, 4)); let vtx_unk_118 = 0x1c; // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. - let isp_ctl = 0xc000u32 - | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + let isp_ctl = + 0xc000u32 | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT); // Always allow preemption at the UAPI level let no_preemption = false; @@ -575,7 +576,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_8c_g14: U64(0), restart_branch_offset: off, - has_attachments: (fragment_attachments.count > 0) as u32, + has_attachments: u32::from(fragment_attachments.count > 0), #[ver(V >= V13_0B4)] unk_9c: Default::default(), })?; @@ -597,10 +598,10 @@ impl super::QueueInner::ver { |inner, _ptr| { let vm_slot = vm_bind.slot(); let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { - isp_ctl: isp_ctl, + isp_ctl, unk2: 0, - width: cmdbuf.width_px as u32, - height: cmdbuf.height_px as u32, + width: u32::from(cmdbuf.width_px), + height: u32::from(cmdbuf.height_px), #[ver(V >= V13_0B4)] unk3: U64(0x100000), }; @@ -619,7 +620,7 @@ impl super::QueueInner::ver { unk_buffer_buf: inner.scene.kernel_buffer_pointer(), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - samples: cmdbuf.samples as u32, + samples: u32::from(cmdbuf.samples), tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, unk_50: U64(0), @@ -627,21 +628,21 @@ impl super::QueueInner::ver { isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), unk_68: U64(0), - tile_count: U64(tile_info.tiles as u64), + tile_count: U64(u64::from(tile_info.tiles)), #[ver(G < G14X)] job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { utile_config, unk_4: 0, bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), - address: U64(cmdbuf.bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.bg.usc)), }, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), isp_scissor_base: U64(cmdbuf.isp_scissor_base), isp_dbias_base: U64(cmdbuf.isp_dbias_base), isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), aux_fb_info, - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), zls_ctrl: U64(cmdbuf.zls_ctrl), #[ver(G >= G14)] unk_58_g14_0: U64(g14_unk), @@ -653,21 +654,21 @@ impl super::QueueInner::ver { s_store: U64(cmdbuf.stencil.base), #[ver(G >= G14)] unk_68_g14_0: Default::default(), - z_load_stride: U64(cmdbuf.depth.stride as u64), - z_store_stride: U64(cmdbuf.depth.stride as u64), - s_load_stride: U64(cmdbuf.stencil.stride as u64), - s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_stride: U64(u64::from(cmdbuf.depth.stride)), + z_store_stride: U64(u64::from(cmdbuf.depth.stride)), + s_load_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_store_stride: U64(u64::from(cmdbuf.stencil.stride)), z_load_comp: U64(cmdbuf.depth.comp_base), - z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_load_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store_comp: U64(cmdbuf.depth.comp_base), - z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), s_load_comp: U64(cmdbuf.stencil.comp_base), - s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_load_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store_comp: U64(cmdbuf.stencil.comp_base), - s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), tvb_layermeta: inner.scene.tvb_layermeta_pointer(), - mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + mtile_stride_dwords: U64((4 * u64::from(tile_info.params.rgn_size)) << 24), tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), tile_config: U64(tile_config), aux_fb: inner.aux_fb.gpu_pointer(), @@ -737,7 +738,7 @@ impl super::QueueInner::ver { r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 r.add( 0x15211, - ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + (u64::from(cmdbuf.height_px) << 32) | u64::from(cmdbuf.width_px), ); // aux_fb_info.{width, heigh r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c @@ -756,21 +757,21 @@ impl super::QueueInner::ver { r.add(0x15221, 0); r.add(0x15239, 0); r.add(0x15229, 0); - r.add(0x15401, cmdbuf.depth.stride as u64); // load - r.add(0x15421, cmdbuf.depth.stride as u64); // store - r.add(0x15409, cmdbuf.stencil.stride as u64); // load - r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x15401, u64::from(cmdbuf.depth.stride)); // load + r.add(0x15421, u64::from(cmdbuf.depth.stride)); // store + r.add(0x15409, u64::from(cmdbuf.stencil.stride)); // load + r.add(0x15429, u64::from(cmdbuf.stencil.stride)); r.add(0x153c1, cmdbuf.depth.comp_base); // load - r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x15411, u64::from(cmdbuf.depth.comp_stride)); // load r.add(0x153c9, cmdbuf.depth.comp_base); // store - r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x15431, u64::from(cmdbuf.depth.comp_stride)); // store r.add(0x153d1, cmdbuf.stencil.comp_base); // load - r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x15419, u64::from(cmdbuf.stencil.comp_stride)); // load r.add(0x153d9, cmdbuf.stencil.comp_base); // store - r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x15439, u64::from(cmdbuf.stencil.comp_stride)); // store r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); - r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x16431, (4 * u64::from(tile_info.params.rgn_size)) << 24); // ISP_RGN? r.add(0x10039, tile_config); // tile_config ISP_CTL? r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN r.add(0x11821, cmdbuf.fragment_helper.binary.into()); @@ -807,28 +808,28 @@ impl super::QueueInner::ver { unk_118: U64(0x0), unk_120: Default::default(), unk_partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, unk_258: U64(0), unk_260: U64(0), unk_268: U64(0), unk_270: U64(0), partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, zls_ctrl: U64(reload_zlsctrl), unk_290: U64(g14_unk), z_load: U64(cmdbuf.depth.base), - z_partial_stride: U64(cmdbuf.depth.stride as u64), - z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_partial_stride: U64(u64::from(cmdbuf.depth.stride)), + z_partial_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store: U64(cmdbuf.depth.base), z_partial: U64(cmdbuf.depth.base), z_partial_comp: U64(cmdbuf.depth.comp_base), s_load: U64(cmdbuf.stencil.base), - s_partial_stride: U64(cmdbuf.stencil.stride as u64), - s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_partial_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_partial_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store: U64(cmdbuf.stencil.base), s_partial: U64(cmdbuf.stencil.base), s_partial_comp: U64(cmdbuf.stencil.comp_base), @@ -848,13 +849,13 @@ impl super::QueueInner::ver { ), isp_bgobjdepth: cmdbuf.isp_bgobjdepth, isp_bgobjvals: cmdbuf.isp_bgobjvals, - sample_size: cmdbuf.sample_size_B as u32, + sample_size: u32::from(cmdbuf.sample_size_B), unk_37c: 0x0, unk_380: U64(0x0), unk_388: U64(0x0), #[ver(V >= V13_0B4)] unk_390_0: U64(0x0), - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), }), unk_758_flag: 0, unk_75c_flag: 0, @@ -871,12 +872,11 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), - process_empty_tiles: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0) as u32, + process_empty_tiles: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0), // TODO: needs to be investigated no_clear_pipeline_textures: 1, // TODO: needs to be investigated @@ -887,13 +887,13 @@ impl super::QueueInner::ver { meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_frag.stamp_pointer, fw_stamp: ev_frag.fw_stamp_pointer, stamp_value: ev_frag.value.next(), stamp_slot: ev_frag.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_3d, event_seq: ev_frag.event_seq as u32, }), @@ -1027,7 +1027,7 @@ impl super::QueueInner::ver { #[ver(V < V13_0B4)] unk_178: 0x0, // padding? #[ver(V >= V13_0B4)] - unk_178: (!clustering) as u32, + unk_178: u32::from(!clustering), })?; builder.add(microseq::Timestamp::ver { @@ -1089,7 +1089,7 @@ impl super::QueueInner::ver { #[ver(G >= G14 && V < V13_0B4)] unk_68_g14: U64(0), restart_branch_offset: off, - has_attachments: (vertex_attachments.count > 0) as u32, + has_attachments: u32::from(vertex_attachments.count > 0), #[ver(V >= V13_0B4)] unk_74: Default::default(), // Ventura })?; @@ -1140,7 +1140,7 @@ impl super::QueueInner::ver { tvb_cluster_meta1: inner .scene .meta_1_pointer() - .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + .map(|x| x.or(u64::from(tile_info.meta1_layer_stride) << 50)), utile_config, unk_4c: 0, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed @@ -1268,8 +1268,8 @@ impl super::QueueInner::ver { r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL r.add( 0x10129, - tile_info.params.x_max as u64 - | ((tile_info.params.y_max as u64) << 16), + u64::from(tile_info.params.x_max) + | (u64::from(tile_info.params.y_max) << 16), ); // PPP_SCREEN r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 @@ -1323,26 +1323,25 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), unk_55c: 0, unk_560: 0, sync_grow: 0, unk_568: 0, - uses_scratch: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 - != 0) as u32, + uses_scratch: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH != 0), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_vtx.stamp_pointer, fw_stamp: ev_vtx.fw_stamp_pointer, stamp_value: ev_vtx.value.next(), stamp_slot: ev_vtx.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_ta, event_seq: ev_vtx.event_seq as u32, }), diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 772d969d9baff9..d07de2c6e73845 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -11,7 +11,8 @@ //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ //! context); the sysfs `show` callback runs in arbitrary process context. All -//! fields use `AtomicU*` so neither side takes a lock. +//! legacy fields use `AtomicU*`. G17 instead uses the optional reader under +//! its device mutex and does not populate these legacy atomic fields. //! //! Field semantics (firmware units, late-validated): //! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning @@ -58,6 +59,9 @@ pub(crate) struct StatsSnapshot { /// `JobFence::command_complete` when the last command of a submission /// completes. pub(crate) jobs: AtomicU64, + /// Optional qualified reader. The legacy C layout above remains a prefix; + /// C accesses this extension only through the registered Rust callbacks. + pub(crate) g17_reader: Option, } // Keep the Rust half of the C sysfs layout contract checked as well. @@ -65,7 +69,7 @@ const _: () = { assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); - assert!(core::mem::size_of::() == 56); + assert!(core::mem::offset_of!(StatsSnapshot, g17_reader) == 56); }; impl StatsSnapshot { diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7d7d785a831334..a704e3f1f7afdf 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,7 +18,7 @@ #include "sysfs.h" -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ +/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -48,8 +50,69 @@ struct asahi_stats_attribute { const struct asahi_stats_snapshot *snapshot; const struct asahi_gpu_memory *memory; bool export_enabled; + asahi_stats_read_fn read; + asahi_stats_suspend_fn suspend; + struct notifier_block pm_notifier; }; +/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ +static const char *asahi_stats_quality(u32 status) +{ + return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; +} + +static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) +{ + struct asahi_qualified_stats snap; + ssize_t n = 0; + + stats->read(stats->snapshot, &snap); + n += sysfs_emit_at(buf, n, "utilisation_status %s\n", + asahi_stats_quality(snap.util_status)); + if (snap.util_status == 1) { + n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); + n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); + n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); + } else { + n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" + "util_renderer_percent unsupported\n" + "util_device_percent unsupported\n"); + } + n += sysfs_emit_at(buf, n, "pstate_status %s\n", + asahi_stats_quality(snap.pstate_status)); + if (snap.pstate_status == 1) + n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); + else + n += sysfs_emit_at(buf, n, "pstate unsupported\n"); + /* No A/B selector, packet duration, measured power or temperature. */ + n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" + "achieved_frequency_hz unsupported\n" + "busy_ns unsupported\n" + "power_mw unsupported\n" + "temperature_raw unsupported\n"); + return n; +} + +static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct asahi_stats_attribute *stats = + container_of(nb, struct asahi_stats_attribute, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + stats->suspend(stats->snapshot, true); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + stats->suspend(stats->snapshot, false); + break; + } + return NOTIFY_OK; +} + static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -60,6 +123,8 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); + if (stats->read) + return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -118,14 +183,18 @@ static ssize_t gpu_memory_show(struct device *dev, * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle) + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || + (!!read != !!suspend)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); + BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); @@ -145,6 +214,16 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; + stats->read = read; + stats->suspend = suspend; + if (suspend) { + stats->pm_notifier.notifier_call = asahi_stats_pm_notify; + ret = register_pm_notifier(&stats->pm_notifier); + if (ret) { + kfree(stats); + return ret; + } + } sysfs_attr_init(&stats->memory_attr.attr); stats->memory_attr.attr.name = "gpu_memory"; @@ -154,6 +233,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->attr); if (ret) { + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -161,6 +242,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->memory_attr); if (ret) { device_remove_file(dev, &stats->attr); + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -217,6 +300,9 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; + /* Both notifier removal and file removal join readers of the Rust Arc. */ + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index f1873f9be6caaa..d43f482903e559 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,10 +3,30 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H +#include + struct device; +/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ +struct asahi_qualified_stats { + u32 util_status; + u32 pstate_status; + u32 tiler; + u32 renderer; + u32 device; + u32 pstate; + u64 util_host_ns; + u64 pstate_host_ns; + u64 util_samples; + u64 pstate_timestamp; +}; + +typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); +typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); + int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle); + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index e7f842d8ac6fdd..7c34f0627b0200 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,16 +3,21 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{memacct::Ledger, stats::StatsSnapshot}; +use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; +type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); +type SuspendStats = unsafe extern "C" fn(*const c_void, bool); + extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const StatsSnapshot, + snapshot: *const c_void, export_enabled: c_int, memory: *const Ledger, handle: *mut *mut c_void, + read: Option, + suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } @@ -43,6 +48,66 @@ pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks(dev, snapshot, export_enabled, None, None) +} + +/// The caller retains the G17 snapshot Arc until unregister joins both sysfs +/// readers and the PM notifier. The callbacks never power the GPU up. +pub(crate) fn register_g17( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks( + dev, + snapshot, + export_enabled, + Some(read_g17), + Some(suspend_g17), + ) +} + +/// Copy the locked qualified snapshot to the C formatter's local output. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot retained by the registration. +/// `out` must be aligned and writable for one Readout for this call. +unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { + // SAFETY: C calls only while registration retains the snapshot Arc, and + // supplies a writable, aligned output for this call. + let snapshot = unsafe { &*snapshot.cast::() }; + let view = snapshot + .g17_reader + .as_ref() + .map_or_else(Readout::default, |reader| reader.read()); + // SAFETY: As above; the C output is initialized in one locked read. + unsafe { out.write(view) }; +} + +/// End the telemetry epoch at either side of system sleep. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot. The caller must join this +/// callback before releasing the registration's owning Arc. +unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { + // SAFETY: The notifier is unregistered and joined before the Arc is dropped. + if let Some(reader) = unsafe { &*snapshot.cast::() } + .g17_reader + .as_ref() + { + reader.suspend(suspended); + } +} + +fn register_with_callbacks( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, + read: Option, + suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -50,10 +115,12 @@ pub(crate) fn register( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot, + snapshot.cast(), c_int::from(export_enabled), &crate::memacct::LEDGER, &mut handle, + read, + suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index a5f9455b23f5fd..d1091c7026ab21 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,6 +96,13 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } +/// A value that can be populated from an untrusted byte record. +/// +/// # Safety +/// +/// Every bit pattern must be valid for Self. Its byte representation must +/// contain no uninitialized bytes (including padding) when produced by Default, +/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -117,7 +124,9 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + let dst = unsafe { + core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) + }; dst.copy_from_slice(&src[..size]); self.offset = end; diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 315396ef716ce1..4cff78b90a6ef4 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -681,7 +681,7 @@ impl WorkQueue::ver { } let inner = WorkQueueInner::ver { - dev: dev.into(), + dev, event_manager, // Use shared (coherent) state with verbose faults so we can dump state correctly info: if debug_enabled(DebugFlags::VerboseFaults) { diff --git a/tools/asahi/binding-lifetime-tests.py b/tools/asahi/binding-lifetime-tests.py new file mode 100644 index 00000000000000..dedcbcd4326e5c --- /dev/null +++ b/tools/asahi/binding-lifetime-tests.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production bound-device gate with host locking/refcount adapters.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const ENODEV:i32=-19; + pub const EOVERFLOW:i32=-75; + pub const GFP_KERNEL:()=(); + pub struct KVec(Vec); + impl KVec { pub const fn new()->Self {Self(Vec::new())} pub fn push(&mut self,v:T,_:())->Result {self.0.push(v);Ok(())} } + impl std::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} + impl std::ops::DerefMut for KVec{fn deref_mut(&mut self)->&mut Vec{&mut self.0}} +} +#[macro_export] +macro_rules! new_mutex {($v:expr $(,$name:expr)?)=>{$crate::sync::Mutex::new($v)};} +#[macro_export] +macro_rules! global_lock { (unsafe(uninit) static $name:ident: Mutex<$ty:ty> = $value:expr;) => { static $name:$crate::sync::Mutex<$ty>=$crate::sync::Mutex::new($value); }; } +#[macro_export] +macro_rules! new_condvar {($name:expr)=>{$crate::sync::CondVar(std::sync::Condvar::new())};} +macro_rules! pin_init {($($t:tt)*)=>{$($t)*};} +mod sync { + pub use crate::global_lock; + pub struct Arc(std::sync::Arc); + impl Clone for Arc{fn clone(&self)->Self{Self(self.0.clone())}} + impl std::ops::Deref for Arc{type Target=T;fn deref(&self)->&T{&self.0}} + impl Arc{pub fn pin_init(v:T,_:())->crate::prelude::Result{Ok(Self(std::sync::Arc::new(v)))} pub fn ptr_eq(a:&Self,b:&Self)->bool{std::sync::Arc::ptr_eq(&a.0,&b.0)}} + pub struct Mutex(std::sync::Mutex); + impl Mutex{pub const fn new(v:T)->Self{Self(std::sync::Mutex::new(v))} pub unsafe fn init(&self){} pub fn lock(&self)->Guard<'_,T>{Guard(Some(self.0.lock().unwrap()))}} + pub struct Guard<'a,T>(Option>); + impl std::ops::Deref for Guard<'_,T>{type Target=T;fn deref(&self)->&T{self.0.as_ref().unwrap()}} + impl std::ops::DerefMut for Guard<'_,T>{fn deref_mut(&mut self)->&mut T{self.0.as_mut().unwrap()}} + pub struct CondVar(pub std::sync::Condvar); + impl CondVar {pub fn wait(&self,g:&mut Guard<'_,T>){let old=g.0.take().unwrap();g.0=Some(self.0.wait(old).unwrap());} pub fn notify_all(&self){self.0.notify_all();}} + pub mod aref { + pub struct ARef(&'static T); + impl std::ops::Deref for ARef{type Target=T;fn deref(&self)->&T{self.0}} + impl From<&crate::device::Device> for ARef{fn from(d:&crate::device::Device)->Self { + // The host fixtures leak their devices for the test process. + Self(unsafe{&*(d as *const _ as *const crate::device::Device)}) + }} + } +} +mod device { + pub struct Normal; + pub struct Bound; + #[repr(C)] + pub struct Device{pub bound:std::sync::atomic::AtomicBool,pub marker:std::marker::PhantomData} + impl Device{pub fn new()->Self{Self{bound:std::sync::atomic::AtomicBool::new(true),marker:std::marker::PhantomData}}} + impl Device {pub unsafe fn as_bound(&self)->&Device{assert!(self.bound.load(std::sync::atomic::Ordering::SeqCst));unsafe{&*(self as *const _ as *const Device)}}} +} +mod driver {pub struct AsahiDrmDevice(pub u64);} +mod binding; +''' +TESTS = r''' +use {binding::*,device::*,driver::AsahiDrmDevice}; +use std::{sync::{mpsc,atomic::Ordering},time::Duration}; +fn device()->&'static Device{Box::leak(Box::new(Device::new()))} +#[test] +fn close_joins_existing_access_and_rejects_future_access() { + let dev=device();let drm=Box::new(AsahiDrmDevice(1)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap(); + let gate=lookup(&drm).unwrap();let access=gate.enter().unwrap();let _=access.device(); + let (started_rx,started)= {let (a,b)=mpsc::channel();(b,a)}; + let (done,finished)=mpsc::channel(); + let thread=std::thread::spawn(move||{started.send(()).unwrap();owner.close();done.send(()).unwrap();}); + started_rx.recv().unwrap(); + assert!(finished.recv_timeout(Duration::from_millis(50)).is_err(),"unbind returned with a live device borrow"); + drop(access);finished.recv_timeout(Duration::from_secs(2)).unwrap();thread.join().unwrap(); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +#[test] +fn rebind_cannot_reopen_an_old_vm_gate() { + let dev=device();let a=Box::new(AsahiDrmDevice(2));let b=Box::new(AsahiDrmDevice(3)); + let first=unsafe{Owner::new(dev,&a)}.unwrap();let old=lookup(&a).unwrap(); + first.close();let second=unsafe{Owner::new(dev,&b)}.unwrap(); + assert!(old.enter().is_err());assert!(lookup(&a).is_err()); + let new=lookup(&b).unwrap();let access=new.enter().unwrap();let _=access.device();drop(access);second.close(); +} +#[test] +fn failed_probe_owner_drop_closes_retained_vm_handles() { + let dev=device();let drm=Box::new(AsahiDrmDevice(4)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap();let gate=lookup(&drm).unwrap();drop(owner); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--mutation', choices=['no-close', 'no-join']) + args = ap.parse_args() + work = (args.out or Path(tempfile.mkdtemp(prefix='asahi-binding-'))).resolve() + work.mkdir(parents=True, exist_ok=True) + source = (ROOT / 'drivers/gpu/drm/asahi/binding.rs').read_text() + source = '\n'.join(line for line in source.splitlines() + if line.strip() not in ('#[pin_data]', '#[pin]')) + source = source.replace(' <- ', ': ') + source = source.replace('// SAFETY: Module initialization initializes this lock before registering adapters.', '') + if args.mutation == 'no-close': + source = source.replace('state.live = false;', 'state.live = true;') + if args.mutation == 'no-join': + source = source.replace('while state.users != 0 {', 'while false {') + (work / 'binding.rs').write_text(source) + (work / 'test.rs').write_text(HEADER + TESTS) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'test.rs'), '-o', str(work / 'test')], check=True) + subprocess.run([str(work / 'test')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 3ee813480ae4fa..aad8fc6efa6c4b 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -76,6 +76,7 @@ def item(text, marker): static FAIL_REGISTRATION: Cell = const { Cell::new(0) }; static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; static ACTIVATED: Cell = const { Cell::new(false) }; + static BINDING_READY: Cell = const { Cell::new(false) }; static ATTACH_ERROR: Cell = const { Cell::new(false) }; static PROBE_ERROR: Cell = const { Cell::new(0) }; } @@ -135,6 +136,11 @@ def item(text, marker): pub trait InPlaceModule: Sized { fn init(module: &'static ThisModule) -> impl PinInit; } +// Registry synchronization is covered by binding-lifetime-tests; this adapter +// verifies that the actual module initializer orders it before registration. +mod binding { + pub unsafe fn init() { crate::BINDING_READY.set(true); } +} mod cleanup { pub struct QueueOwner; impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } @@ -145,6 +151,7 @@ def item(text, marker): pub struct Registration { stage: u8, marker: PhantomData } impl Registration { pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { + assert!(BINDING_READY.get(), "adapter registered before binding registry init"); try_pin_init!(Self { stage: { let stage = if name.to_bytes() == b"asahi-g15" { 1 } else { 2 }; @@ -192,6 +199,7 @@ def item(text, marker): } fn reset(fail_alloc: bool, touch: bool, error: i32) { MODULE_READY.store(true, Ordering::Release); + BINDING_READY.set(false); FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); } diff --git a/tools/asahi/g17-telemetry-integration-tests.py b/tools/asahi/g17-telemetry-integration-tests.py new file mode 100644 index 00000000000000..5c36b6beed579f --- /dev/null +++ b/tools/asahi/g17-telemetry-integration-tests.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production ring transport and C formatter without hardware.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = ROOT / 'drivers/gpu/drm/asahi' + + +def item(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +RUST_HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const EIO: i32 = -5; + pub const EAGAIN: i32 = -11; +} +mod device { + pub struct Device; + impl AsRef for Device { fn as_ref(&self) -> &Device { self } } +} +mod fmt { pub use std::fmt::*; } +macro_rules! dev_err { ($dev:expr, $($t:tt)*) => { let _ = ($dev, format_args!($($t)*)); }; } +mod g17 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NOW: AtomicU64 = AtomicU64::new(1); + fn now_ns() -> u64 { NOW.load(Ordering::Relaxed) } + #[derive(Clone, Copy, Debug)] + enum Role { Primary = 0, Secondary = 1 } + mod trace { pub fn report(_:u8, _:u8, _:&[u32;18]) {} pub fn message(_:u32,_:u64) {} pub const CONTROL:u32=0; } + mod fw { + pub mod variant { + #[derive(Clone, Copy)] + pub enum InitGraph { V25G83, G17P } + } + pub mod channels { + pub const EVENT_WORDS:usize=18; + pub const EVENT_RECORD_SIZE:usize=72; + pub const RX_PRODUCER_OFFSET:usize=0x20; + pub const TRACE_SLOTS:u32=4; + pub const TRACE_TYPES:u32=1; + pub const STATISTICS_SLOTS:u32=8; + pub const STATISTICS_TYPES:u32=0x5fff0000; + pub const DEVICE_CONTROL_SLOTS:u32=8; + pub struct ControlRecord; + #[derive(Clone, Copy)] + pub enum Opcode { Dummy } + impl Opcode { pub fn asserts_gpu_power(self)->bool { false } } + impl ControlRecord { + pub fn bytes(&self)->&[u8] { &[] } + pub fn opcode(&self)->Opcode { Opcode::Dummy } + pub fn asserts_gpu_power(&self)->bool { false } + } + } + pub mod initdata { pub mod status { + pub const TRACE_STATE:u32=0; + pub const TRACE_RING:u32=0x100; + pub const STATS_STATE:u32=0x1000; + pub const STATS_RING:u32=0x2000; + } } + } + mod initdata { + use super::{Role,fw::{channels::*,initdata::status::*}}; + use crate::prelude::*; + use std::{cell::RefCell,sync::atomic::AtomicU32}; + pub struct InitData { + pub cursors:[[AtomicU32;2];2], + pub slots:RefCell<[[[u32;18];8];2]>, + } + impl InitData { + pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]) } } + pub fn push(&self,role:Role,words:[u32;18]) { + use std::sync::atomic::Ordering::Relaxed; + let producer=self.cursors[role as usize][1].load(Relaxed); + self.slots.borrow_mut()[role as usize][producer as usize]=words; + self.cursors[role as usize][1].store((producer+1)%STATISTICS_SLOTS,Relaxed); + } + pub fn status_word(&self,role:Role,offset:usize)->Result<&AtomicU32> { + Ok(&self.cursors[role as usize][usize::from(offset==STATS_STATE as usize+RX_PRODUCER_OFFSET)]) + } + pub fn status_read_words(&self,role:Role,offset:usize)->Result<[u32;N]> { + let mut words=[0;N]; + words.copy_from_slice(&self.slots.borrow()[role as usize][(offset-STATS_RING as usize)/EVENT_RECORD_SIZE][..N]); + Ok(words) + } + pub fn control_consumer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][0]) } + pub fn control_producer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][1]) } + pub fn publish_control(&self,_:u32,_:&[u8],_:bool,_:u32)->Result { Ok(()) } + } + } + mod reports { + use super::Role; + pub struct Reports; + impl Reports { + pub fn new()->crate::prelude::Result { Ok(Self) } + pub fn record(&mut self,_:Role,_:[u32;18],_:u64) {} + pub fn write(&self,_:&mut crate::fmt::Formatter<'_>)->crate::fmt::Result { Ok(()) } + } + } +''' +RUST_TESTS = r''' + #[test] + fn roles_merge_and_any_statistics_quarantine_disables_both() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + let sample=|ta,render,device|[0x13,0,0,ta,render,0xdeadbeef,device,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,sample(25,75,50)); + init.push(Secondary,sample(35,85,65)); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + let view=rings.read_statistics(1); + assert_eq!((view.tiler,view.renderer,view.device,view.util_samples),(30,80,57,2)); + assert_eq!(view.util_status,telemetry::VALID); + init.cursors[1][1].store(fw::channels::STATISTICS_SLOTS,Ordering::Relaxed); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.cursors[1][1].store(1,Ordering::Relaxed); + for _ in 0..3 { + init.push(Primary,sample(100,100,100)); + rings.drain_statistics(&dev,&init,Primary); + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + } + #[test] + fn suspend_gating_and_restart_discard_queued_epoch_records() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::V25G83).unwrap(); + let words=[0x13,0,0,25,75,0,50,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + for boundary in 0..4 { + init.push(Primary,words); // Unread at the epoch boundary. + match boundary { + 0=> { rings.suspend_statistics(true); rings.drain_statistics(&dev,&init,Primary); rings.suspend_statistics(false); init.push(Primary,words); } + 1=>rings.observe_statistics_power(false,0), + 2=>rings.observe_statistics_power(true,1), // Wake observed by a fast publisher. + _=>rings.invalidate_statistics(), // Restart handshake. + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + } + } + #[test] + fn pstate_uses_drain_order_without_role_deduplication() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + init.push(Secondary,[0x10,0x2500,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).pstate,6); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).pstate_status,telemetry::UNAVAILABLE); + } +} +''' + +C_HEADER = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define PAGE_SIZE 4096 +#define READ_ONCE(x) (x) +#define container_of(ptr,type,member) ((type*)((char*)(ptr)-offsetof(type,member))) +struct device {}; +struct device_attribute { + struct { const char *name; unsigned mode; } attr; + ssize_t (*show)(struct device*,struct device_attribute*,char*); +}; +struct notifier_block { int (*notifier_call)(struct notifier_block*,unsigned long,void*); }; +#define PM_SUSPEND_PREPARE 1 +#define PM_HIBERNATION_PREPARE 2 +#define PM_RESTORE_PREPARE 3 +#define PM_POST_SUSPEND 4 +#define PM_POST_HIBERNATION 5 +#define PM_POST_RESTORE 6 +#define NOTIFY_OK 1 +#define ENODEV 19 +#define ENOMEM 12 +#define BUILD_BUG_ON(x) _Static_assert(!(x),#x) +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define sysfs_attr_init(x) ((void)0) +#define kfree(x) free(x) +static int registered, removed, created, fail_file, fail_notifier, suspend_calls; +static int register_pm_notifier(struct notifier_block *nb) { (void)nb; if(fail_notifier)return -12; registered++; return 0; } +static int unregister_pm_notifier(struct notifier_block *nb) { (void)nb; removed++; return 0; } +static int device_create_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; return ++created==fail_file?-12:0; } +static void device_remove_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; } +static int emit(char *buf,size_t size,const char *fmt,...) { + va_list ap;va_start(ap,fmt);int n=vsnprintf(buf,size,fmt,ap);va_end(ap);return n<(int)size?n:(int)size-1; +} +#define scnprintf emit +#define sysfs_emit(buf,...) emit(buf,PAGE_SIZE,__VA_ARGS__) +#define sysfs_emit_at(buf,at,...) emit(buf+at,PAGE_SIZE-(at),__VA_ARGS__) +static ssize_t gpu_memory_show(struct device *dev,struct device_attribute *attr,char *buf) { (void)dev;(void)attr;(void)buf;return 0; } +static u32 *asahi_idle_word; +static struct device_attribute dev_attr_idle_off_delay_ms; +''' +C_TESTS = r''' +static struct asahi_qualified_stats current; +static void read_stats(const void *ptr,struct asahi_qualified_stats *out) { (void)ptr;*out=current; } +static void suspend_stats(const void *ptr,bool suspended) { (void)ptr;suspend_calls+=suspended?1:10; } +int main(void) { + struct asahi_stats_snapshot snap; + struct asahi_stats_attribute stats={.snapshot=&snap,.export_enabled=true}; + char old[PAGE_SIZE],now[PAGE_SIZE]; + for(unsigned seed=0;seed<512;seed++) { + snap=(struct asahi_stats_snapshot){ + .util1=seed, .util2=seed+11, .util3=~seed, .util4=seed*0x01010101u, + .pstate=seed%16, .avg_power_mw=seed*1234567u, + .temperature_raw=seed^0xff00aa11u, .temperature_scale=seed+37, + .temperature_tmin=seed+42, .temperature_tmax=seed+73, + .busy_ns=((u64)seed<<32)|(UINT32_MAX-seed), + .jobs=UINT64_MAX-(u64)seed*101, + }; + ssize_t before=old_agx_stats_show(NULL,&stats.attr,old); + ssize_t after=agx_stats_show(NULL,&stats.attr,now); + assert(before==after && !memcmp(old,now,before+1)); + } + stats.export_enabled=false; + assert(agx_stats_show(NULL,&stats.attr,now)==12); + assert(!strcmp(now,"unsupported\n")); + stats.export_enabled=true;stats.read=read_stats; + for(unsigned status=0;status<3;status++) { + current=(struct asahi_qualified_stats){.util_status=status,.pstate_status=status,.tiler=30,.renderer=80,.device=57,.pstate=6}; + agx_stats_show(NULL,&stats.attr,now); + assert(strstr(now,"busy_ns unsupported\n")); + assert(strstr(now,"power_mw unsupported\n")); + assert(strstr(now,"temperature_raw unsupported\n")); + assert(strstr(now,"requested_frequency_hz unsupported\n")); + assert(strstr(now,"achieved_frequency_hz unsupported\n")); + if(status==1) { + assert(strstr(now,"util_tiler_percent 30\nutil_renderer_percent 80\nutil_device_percent 57\n")); + assert(strstr(now,"pstate 6\n")); + } else { + assert(strstr(now,"util_device_percent unsupported\n")); + assert(strstr(now,"pstate unsupported\n")); + assert(strstr(now,status==2?"pstate_status stale\n":"pstate_status unavailable\n")); + } + } + struct device dev; + struct asahi_gpu_memory memory={0}; + void *handle=NULL; + assert(!asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)); + struct asahi_stats_attribute *registered_stats=handle; + assert(registered==1 && removed==0); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_SUSPEND_PREPARE,NULL); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_POST_SUSPEND,NULL); + assert(suspend_calls==11); + asahi_sysfs_unregister(&dev,handle); + assert(removed==1); + for(int failure=1;failure<=2;failure++) { + created=0;fail_file=failure;handle=NULL; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(registered==removed); + assert(handle==NULL); + } + fail_file=0;fail_notifier=1;created=0; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(created==0 && registered==removed); + puts("PASS: 512 byte-identical legacy outputs, G17 capabilities, PM lifecycle and registration rollback"); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--baseline', default='ebcf9cb777a434867e9d880892ee2b46887728bb', + help='revision containing the legacy formatter') + ap.add_argument('--mutation', choices=['fields', 'range', 'timestamp', 'age', + 'quarantine', 'epoch', 'legacy-format']) + args = ap.parse_args() + work = args.out or Path(tempfile.mkdtemp(prefix='asahi-g17-telemetry-')) + work.mkdir(parents=True, exist_ok=True) + work = work.resolve() + channel = (DRIVER / 'g17/channel.rs').read_text().split('/// One bounded event-ring pass.')[0] + # Compile the production transport and decoder; only MMIO/shared memory, + # kernel logging and allocation adapters are replaced by host fixtures. + channel = '\n'.join(line for line in channel.splitlines() if not line.startswith('//!')) + (work / 'channel.rs').write_text(channel) + telemetry = (DRIVER / 'g17/telemetry.rs').read_text() + changes = { + 'fields': ('[words[3], words[4], words[6]]', '[words[3], words[4], words[5]]'), + 'range': ('value > 100', 'value > 101'), + 'timestamp': ('u64::from(words[1]) | (u64::from(words[2]) << 32)', + 'u64::from(words[2]) | (u64::from(words[1]) << 32)'), + 'age': ('now_ns - received >= max_age_ns', 'now_ns - received > max_age_ns'), + 'quarantine': ('.any(|rings| rings[ReportRing::Statistics as usize])', '.any(|_| false)'), + 'epoch': ('self.discard_statistics = [true; 2]', 'self.discard_statistics = [false; 2]'), + } + if args.mutation in changes: + before, after = changes[args.mutation] + if args.mutation in ('quarantine', 'epoch'): + assert before in channel + channel = channel.replace(before, after, 1) + (work / 'channel.rs').write_text(channel) + else: + assert before in telemetry + telemetry = telemetry.replace(before, after, 1) + (work / 'telemetry.rs').write_text(telemetry) + policy = item((DRIVER / 'g17/stats.rs').read_text(), 'pub(super) const fn max_age_ns') + (work / 'rings.rs').write_text(RUST_HEADER + 'mod stats { use super::fw::variant::InitGraph;\n' + + policy + '\n}\nmod telemetry;\nmod channel;\n' + RUST_TESTS) + # Rust resolves child modules relative to the inline parent's directory. + (work / 'g17').mkdir(exist_ok=True) + for name in ['channel.rs', 'telemetry.rs']: + (work / 'g17' / name).write_text((work / name).read_text()) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'rings.rs'), '-o', str(work / 'rings')], check=True) + subprocess.run([str(work / 'rings')], check=True) + source = (DRIVER / 'sysfs.c').read_text() + if args.mutation == 'legacy-format': + source = source.replace('READ_ONCE(snap->util1)', 'READ_ONCE(snap->util2)', 1) + header = (DRIVER / 'sysfs.h').read_text() + baseline = subprocess.check_output(['git', '-C', str(ROOT), 'show', args.baseline + ':drivers/gpu/drm/asahi/sysfs.c'], text=True) + pieces = [C_HEADER] + for signature in ['struct asahi_qualified_stats']: + pieces.append(item(header, signature) + ';') + pieces += [line for line in header.splitlines() if line.startswith('typedef')] + for signature in ['struct asahi_stats_snapshot', 'struct asahi_gpu_memory', 'struct asahi_stats_attribute']: + pieces.append(item(source, signature) + ';') + pieces.append(item(baseline, 'static ssize_t agx_stats_show').replace('agx_stats_show(', 'old_agx_stats_show(', 1)) + for signature in ['static const char *asahi_stats_quality', 'static ssize_t agx_qualified_stats_show', + 'static int asahi_stats_pm_notify', 'static ssize_t agx_stats_show', + 'int asahi_sysfs_register(', 'void asahi_sysfs_unregister(']: + pieces.append(item(source, signature)) + pieces.append(C_TESTS) + (work / 'formatter.c').write_text('\n'.join(pieces)) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + str(work / 'formatter.c'), '-o', str(work / 'formatter')], check=True) + subprocess.run([str(work / 'formatter')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + print(f'child exit: {error.returncode}') + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g17-telemetry-tests.rs b/tools/asahi/g17-telemetry-tests.rs new file mode 100644 index 00000000000000..bfedf6ea031553 --- /dev/null +++ b/tools/asahi/g17-telemetry-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production decoder's synthetic vectors and epoch tests on the host. +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/g17/telemetry.rs"] +mod telemetry; From 79943c926549f6e04e9ace5c9830f203fa9a6a43 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 27 Oct 2026 16:25:57 -0300 Subject: [PATCH 034/171] drm/asahi: lock only the VM reservation for read-only VM queries Submit validation (covers_ranges_batch and friends), translate_offset, vm_context_mask, new_execution_context and capacity only read VmInner, but took the full GPUVM exec lock: the VM reservation plus every external BO's reservation. Each submit therefore cost O(external BOs). Under virtio-gpu every guest BO is external (libkrun maps blobs through dma-buf export), so a game with ~15k BOs spent 12-20 ms per submit in drm_gpuvm_prepare_objects and was capped at single-digit fps. Every page-table and mapping change already holds the VM reservation, so readers need only that lock; use GpuVm::lock_inner() for them. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit cde8dcbfc7ce6bda3a9d010457c009a757c65e91 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu.rs | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 50e06d3138d5ed..a7281ce144c160 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1350,9 +1350,7 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)? - .page_table - .translate_iova(address) + self.0.owner.lock_inner().page_table.translate_iova(address) } /// Writes a userspace timestamp through the retained GEM mapping. Aligned results @@ -2318,9 +2316,7 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2335,9 +2331,7 @@ impl Vm { if stride <= block_size { return false; } - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; @@ -2347,12 +2341,12 @@ impl Vm { }) } - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return (false, None); - }; + // Every mapping change holds the VM reservation, so readers need only that one. The full + // exec lock would also lock every external BO, making each submit O(external BOs). + let mut inner = self.gpuvm.lock_inner(); let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -3289,7 +3283,7 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } @@ -3448,7 +3442,7 @@ impl Uat { return Err(EINVAL); } { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3843,7 +3837,7 @@ impl Vm { /// Diagnostic reservation occupancy (including guards), not a placement promise. /// Removal can progress between queries; allocations are excluded by the VM lock. pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = lock_vm(&self.gpuvm, None, false)?; + let inner = self.gpuvm.lock_inner(); let mut next = range.start; let mut used = 0; let mut largest = 0; From f8835ae8bfdb680a147c385b0e20c7b04156df59 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 27 Oct 2026 16:25:57 -0300 Subject: [PATCH 035/171] drm/asahi: lock only the VM reservation for VA updates Every map and unmap (user VM_BIND, kernel mappings such as each G17 submission's completion record, object teardown) took GpuVm::exec_lock(), i.e. drm_gpuvm_exec_lock(): the VM reservation plus the reservation of every external object. Under virtio-gpu every guest BO is external, so with ~15k BOs each bind and each submit spent around a millisecond in drm_gpuvm_prepare_objects(). This VM is IMMEDIATE_MODE and RESV_PROTECTED: VA updates need the VM reservation and each object's gpuva lock, which the link and unlink paths take themselves. Backing is pinned and its segments copied before any reservation is taken, no caller adds fences, and the driver already unmaps under the VM reservation alone on its low-memory path. Add GpuVm::exec_lock_vm(), which runs the drm_exec loop over the VM reservation (and an external obj, if given) without drm_gpuvm_prepare_objects(), and use it for all VA updates. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: integration/aurora's SAFETY comments on the two exec locks are kept, naming the VM reservation] (cherry picked from commit f3cc4a691438be2e067a4035241ee028370ad1fd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu.rs | 16 +++--- drivers/gpu/drm/asahi/mmu/bind.rs | 4 +- rust/helpers/drm_gpuvm.c | 6 +++ rust/kernel/drm/gpuvm/mod.rs | 81 +++++++++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index a7281ce144c160..c1158ac5d3329e 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2475,7 +2475,7 @@ impl Vm { // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2563,7 +2563,7 @@ impl Vm { // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock(/*Some(&gem)*/ None, false) + .exec_lock_vm(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? @@ -2656,7 +2656,7 @@ impl Vm { // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2728,8 +2728,8 @@ impl Vm { prot: Prot, ) -> Result { // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // every subsequent access to VM data is serialized by the VM reservation. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2834,8 +2834,8 @@ impl Vm { let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // every subsequent access to VM data is serialized by the VM reservation. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2866,7 +2866,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index e222c7cd8e6a98..acce5677b9f923 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -135,7 +135,7 @@ impl Vm { PreparedUserBindOp::Map(map) => { // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is // not locked: the map step may still populate its pages. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let (flags, repeat) = if map.single_page { @@ -185,7 +185,7 @@ impl Vm { } PreparedUserBindOp::Unmap(unmap) => { // SAFETY: See `lock_vm()`. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let mut ctx = diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c index 30fb1777fbf2b7..d42d3d3491006f 100644 --- a/rust/helpers/drm_gpuvm.c +++ b/rust/helpers/drm_gpuvm.c @@ -23,6 +23,12 @@ bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, return drm_gpuvm_is_extobj(gpuvm, obj); } +__rust_helper +bool rust_helper_drm_exec_is_contended(struct drm_exec *exec) +{ + return drm_exec_is_contended(exec); +} + __rust_helper void rust_helper_drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) { diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index faecbedfa282fb..63fa50614c0374 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -60,6 +60,43 @@ pub use self::vm_bo::*; mod va; pub use self::va::*; +/// `drm_gpuvm_exec_lock()` without `drm_gpuvm_prepare_objects()`: the +/// `drm_exec_until_all_locked()` loop over the VM reservation and `vm_exec.extra`. +/// +/// # Safety +/// +/// `vm_exec` must be initialized as for `drm_gpuvm_exec_lock()`. +unsafe fn exec_lock_vm_raw(vm_exec: &mut bindings::drm_gpuvm_exec) -> core::ffi::c_int { + let exec = &raw mut vm_exec.exec; + // SAFETY: Per the safety requirements, `exec` belongs to a valid `drm_gpuvm_exec`. + unsafe { bindings::drm_exec_init(exec, vm_exec.flags, 0) }; + let mut ret = 0; + // SAFETY (whole loop): `exec` was initialized above and `vm_exec.vm` is a valid GPUVM. + // `drm_exec_cleanup()` drops all locks and restarts after contention, as the C macros do. + while unsafe { bindings::drm_exec_cleanup(exec) } { + ret = unsafe { bindings::drm_gpuvm_prepare_vm(vm_exec.vm, exec, vm_exec.num_fences) }; + if unsafe { bindings::drm_exec_is_contended(exec) } { + continue; + } + if ret == 0 { + if let Some(extra) = vm_exec.extra.fn_ { + ret = unsafe { extra(vm_exec) }; + if unsafe { bindings::drm_exec_is_contended(exec) } { + continue; + } + } + } + if ret != 0 { + break; + } + } + if ret != 0 { + // SAFETY: Finalize the context initialized above, as `drm_gpuvm_exec_lock()` does on error. + unsafe { bindings::drm_exec_fini(exec) }; + } + ret +} + pub(super) unsafe extern "C" fn exec_lock_gem_object( vm_exec: *mut bindings::drm_gpuvm_exec, ) -> core::ffi::c_int { @@ -260,6 +297,36 @@ impl GpuVm { &'a self, obj: Option<&T::Object>, interruptible: bool, + ) -> Result> { + // SAFETY: Same contract as this function. + unsafe { self.exec_lock_impl(obj, interruptible, true) } + } + + /// HACK: added for the asahi downstream driver. Like [`GpuVm::exec_lock`], but locks only + /// the VM's shared reservation (and `obj` if it is external), not every external object. + /// + /// Enough for VA updates of this immediate-mode, `DRM_GPUVM_RESV_PROTECTED` VM: they need + /// the VM reservation and each object's gpuva lock, which the link and unlink paths take. + /// Callers must not need external objects' reservations (fences, validation, eviction). + /// Locking all of them made each update cost O(external objects). + /// + /// # Safety + /// + /// Same as [`GpuVm::exec_lock`]. + pub unsafe fn exec_lock_vm<'a>( + &'a self, + obj: Option<&T::Object>, + interruptible: bool, + ) -> Result> { + // SAFETY: Same contract as this function. + unsafe { self.exec_lock_impl(obj, interruptible, false) } + } + + unsafe fn exec_lock_impl<'a>( + &'a self, + obj: Option<&T::Object>, + interruptible: bool, + all_objects: bool, ) -> Result> { // Do not try to lock the object if it is internal (since it is already locked). let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); @@ -286,10 +353,16 @@ impl GpuVm { GFP_KERNEL, )?; - // SAFETY: The object is valid and was initialized above. On failure, - // `drm_gpuvm_exec_lock()` has already finalized the exec context, so the box only needs to - // be freed, which dropping it does. - to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *vm_exec) })?; + // SAFETY: The object is valid and was initialized above. On failure, both lock functions + // have already finalized the exec context, so the box only needs to be freed, which + // dropping it does. + to_result(unsafe { + if all_objects { + bindings::drm_gpuvm_exec_lock(&mut *vm_exec) + } else { + exec_lock_vm_raw(&mut *vm_exec) + } + })?; // Build the guard, and take its VM reference, only once the locks are held: its `Drop` // unlocks them. From 7dee754efcdc26a612d27028ae76e34b5262d2d2 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Wed, 28 Oct 2026 11:47:53 -0300 Subject: [PATCH 036/171] drm/asahi: g17: stop cancelling behind a failed command once it has retired A queue remembered the earliest failed ordinal of each engine for good, and every later command whose prefix covered it was cancelled. GL orders each submission after every earlier one, so once one command of a queue failed, every later command of that queue was cancelled too. With cancellation no longer reported as a lost device, the client never noticed: after a CS2 GPU hang, one Hyprland render that sampled the game's failed buffer was cancelled, then every later Hyprland render (about 47 a second, all fences ECANCELED), and the screen kept the game's last frame until Hyprland or the machine restarted. New buffers did not help; the queue itself was poisoned. The record exists so that nothing runs before a failed producer the firmware may still own or replay. That holds only while the command keeps its VM pin: complete() and fail_unpublished() release it, fail_while_owned() keeps it until release_after_retirement(). Track the failed commands that still hold their pin and stop counting each one once it has released it. A failure that cannot be tracked for lack of memory keeps cancelling for good, as before. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/queue/mod.rs] (cherry picked from commit 925df2acdf61c4e4b21e98c35f0ffaa358eb9cd5 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/completion.rs | 5 + drivers/gpu/drm/asahi/g17/dependency.rs | 107 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 9 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 132 +++++++++--------------- 4 files changed, 166 insertions(+), 87 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index e7249cdc3d32cb..7d887f17d22c68 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -631,6 +631,11 @@ impl Completion { drop(lease); } + /// Whether the command still holds its VM pin: firmware may still own or replay it. + pub(crate) fn retains_vm(&self) -> bool { + self.vm_job.lock().is_some() + } + fn release_healthy(&self) { if let Some(sources) = &self.sources { sources.release(); diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 0bc1d5024566e7..387ce9ba2fb3c1 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -25,6 +25,48 @@ pub(crate) fn replay_status(status: i32) -> Result { else { Ok(()) } } +/// Whether a failed command may still run or write: it holds its VM pin until it has physically +/// retired, or was never published. +pub(crate) trait Retention { + fn retained(&self) -> bool; +} + +/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix +/// covers one is cancelled, so none runs before a failed producer the firmware may still replay. +/// Once that producer has retired nothing of it can run or write any more, so it stops counting: +/// otherwise one cancelled command cancelled every later command of its queue for good. +pub(crate) struct FailureRecord { + retained: KVec<(u64, T)>, + /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did. + untracked: Option<(u64, T)>, +} + +impl FailureRecord { + pub(crate) const fn new() -> Self { + Self { retained: KVec::new(), untracked: None } + } + + /// A finished frontier entry, at pruning. + pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) { + if status >= 0 || !entry.retained() { + return; + } + if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err() + && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first) + { + self.untracked = Some((sequence, entry.clone())); + } + } + + /// The earliest failure later prefixes must not bypass. + pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> { + self.retained.retain(|(_, entry)| entry.retained()); + Iterator::chain(self.retained.iter(), self.untracked.iter()) + .min_by_key(|(sequence, _)| *sequence) + .map(|(sequence, entry)| (*sequence, entry)) + } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], @@ -89,3 +131,68 @@ impl Frontier { self.insert(sequence, dependency) } } + +#[kunit_tests(asahi_g17_dependency)] +mod tests { + use super::*; + + #[derive(Clone)] + struct Probe(kernel::sync::Arc); + + impl Probe { + fn new(retained: bool) -> Self { + Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap()) + } + fn retire(&self) { + self.0.store(false, core::sync::atomic::Ordering::Relaxed); + } + } + + impl Retention for Probe { + fn retained(&self) -> bool { + self.0.load(core::sync::atomic::Ordering::Relaxed) + } + } + + fn earliest(record: &mut FailureRecord) -> Option { + record.earliest().map(|(sequence, _)| sequence) + } + + #[test] + fn a_cancelled_command_does_not_poison_its_queue() { + // Cancelled or failed before publication: it released its pin and can never run. + let mut record = FailureRecord::new(); + record.note(5, ECANCELED.to_errno(), &Probe::new(false)); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_retained_failure_cancels_until_it_retires() { + let mut record = FailureRecord::new(); + let failed = Probe::new(true); + record.note(5, EIO.to_errno(), &failed); + assert_eq!(earliest(&mut record), Some(5)); + failed.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_later_retained_failure_survives_an_earlier_retirement() { + let mut record = FailureRecord::new(); + let (first, second) = (Probe::new(true), Probe::new(true)); + record.note(5, EIO.to_errno(), &first); + record.note(9, ECANCELED.to_errno(), &second); + assert_eq!(earliest(&mut record), Some(5)); + first.retire(); + assert_eq!(earliest(&mut record), Some(9)); + second.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn successes_are_not_recorded() { + let mut record = FailureRecord::new(); + record.note(5, 1, &Probe::new(true)); + assert_eq!(earliest(&mut record), None); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b6c61e3bba847d..4fabe7d4b3f4ce 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -401,6 +401,13 @@ pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, pub(crate) publication: Option>, + pub(crate) owner: Arc, +} + +impl super::dependency::Retention for Fences { + fn retained(&self) -> bool { + self.owner.retains_vm() + } } impl Fences { @@ -479,6 +486,6 @@ impl Scheduler { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed, publication: host_publication }) + Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() }) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index e593c5879cf741..f6637f2858c26d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,6 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, + dependency::FailureRecord, fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, @@ -84,7 +85,6 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, - usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -99,16 +99,13 @@ pub(crate) struct Queue { /// Finished frontier entries pruned during this ioctl. Their last references /// can tear a whole completion down; that waits until the ioctl has published. finished: KVec, - failures: [Option<(u64, Fence)>; 2], + failures: [FailureRecord; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } -type ValidatedBatch = ( - KVec<(Validated, Order, Destinations)>, - KVec, -); +type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); impl Queue { pub(crate) fn new( @@ -117,7 +114,6 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, - usage: Arc, ) -> Result { let Frontend { fences, @@ -130,7 +126,6 @@ impl Queue { Ok(Self { dev: dev.into(), context, - usage, backend, render, compute: None, @@ -143,7 +138,7 @@ impl Queue { sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), finished: KVec::new(), - failures: [None, None], + failures: [FailureRecord::new(), FailureRecord::new()], _slot: slot, }) } @@ -165,11 +160,7 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; // Without room the entries are released here, as they always could be. - let mut finished = if self - .finished - .reserve(self.frontiers[index].len(), GFP_KERNEL) - .is_ok() - { + let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { Some(&mut self.finished) } else { None @@ -182,11 +173,7 @@ impl Queue { if let Some(finished) = finished.as_mut() { let _ = finished.push_within_capacity(fences.clone()); } - // One earliest failed ordinal covers every later prefix without - // retaining its packet, mappings, or every failed fence. - if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { - *failure = Some((*sequence, fences.completed.clone())); - } + failure.note(*sequence, status, fences); false }); } @@ -211,19 +198,13 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self - .backend - .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { - return Err(EINVAL); - } + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { - return Err(EINVAL); - }; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } // Only convert inputs when no independent host-prefix wait could keep @@ -231,34 +212,21 @@ impl Queue { if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute - && !inputs.is_empty() + let sources = if engine == Engine::Compute && !inputs.is_empty() && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| { - !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 - }) - && self.frontiers[1].iter().all(|(sequence, fences)| { + && self.frontiers[0].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[0], *sequence) + || fences.status() > 0) + && self.frontiers[1].iter().all(|(sequence, fences)| !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0 - }) - && self.failures.iter().enumerate().all(|(index, failure)| { - failure.as_ref().is_none_or(|(sequence, _)| { - !Order::contains(order.wait_through[index], *sequence) - }) - }) { - self.backend.input_sources(inputs) - } else { - None - }; + || fences.status() > 0) + && self.failures.iter_mut().enumerate().all(|(index, failure)| + failure.earliest().is_none_or(|(sequence, _)| + !Order::contains(order.wait_through[index], sequence))) + { self.backend.input_sources(inputs) } else { None }; if engine == Engine::Compute { - super::trace::inputs( - self.context.id(), - order.sequence, - 1, - inputs.len(), - sources.as_deref(), - ); + super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); } // Only earlier render ordinals covered by the explicit logical prefix @@ -266,7 +234,8 @@ impl Queue { let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -280,10 +249,8 @@ impl Queue { host_predecessors.clone().next().is_some(), self.backend.feed(), sources.clone(), - self.usage.clone(), )?; - let publication = completion - .publication() + let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -314,16 +281,15 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); - let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { - self.failures[index].as_ref().and_then(|(sequence, fence)| { - Order::contains(order.wait_through[index], *sequence).then_some(fence) - }) - }); + let mut failures: [Option; 2] = [None, None]; + for (index, failure) in self.failures.iter_mut().enumerate() { + failures[index] = failure.earliest().and_then(|(sequence, fences)| { + Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone()) + }); + } let failure_count = failures.iter().flatten().count(); - let mut dependencies = KVec::with_capacity( - inputs.len() + count + timestamp_count + failure_count, - GFP_KERNEL, - )?; + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -361,10 +327,7 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } - if sources - .as_ref() - .is_some_and(|sources| sources.contains_input(fence)) - { + if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { continue; } dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -394,11 +357,10 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = - scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -465,10 +427,8 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [ - super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), - ]; + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -482,22 +442,22 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending - .iter() - .any(|(command, _, _)| matches!(command, Validated::Render { .. })) - { + if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { self.context.note_render(); } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { - command.allow_long_job(); - } + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } let mut guards = guards.into_iter(); let firmware_source = pending.len() == 1 From a2c04516a88f174e4b6a2b41ee33e098c3b19085 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 14:07:55 +1100 Subject: [PATCH 037/171] drm/asahi: take only the VM reservation in drop_mappings_before() aurora-wip now takes GpuVm::exec_lock_vm() for every VA update, and drop_mappings_now() unmaps under the VM reservation alone (f3cc4a691438, "drm/asahi: lock only the VM reservation for VA updates"). The deferred close path from omacom 13c3371ce23d was written against the full exec_lock(), which also locks the reservation of every external object, and the merge kept that call. Use exec_lock_vm() there too. The newest-binding check still runs under the lock every bind takes, and the unmap needs nothing more than the VM reservation and each object's gpuva lock, as in drop_mappings_now(). Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 2d5d141e229376869a205ec81e887e929bdc2d18 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index c1158ac5d3329e..ca224008dc573a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2905,7 +2905,8 @@ impl Vm { // The BO's newest sequence is checked under the same lock as binds. If // it is not newer than the close, every mapping belongs to that close // and the per-object list is safe and substantially cheaper to walk. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => { return self.drop_mappings_filtered(gem, sequence); } From d395fa703c56bd83c97e3e0bb38b3299c5586b30 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 27 Oct 2026 23:56:08 -0300 Subject: [PATCH 038/171] drm/asahi: g17: keep innocent clients alive across another client's recovery A recovery for one client's hang still killed the others: a CS2 reason-0 recovery cancelled 42 commands and Hyprland got ECANCELED from fail_unpublished(), which Honeykrisp treats as a lost device, so the desktop froze until a reboot. - ECANCELED is not reported as this client's failure from fail_unpublished(), the ordered output publication, or fail_while_owned() for spared work. Work the firmware listed as interrupted keeps reporting its failure. - Publication waits for a pending recovery, or for this owner's spared pair to drain, for up to RECOVERY_WAIT_MS (26 s) instead of the usual 2 s: the compute admission deadline extends (Admission:: extend_for_recovery) in ensure_compute, in the recovery waits of compute_packet including the input-source wait, and in Gate::enter_until, whose gate only a recovery closes; the render waits keep a WaitClock whose usual bound starts afresh once the recovery ends, and render admission extends its deadline while a recovery is pending. A spared quarantined pair waits until the caller's deadline; a guilty pair's publication still fails after a second. - Commands with failed inputs or predecessors stay cancelled: a failed fence does not prove its producer retired. The compute deadline host tests move to the longer recovery budget, and a mutant now checks that removing the extension fails them. Link: https://github.com/aurora-silicon/linux/pull/433 Co-developed-by: Ryan Murray Signed-off-by: Ryan Murray Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of (no-host-test:g17-admission-deadline-tests.py) (no-host-test:g17-compute-constructor-tests.py) (no-host-test:g17-compute-deadline-tests.py) (no-host-test:g17-compute-growth-deadline-tests.py)] (cherry picked from commit 62109cc021f51b246875bd513471fa2a1bf56bf1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- .../drm/asahi/g17/completion/publication.rs | 4 +- drivers/gpu/drm/asahi/g17/dependency.rs | 11 ++ drivers/gpu/drm/asahi/g17/preparation.rs | 51 +++++-- drivers/gpu/drm/asahi/g17/queue/admission.rs | 38 +++-- drivers/gpu/drm/asahi/g17/runtime.rs | 19 ++- .../drm/asahi/g17/runtime/render/submit.rs | 130 +++++++++++++++--- drivers/gpu/drm/asahi/g17/timeout.rs | 7 + include/uapi/drm/asahi_drm.h | 5 +- 9 files changed, 228 insertions(+), 45 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 7d887f17d22c68..90f6d77780bff0 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -694,7 +694,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); - if self.accepted.load(Ordering::Acquire) { + // As in complete(): a cancellation is not this client's device loss. A guilty VM + // already carries its own sticky error. + if self.accepted.load(Ordering::Acquire) && error != ECANCELED { self.status.report_failure(error); } self.release_healthy(); @@ -713,7 +715,9 @@ impl Completion { guard.quarantine(); } if self.spared() { - self.status.report_failure(error); + if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } } else { self.report(error); } diff --git a/drivers/gpu/drm/asahi/g17/completion/publication.rs b/drivers/gpu/drm/asahi/g17/completion/publication.rs index a1e05a9408552c..fc4b4f8739015e 100644 --- a/drivers/gpu/drm/asahi/g17/completion/publication.rs +++ b/drivers/gpu/drm/asahi/g17/completion/publication.rs @@ -96,7 +96,9 @@ impl Publication { if let Err(error) = result { // Inherited failure is visible to the client without inventing a // permanent admission failure in this command's VM. - self.status.report_failure(error); + if error != ECANCELED { + self.status.report_failure(error); + } } self.output.publish(result); let result = result.map(|_| ()); diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 387ce9ba2fb3c1..b7cb74a59644f1 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -195,4 +195,15 @@ mod tests { record.note(5, 1, &Probe::new(true)); assert_eq!(earliest(&mut record), None); } + + #[test] + fn failed_predecessor_cannot_be_bypassed() { + let mut frontier = Frontier::new(); + for error in [ECANCELED, EIO, ETIMEDOUT] { + assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED)); + } + assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(())); + assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(())); + assert_eq!(frontier.count, 0); + } } diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 2aa32131150767..c7d1761699a24b 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -6,6 +6,7 @@ //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is //! stored after its tentative mappings so those mappings are destroyed before the join completes. +use core::cell::Cell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::{ new_condvar, new_mutex, @@ -24,25 +25,37 @@ const EPOCH: u64 = CLOSED << 3; /// packet cancellation, logical close, VM status and device loss even when no /// capacity/recovery event arrives. Wakes and nested constructors never renew it. pub(super) struct Admission<'a> { - deadline: u64, + deadline: Cell, + started: u64, check: &'a dyn Fn() -> Result, } impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { - Self { deadline, check } + Self { deadline: Cell::new(deadline), started: super::now_ns(), check } + } + + /// A recovery is pending or this owner's spared work drains: wait as long as the + /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing. + pub(super) fn extend_for_recovery(&self) { + let limit = self + .started + .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000); + if self.deadline.get() < limit { + self.deadline.set(limit); + } } pub(super) fn check(&self) -> Result { (self.check)()?; - if super::now_ns() >= self.deadline { return Err(EBUSY); } + if super::now_ns() >= self.deadline.get() { return Err(EBUSY); } Ok(()) } /// Ignore condvar remaining-time reports: only the monotonic deadline counts. pub(super) fn slice_ms(&self) -> Result { self.check()?; - let remaining = self.deadline.saturating_sub(super::now_ns()); + let remaining = self.deadline.get().saturating_sub(super::now_ns()); if remaining == 0 { return Err(EBUSY); } Ok(remaining.div_ceil(1_000_000).min(10) as u32) } @@ -137,19 +150,20 @@ impl Gate { } } - /// Packet admission may wait for recovery only within its original budget. + /// Only a recovery closes the gate: packet admission waits for it up to + /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery). /// Called with no device mutex or existing preparation lease. pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let mut wait = self.wait.lock(); loop { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let _ = self.changed.wait_timeout( @@ -322,6 +336,20 @@ mod growth_gate_tests { mod compute_deadline_tests { use super::*; + #[test] + fn a_recovery_extends_the_deadline_once_from_the_start() { + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert_eq!(admission.check(), Err(EBUSY)); + admission.extend_for_recovery(); + assert_eq!(admission.check(), Ok(())); + let limit = admission.started + + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000; + assert_eq!(admission.deadline.get(), limit); + admission.extend_for_recovery(); + assert_eq!(admission.deadline.get(), limit); + } + #[test] fn spent_budget_refuses_even_an_open_gate() { let gate = Gate::new().unwrap(); @@ -332,13 +360,16 @@ mod compute_deadline_tests { } #[test] - fn closed_gate_expires_without_any_recovery_wake() { + fn closed_gate_waits_past_the_budget_for_recovery() { let gate = Gate::new().unwrap(); gate.close(); let start = super::super::now_ns(); - let check = || Ok(()); + // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting. + let check = || { + if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) } + }; let admission = Admission::new(start + 20_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); assert!(super::super::now_ns() - start < 500_000_000); gate.wait_drained(); assert!(gate.reopen()); diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ed73323e34e6ff..f09e629aa047e8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -494,6 +494,16 @@ impl Pool { } } } + /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. + pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { + let state = self.state.lock(); + state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) + } + pub(crate) fn acquire_render( self: &Arc, owner: u64, @@ -525,17 +535,22 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if now >= quarantine_deadline { - // Behind an innocent pair's quarantine (another VM wedged the GPU) only this - // publication fails, not the VM: it recovers once the pair drains. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); - return Err(if spared { EBUSY } else { EIO }); + // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the + // pair to drain until the caller's deadline, which the caller extends while it + // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + if now >= quarantine_deadline && !spared { + return Err(EIO); } - let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + let slice = if spared { + remaining_ms + } else { + remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?) + }; if matches!(self.released.wait_interruptible_timeout( &mut state, kernel::time::msecs_to_jiffies(slice) ), CondVarTimeoutResult::Signal { .. }) { @@ -1001,10 +1016,13 @@ mod fault_capacity_tests { let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); + assert!(!pool.spared_quarantined(1)); lease.quarantine(true); assert_ne!(pool.state.lock().spared & bit, 0); + assert!(pool.spared_quarantined(1)); lease.quarantine(false); assert_eq!(pool.state.lock().spared & bit, 0); + assert!(!pool.spared_quarantined(1)); // Once guilty, a later spared failure does not make it spared again. lease.quarantine(true); assert_eq!(pool.state.lock().spared & bit, 0); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index a6edfd0a80b384..d391fbb3001bbe 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1756,8 +1756,10 @@ impl Backend { } } // Other owners continue submitting while this owner waits for its - // queue-local drain witness. No new command has been accepted: the - // spared queue and its retained work stay intact past the deadline. + // queue-local drain witness. No new command has been accepted; if the + // admission deadline passes, the spared queue and its retained work stay + // intact until a later admission observes its witness. + admission.extend_for_recovery(); self.shared.queue_events(); self.wait_release(&self.shared.progress, observed, admission)?; } @@ -2090,6 +2092,13 @@ impl Backend { admission.check()?; let observed = progress.observe(); if !self.sources_published(packet)? { + let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { + firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + }); + if recovering { + admission.extend_for_recovery(); + } self.shared.queue_events(); self.wait_release(progress, observed, &admission)?; continue; @@ -2131,6 +2140,7 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { + admission.extend_for_recovery(); drop(state); drop(lease); self.shared.queue_events(); @@ -2162,6 +2172,11 @@ impl Backend { { // Recovery or backpressure: nothing is firmware-visible. Other // queues keep submitting and the event worker retires meanwhile. + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + admission.extend_for_recovery(); + } drop(state); drop(lease); self.shared.queue_events(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index f1a433ddb0db49..970035547dfee2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -54,13 +54,22 @@ impl Backend { ) -> Result<(RenderLease, bool)> { let pool = &self.admission; let started = super::super::now_ns(); - let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let mut deadline_ns = + started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let recovery_deadline_ns = started.saturating_add(recovery_wait_ns()); // A capacity refusal samples the render domain and retries once; only // a refusal after that sample waits, so no release can be missed. let mut observed = None; + let mut capacity = WaitClock::default(); loop { self.admission_open()?; - if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if super::super::now_ns() >= deadline_ns { + if deadline_ns < recovery_deadline_ns && self.waiting_for_recovery()? { + deadline_ns = recovery_deadline_ns; + } else { + return Err(EBUSY); + } + } if packet.cancel_requested() { return Err(ECANCELED); } @@ -95,11 +104,13 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(&mut observed, started, sleep)?; + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); continue; } + // The deadline check above decides whether a recovery extends the wait. + Err(error) if error == EBUSY => continue, Err(error) => return Err(error), } }; @@ -133,7 +144,7 @@ impl Backend { drop(state); // A full queue table makes admission reuse a free killed pair instead. if reclaimed.is_none() && !ready && !pool.killed_pair_free() { - self.wait_render_capacity(&mut observed, started, sleep)?; + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); } @@ -221,13 +232,29 @@ impl Backend { domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) } + /// A recovery is pending, or this owner's pair is spared and draining after one: an + /// innocent publication waits up to RECOVERY_WAIT_MS for it instead of failing. + fn waiting_for_recovery(&self) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + Ok(firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + || firmware.queues.admission.spared_quarantined(self.owner)) + } + + /// The deadline of the wait `clock` measures (WaitClock::deadline). + fn wait_deadline(&self, clock: &mut WaitClock, bound_ns: u64) -> Result { + let recovering = self.waiting_for_recovery()?; + Ok(clock.deadline(super::super::now_ns(), recovering, bound_ns)) + } + /// A pair is borrowed, still settling, waiting for firmware room or for the /// release of the scheduler it last ran for. Every pair return, retirement /// and teardown step signals the render domain, and each event pass does /// while a publisher waits; the deadline is only the failure bound. fn wait_render_busy( &self, - since: &mut Option, + clock: &mut WaitClock, observed: &mut Option, sleep: bool, packet: &Packet, @@ -235,8 +262,7 @@ impl Backend { if !sleep { return Err(FALLBACK); } - let now = super::super::now_ns(); - let deadline = since.get_or_insert(now).saturating_add(super::super::WORK_RING_WAIT_NS); + let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; // Sample, then retry once: only a refusal after the sample waits. let Some(sample) = observed.take() else { *observed = Some(self.shared.renders.observe()); @@ -249,7 +275,12 @@ impl Backend { /// Render capacity is short: return what idle owners hold, then block until /// a pair, lease or owner returns. No packet is firmware-visible here, and /// neither the device mutex nor a preparation lease is held while waiting. - fn wait_render_capacity(&self, observed: &mut Option, started: u64, sleep: bool) -> Result { + fn wait_render_capacity( + &self, + observed: &mut Option, + clock: &mut WaitClock, + sleep: bool, + ) -> Result { if !sleep { return Err(FALLBACK); } @@ -263,8 +294,8 @@ impl Backend { }; self.shared.queue_events(); self.admission_open()?; - let remaining = started - .saturating_add(super::super::WORK_RING_WAIT_NS) + let remaining = self + .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? .saturating_sub(super::super::now_ns()); if remaining == 0 { return Err(EBUSY); @@ -367,9 +398,10 @@ impl Backend { return result; } } - let mut busy_since = None; - let capacity_started = super::super::now_ns(); - let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); + let mut busy = WaitClock::default(); + let mut capacity = WaitClock::default(); + let mut state_clock = WaitClock::default(); + let mut recovery_since = None; let mut seen = None; let mut states = None; loop { @@ -407,8 +439,11 @@ impl Backend { let work = packet.completion.work_state()?; match states.take() { None => states = Some(work.returns().observe()), + // Work states free as earlier, possibly spared, work retires. Some(sample) => { - self.wait_domain(work.returns(), sample, deadline_ns, packet)? + let deadline = self + .wait_deadline(&mut state_clock, super::super::WORK_RING_WAIT_NS)?; + self.wait_domain(work.returns(), sample, deadline, packet)? } } continue; @@ -426,19 +461,19 @@ impl Backend { }; match ensured { Err(error) if error == ENOSPC => { - self.wait_render_capacity(&mut seen, capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(&mut seen, capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { @@ -458,10 +493,13 @@ impl Backend { // Recovery closes the preparation gate before touching // any queue; the next pass blocks in enter() until it // reopens. The worker raises the closure. + let since = *recovery_since.get_or_insert(super::super::now_ns()); self.shared.queue_events(); - self.shared.preparations.wait_closure(gate, deadline_ns)?; + self.shared + .preparations + .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } @@ -699,3 +737,57 @@ impl crate::g17::Firmware { result } } + +fn recovery_wait_ns() -> u64 { + u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 +} + +/// One bounded wait (Backend::wait_deadline). +#[derive(Default)] +struct WaitClock { + first: Option, + calm: u64, +} + +impl WaitClock { + /// `bound_ns` after the later of the first call and the last call that saw a recovery + /// pending or this owner's spared pair draining (so the usual bound applies afresh once a + /// recovery ends), and at most RECOVERY_WAIT_MS after the first call. + fn deadline(&mut self, now: u64, recovering: bool, bound_ns: u64) -> u64 { + let first = *self.first.get_or_insert(now); + self.calm = if recovering { now } else { self.calm.max(first) }; + first + .saturating_add(recovery_wait_ns()) + .min(self.calm.saturating_add(bound_ns)) + } +} + +#[kunit_tests(asahi_g17_wait_clock)] +mod tests { + use super::*; + + const BOUND: u64 = 2_000_000_000; + + #[test] + fn a_calm_wait_keeps_its_first_deadline() { + let mut clock = WaitClock::default(); + assert_eq!(clock.deadline(100, false, BOUND), 100 + BOUND); + assert_eq!(clock.deadline(500, false, BOUND), 100 + BOUND); + } + + #[test] + fn the_bound_starts_afresh_once_a_recovery_ends() { + let mut clock = WaitClock::default(); + clock.deadline(100, false, BOUND); + assert_eq!(clock.deadline(1_000, true, BOUND), 1_000 + BOUND); + assert_eq!(clock.deadline(2_000, false, BOUND), 1_000 + BOUND); + } + + #[test] + fn a_recovery_cannot_extend_past_the_recovery_wait() { + let mut clock = WaitClock::default(); + clock.deadline(100, true, BOUND); + let limit = 100 + recovery_wait_ns(); + assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); + } +} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index b83ea5d07f406e..ce003e6b181961 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -18,6 +18,13 @@ pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS pub(crate) const POLL_MS: u32 = 2000; const EXECUTION_RENEWALS: u8 = 4; +/// How long an innocent client's publication waits for a recovery to settle and its +/// spared work to drain, instead of failing with EBUSY: the watchdog defers work behind +/// another VM at most this long before it settles it (`classify`; ray work gets the +/// execution renewals on top). +pub(crate) const RECOVERY_WAIT_MS: u32 = + POLL_MS * (1 + EXECUTION_RENEWALS as u32 + MAX_RENEWALS as u32); + /// A background control/replay poll has no scheduler fence to expire it. /// Its timestamp belongs to the exact transaction and is never renewed by work. pub(crate) fn background_stalled(since: Option, now: u64) -> bool { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index dc34315ec8c822..418522205ee6f8 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -445,7 +445,10 @@ enum drm_asahi_feature { * The driver publishes an accepted job's failure before signaling its fence. * Thus userspace must query after a successful DRM syncobj wait/poll before * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. + * an execution-success indication. A job cancelled (ECANCELED) because + * another VM's recovery discarded it, an input did not signal in time, or + * its queue was closed, is not a failure of this VM: its fence carries the + * error, error stays 0, and its output is undefined. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. From 58f751efd5e9f0bb6725dcff4d4607d1cf004db5 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Wed, 28 Oct 2026 09:44:12 -0300 Subject: [PATCH 039/171] drm/asahi: g17: cancel consumers of a failed producer whatever it failed with A command whose input fence, ordered predecessor or compute source failed inherits the failure through inherited_error(). Only EIO, ENODEV and ETIMEDOUT became ECANCELED; any other error passed through unchanged and was reported as the consumer's own failure, which userspace treats as a lost device. A GPU recovery fails the render groups it dropped and cannot replay with ENODATA. One such recovery (render slot 10, 11 groups of a muvm GPU worker) took down Xwayland, whose next command consumed one of the worker's buffers. Xwayland's own commands then failed, so its buffers kept that failure in their fences, and 85 minutes later a newly started muvm GPU worker that imported them failed with ENODATA too, without any recovery. Cancel the consumer for every producer failure. ECANCELED is not reported as a device loss and does not poison the consumer's VM; its fence still carries the error, as the UAPI documents. An interrupted fence wait is not a producer failure and keeps its own error. G15 reports every input failure as a client failure anyway, so it keeps inherited_error() unchanged. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 432f851c8e3195e77f1c334574130f1743e23407 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/completion/publication.rs | 8 +++----- drivers/gpu/drm/asahi/g17/dependency.rs | 8 +++++++- drivers/gpu/drm/asahi/g17/fence.rs | 2 +- drivers/gpu/drm/asahi/g17/job.rs | 5 +++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +--- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 1 + 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion/publication.rs b/drivers/gpu/drm/asahi/g17/completion/publication.rs index fc4b4f8739015e..8e16a14f054886 100644 --- a/drivers/gpu/drm/asahi/g17/completion/publication.rs +++ b/drivers/gpu/drm/asahi/g17/completion/publication.rs @@ -8,7 +8,7 @@ //! nor waits for another command; the last prerequisite publishes a ready result. use super::Output; -use crate::g17::{dependency::inherited_error, status::VmStatus}; +use crate::g17::{dependency::PRODUCER_FAILED, status::VmStatus}; use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}}; type Links = Option>; @@ -121,10 +121,8 @@ impl Publication { let ready = { let mut state = self.state.lock(); debug_assert!(state.remaining != 0); - if let Err(error) = result { - if state.predecessor_error.is_none() { - state.predecessor_error = Some(inherited_error(error)); - } + if result.is_err() && state.predecessor_error.is_none() { + state.predecessor_error = Some(PRODUCER_FAILED); } state.remaining -= 1; state.claim() diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index b7cb74a59644f1..5a0ed611b1ddee 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -9,6 +9,7 @@ use super::{ use kernel::prelude::*; /// A failed producer does not establish a fault in its consumer's VM. +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn inherited_error(error: Error) -> Error { if error == EIO || error == ENODEV || error == ETIMEDOUT { ECANCELED @@ -17,11 +18,16 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its +/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with +/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too. +pub(crate) const PRODUCER_FAILED: Error = ECANCELED; + /// A replay's original host inputs must still be signalled. Treat an unmet /// input as a failed replay, never as a wait under the device mutex. pub(crate) fn replay_status(status: i32) -> Result { if status == 0 { Err(EBUSY) } - else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else if status < 0 { Err(PRODUCER_FAILED) } else { Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 81d1d6fe43960d..96043f736c5c46 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -180,7 +180,7 @@ impl Submission { self.remaining.load(Ordering::Acquire) == 0 || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); if error != 0 { - return Err(super::dependency::inherited_error(Error::from_errno(error))); + return Err(super::dependency::PRODUCER_FAILED); } Ok(complete) } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 4fabe7d4b3f4ce..9e3f7092ea91ec 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -177,8 +177,9 @@ impl Packet { kernel::time::msecs_to_jiffies(10) as isize, ) }; + // An interrupted wait is not a producer failure. if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + return Err(Error::from_errno(waited as i32)); } if waited > 0 { break; } } @@ -257,7 +258,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 { fn check_status(fence: &Fence) -> Result { let status = fence_status(fence); if status < 0 { - Err(super::dependency::inherited_error(Error::from_errno(status))) + Err(super::dependency::PRODUCER_FAILED) } else { Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 368e95f144c240..55994066825f81 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2056,9 +2056,7 @@ impl Queue { let ticket = active.ticket.ok_or(EIO)?; let state = crate::g17::job::fence_status(&active.packet.completion.fence()); if state < 0 { - return Err(crate::g17::dependency::inherited_error(Error::from_errno( - state, - ))); + return Err(crate::g17::dependency::PRODUCER_FAILED); } if state > 0 { return Ok(None); diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index 082416f0cb9a66..d71d3cc81553f2 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -126,5 +126,6 @@ mod tests { assert_eq!(replay_status(1), Ok(())); assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED)); } } From de668d6879d75bfd40181f852770f67ccf55667a Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 01:46:40 -0300 Subject: [PATCH 040/171] drm/asahi: g17: accept ray traversal on 25G83 Ray-traversal work is refused on the M4 Pro's 25G83 firmware because a stalled traversal was thought to wedge the GPU with no firmware recovery. That does not reproduce. A BLAS overwritten with a node cycle got a firmware reason-0 recovery in 1.76 s from a compute shader and 0.12 s from a fragment shader, with no MMU fault, and the GPU worked afterwards. Valid traversals are not cut off: one ray that must visit every leaf ran correctly for 31 s in compute and 14 s in fragment, and 16,384 of them finished in a 5 s fragment job. A compute job stops at the driver's own 38 s limit in timeout.rs, and only that client fails. The Vulkan ray tracing CTS lists, 74,833 cases, gave 49,711 Pass, 25,122 NotSupported, 0 Fail and 0 GPU recoveries with the Mesa fixes in aurora-silicon/mesa#139. Two limits remain. A recovery can still fail a running render of another client, as with any GPU hang. And all render work waits behind a fragment job, so a fragment traversal of many seconds freezes the desktop, and past about 24 s the compositor fails with ETIMEDOUT. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 195fbb2d50dd73699211bbd96b524fb94b34bb09 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/variant.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 39c8cfcd77a2a9..03d5bfccafeca2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -126,8 +126,7 @@ pub(crate) const V25G83: Variant = Variant { mcache: false, // A fragment stage with the few-primitives flag hangs. few_primitives: false, - // A stalled ray traversal wedges the GPU without a firmware recovery. - ray_traversal: false, + ray_traversal: true, // When a tiling pass runs out of memory with this set, the firmware resumes it without // adding memory and the tiler stops silently; macOS always writes 0. fragment_busy: false, From 2ceed5cadaeaa52bba10524d0dedfeb1593c6131 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 041/171] drm/asahi: g17: program the tiler's segment limits on 25G83 The TA arrays wrote 0 to 0x1c890, 0x1c918 and 0x1c9a8, upstream's tvb_tiling_control, tiling_control_2 and min_free_tvb_pages. macOS writes 0x200440, 4 and 0x3c on two-cluster parts, all three for every pass. Partial renders need 0x1c9a8. Without it the pages each fragment pass returns shrink geometrically, 122, 66, 37, 24, 13, 7, to none, and the tiler spins through about 140,000 empty stops until the render is killed. With it segments hold at 84-88 pages and the render finishes. 0x1c890 and 0x1c918 alone don't do that, and macOS writes them together. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 6b90842359453a57a3b590cf3dd666c43a3cf803 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index a6bc1704d1418d..3ab33304bc8a89 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -639,6 +639,13 @@ fn ta_registers( /// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage /// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the /// M4 Pro has run with. Returns the array and its used entry count. +/// Tiler segment control of a two-cluster pass, what macOS writes. +const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; +const VDM_SEGMENT_INDEX: u64 = 4; +/// Pages the tiler keeps out of a partial render's segments: without them the pages a pass +/// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. +const MIN_FREE_TVB_PAGES: u64 = 0x3c; + fn ta_registers_25g83( args: &RenderArgs<'_>, g: &Geometry, @@ -686,15 +693,15 @@ fn ta_registers_25g83( push(find(0x1c880, 0)?); push(RegisterWrite::new(0x1c948, gtp(4)?)); push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, 0)); - push(RegisterWrite::new(0x1c918, 0)); + push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); + push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); push(find(0x1c079, 0)?); push(find(0x1c9d8, 0)?); push(RegisterWrite::new(0x1c089, gtp2)); push(RegisterWrite::new(0x1c9e0, gtp2)); push(RegisterWrite::new(0x16c41, gtp6)); push(RegisterWrite::new(0x1ca40, gtp6)); - push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c9a8, MIN_FREE_TVB_PAGES)); push(RegisterWrite::new(0x1c920, gtp(7)?)); for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { push(find(reg, 0)?); From 1827a50695673a28dd7026ecdb88f20995b0cf1c Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 042/171] drm/asahi: g17: double the render pool so a partial render can't starve The render pool holds the USC private memory of a context's vertex and fragment threads. It is fixed at 22 MiB. When the tiler stops for a partial render its threads keep what they hold, and the fragment pass needs its own. At 22 blocks fragment pass 2 stalls for 2.2 ms when it is lucky, and otherwise never finishes and the firmware recovers with reason 0. With process_empty_tiles, pass 1 hangs the same way on its first thread. On a 3456x2234 render of 3M triangles, 22 blocks lost every run with the first stop at 255 pages. 27 blocks passed 12 of 12, heavy shaders included. Clearing the pool registers instead takes the 2.2 ms off pass 2, but then the tiler gets no pages back. 44 is measured, not derived: it leaves margin over the 27 that passed, for shaders heavier than the test's. It costs 22 MiB per rendering context in the 2 GiB lower window, on A18 Pro as well, which shares this constant. Refused allocations in a dEQP run at 8 jobs went from 0 to 15, all retried. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 8efa1323092ae140b5d517ecebe8657ce49d8166 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/freelist.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs index 2e5708dddd4552..86fdc5db75a8f7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -9,7 +9,7 @@ pub(crate) const PAGE_LIST_SIZE: usize = 0x20_0000; pub(crate) const RUN_LIST_SIZE: usize = 0x1_0000; pub(crate) const TABLE_SIZE: usize = PAGE_LIST_SIZE + RUN_LIST_SIZE; pub(crate) const CONTROL_SIZE: usize = 0x4000; -pub(crate) const RENDER_BLOCKS: usize = 22; +pub(crate) const RENDER_BLOCKS: usize = 44; pub(crate) const COMPUTE_BLOCKS: usize = 21; /// Render reserves eight run words per block; compute packs runs consecutively. From fe2e3b051b073109be6e124e278bd417c1301ae8 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 043/171] drm/asahi: g17: make room for the tiler's partial-render records The hardware writes a record per tile-map slot into gtp6 during a partial render, and 0x600 bytes is smaller than macOS's 0x1c00. Size it for the largest pool over all 64 slots. That is a superset of what macOS uses, not a layout claim, and it is the size every partial-render test ran with. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 07728dcdf2db8fc6039ea25696a21f3667670dee in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer/target.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index 42a2c04b40112d..bd96ecc9d460e7 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -29,7 +29,12 @@ fn aligned(value: usize, alignment: usize) -> Result { /// the default parameter-buffer sizes. Sizes are /// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; -const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, GTP6_SIZE, 0x80]; +/// Room for the tiler's records across a partial render. macOS uses 0x1c00, so this is a superset, +/// and the size every partial-render test ran with. +const GTP_SLOTS: usize = 64; +const GTP6_SIZE: usize = GTP_SLOTS + * (((cfg::TVB_MAX_BLOCKS * cfg::TVB_BLOCK_SIZE).div_ceil(0x8000) + 0x17ff) / 24 * 8 + 8); /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] From 46915b73a331f42272959f06862f86df4fd87487 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 044/171] drm/asahi: g17: write partial renders' depth and stencil words as macOS does Every array of a partial render wrote the main control word with the load and store bits of each plane set. A pass that clears its depth then loaded it in its first segment, instead of starting from the clear. macOS leaves the first segment's loads as the pass set them, stores each plane, and adds the loads for the reload arrays: 0x80045, then 0x88045. Keep the main word's loads in the first segment. A pass that loads its depth or stencil needs them there: an overflow of a grown pool with more than a few draws gives that segment real work, and a pass without them stores background depth over the real values. A colour pass at EQUAL after such a pass drew nothing at all in a test scene, and Counter-Strike 2's soundpack preview, with its depth pass overflowing twice a frame, had up to 2,937 of 5,400 tiles black. With the loads kept the scene matches a render that can't overflow byte for byte, and the preview had no black tile in 48 captures. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 02bc0af9cc6ffa1ba03d5c4c288ff89d73ff4f83 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 37 +++++++++++++++++++++----- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 3ab33304bc8a89..825b4f9e50d4e7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -162,8 +162,14 @@ const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; /// The partial-render phases load and store every attached depth and stencil /// surface. -const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; -const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const ZLS_STORE_DEPTH: u64 = 1 << 19; +const ZLS_STORE_STENCIL: u64 = 1 << 18; +const ZLS_LOAD_DEPTH: u64 = 1 << 15; +const ZLS_LOAD_STENCIL: u64 = 1 << 14; +const ZLS_COMPRESS_DEPTH: u64 = 1 << 6; +const ZLS_COMPRESS_STENCIL: u64 = 1 << 8; +/// Bits of the main word a partial render sets for itself. +const ZLS_PARTIAL_CLEAR: u64 = ZLS_COMPRESS_DEPTH | ZLS_COMPRESS_STENCIL | 0xf << 56; const STENCIL_CLEAR_BASE: u32 = 0x300; /// Value of the ZLS swizzle registers and descriptor field. const ZLS_SWIZZLE: u32 = 0x0404_0404; @@ -443,6 +449,7 @@ struct PassWords { layer_mode: u64, tib_blocks: u64, partial_zls_control: u64, + partial_zls_reload: u64, aux_fb_flags: u64, bg_rsrc_spec: u64, bg_va: u64, @@ -478,12 +485,27 @@ impl PassWords { tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; } - let mut partial_zls_control = pass.zls_control; + // The partial store keeps the main word, loads included, and stores and compresses each + // plane it has. The reload arrays add the loads, which a clearing pass doesn't have. + let mut partial_zls_control = pass.zls_control & !ZLS_PARTIAL_CLEAR; if pass.depth.va != 0 { - partial_zls_control |= ZLS_PARTIAL_DEPTH; + partial_zls_control |= ZLS_STORE_DEPTH; + } + if pass.depth.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_STORE_STENCIL; + } + if pass.stencil.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_STENCIL; + } + let mut partial_zls_reload = partial_zls_control; + if pass.depth.va != 0 { + partial_zls_reload |= ZLS_LOAD_DEPTH; } if pass.stencil.va != 0 { - partial_zls_control |= ZLS_PARTIAL_STENCIL; + partial_zls_reload |= ZLS_LOAD_STENCIL; } let mut aux_fb_flags = AUX_FB_FLAGS_BASE; @@ -511,6 +533,7 @@ impl PassWords { * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, + partial_zls_reload, aux_fb_flags, bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), bg_va: usc(pass.bg.offset)?, @@ -937,7 +960,7 @@ fn partial_resume_registers( RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -955,7 +978,7 @@ fn partial_load_registers( RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } From 62c237fb46b30478287fe8835ccb1384c486bda0 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 045/171] drm/asahi: g17: walk every tile in the last segment of a partial render Tiles a segment has no geometry for are skipped, and only the main end-of-tile program runs in the last segment. A multisample colour attachment that isn't stored is resolved by that program alone, since a partial store has to keep the samples for the reload. So a tile that only earlier segments drew to is never resolved, and its part of the resolve target stays as it was: 40-47% of the pixels of a test scene with storeOp DONT_CARE and a resolve. Ask for empty tiles in the last segment. It costs one load and store of each untouched tile, only on a render that already overflowed. A tile entirely outside a smaller render area, in a pass with loadOp DONT_CARE, reloads undefined samples and resolves them over the destination. LOAD and CLEAR already walk those tiles from the first segment. Honeykrisp could take the same path for that case. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit fecc27c0075b51992d3349c9b480b1d2ecab91eb in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 825b4f9e50d4e7..40d83b77f30a73 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -972,7 +972,9 @@ fn partial_load_registers( [ RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), - RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + // Only the main end-of-tile program resolves, and it has to reach tiles that only + // earlier segments drew to. Without this their resolve target is never written. + RegisterWrite::new(0x10039, w.tile_config | TILE_CONFIG_PROCESS_EMPTY_TILES), RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), From f0fd63f1818cde13506e1c285f2574ce1434d8b8 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Mon, 2 Nov 2026 14:54:08 -0300 Subject: [PATCH 046/171] drm/asahi: g17: partial-render ordinary passes on 25G83, as macOS does Every TA asked the firmware to grow the parameter buffer when it overflowed. That can't go past the maximum. In Counter-Strike 2's soundpack preview a depth pre-pass at 3456x2234 grew to the 3353-block maximum and the firmware failed it. macOS sets this word only for memoryless passes. An ordinary pass stops and flushes partial renders when the heap runs out, and the heap grows between renders. Do the same on 25G83, keeping the host's automatic growth: 9M triangles take 527 ms and 20M 1,172 ms, and images match a render that can't overflow. 26A428 keeps the grow request. This has not been run there, and its partial render path may differ. Linux has no memoryless flag, so such a pass would partial-render too; Honeykrisp always backs its attachments. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 8ec7c05672b9dca3284149bdf2e56e7766e6354c in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 40d83b77f30a73..0d18d607f2abcc 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1108,7 +1108,6 @@ unsafe impl Zeroable for TaDescriptor {} impl TaDescriptor { const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; const UNK_876: u32 = u32::MAX; - const UNK_892: u32 = 1; const UNK_932: u8 = 0x44; const UNK_93C: u8 = 1; const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; @@ -1157,7 +1156,9 @@ impl TaDescriptor { self.sampler_heap_va = pass.sampler_heap_va; self.sampler_count = pass.sampler_count; self.sampler_max = sampler_max; - self.unk_892 = Self::UNK_892; + // Asks the firmware to grow the heap. macOS keeps that for memoryless passes, which can't + // partial-render, and Honeykrisp has none. 26A428 keeps it: untested there. + self.unk_892 = u32::from(args.programs != RegisterPrograms::V25G83); self.stamp_va = args.ta.stamp_va; self.aux_stamp_va = args.ta.aux_stamp_va; self.stamp_value_2 = stamp_value; From 3707c2aa89cdaf0368b191cb950ca73c1c5519f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 3 Nov 2026 13:17:08 +1100 Subject: [PATCH 047/171] drm/asahi: g17: keep the 25G83 tiling array's doc comment on its function The partial-render series put the tiler segment constants between ta_registers_25g83()'s doc comment and the function, so rustdoc attached the function's description to VDM_SEGMENT_CONTROL and left the function undocumented. Move the constants above the comment. No code change. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 3afdeac0598c7a073c09971a854849d9cb4a7593 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0d18d607f2abcc..d0000282cef826 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -659,9 +659,6 @@ fn ta_registers( ]) } -/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage -/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the -/// M4 Pro has run with. Returns the array and its used entry count. /// Tiler segment control of a two-cluster pass, what macOS writes. const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; const VDM_SEGMENT_INDEX: u64 = 4; @@ -669,6 +666,9 @@ const VDM_SEGMENT_INDEX: u64 = 4; /// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. const MIN_FREE_TVB_PAGES: u64 = 0x3c; +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. fn ta_registers_25g83( args: &RenderArgs<'_>, g: &Geometry, From eba9f72addc98133734efcd0d4f7fba5159bd0eb Mon Sep 17 00:00:00 2001 From: "Michal S." Date: Sun, 1 Nov 2026 20:34:35 +0000 Subject: [PATCH 048/171] drm/asahi: admit the observed 16-core T6040 layout Allow a configured alternate core-mask layout while keeping the measured 20-core T6040 path unchanged. The 16-core mask and ten slots per cluster have been observed on J614s; reuse of the 20-core command-buffer offsets remains experimental and emits a warning. Generated-by: LLM Assisted-by: LLM Signed-off-by: Michal S. (cherry picked from commit 913e97d80da3cba464a067fce893fe96b6050818 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 19 +++++++++++++++++-- drivers/gpu/drm/asahi/hw/t6040.rs | 5 ++++- drivers/gpu/drm/asahi/hw/t8140.rs | 1 + 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2a0a9c0a632224..06f70507699eea 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -207,9 +207,10 @@ pub(crate) struct Config { /// Layouts of the firmware command buffers that follow a SoC's GPU core layout. pub(crate) struct CoreLayout { - /// The cluster and enabled core counts the layouts were measured on, which the GPU must - /// report at probe; `None` where the layouts hold for every configuration of the SoC. + /// The measured cluster and enabled core counts; `None` admits every configuration. pub(crate) cores: Option<(u32, u32)>, + /// An additional observed layout: core slots per cluster and enabled masks. + pub(crate) alternate_core_masks: Option<(u32, &'static [u32])>, /// A compute queue's operand-state buffer. pub(crate) operand_state: fw::compute::OperandState, /// A render's deflake buffer. @@ -1208,10 +1209,18 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { + clusters == id.num_clusters + && cores != active_cores + && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, masks)| { + id.num_cores == slots && id.core_masks.as_slice() == masks + }) + }); if cfg .core_layout .cores .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + && !alternate { dev_err!( dev.as_ref(), @@ -1221,6 +1230,12 @@ impl Gpu { ); return Err(ENODEV); } + if alternate { + dev_warn!( + dev.as_ref(), + "Experimental GPU layout: command-buffer offsets are not yet validated\n" + ); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 11ed37a7c1ef80..74d1fb7b4338e2 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -99,9 +99,12 @@ pub(crate) const CONFIG: Config = Config { ], variant: &fw::variant::V25G83, }]), - // Measured on the 20-core M4 Pro; other core counts are refused at probe. + // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { cores: Some((2, 20)), + // The observed 16-core M4 Pro reports two clusters with ten slots each. + // Reusing the measured 20-core buffers on it remains experimental. + alternate_core_masks: Some((10, &[0x3db, 0x3db])), operand_state: fw::compute::OperandState::TWO_BY_TEN, deflake: fw::render::Deflake::TWO_CLUSTERS, }, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 47dc4d583a5fcb..ea0f9cd782e81d 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -57,6 +57,7 @@ pub(crate) const CONFIG: Config = Config { // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { cores: None, + alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, }, From 92faefe93a170cd0069cf3ec156ac059cf28c462 Mon Sep 17 00:00:00 2001 From: "Michal S." Date: Wed, 4 Nov 2026 18:10:43 +0000 Subject: [PATCH 049/171] drm/asahi: admit 16-core T6040 GPUs by topology Recognize the experimental 16-core layout by its two clusters, ten slots per cluster and eight enabled cores per cluster. The actual mask bits remain device-specific and are checked against loader calibration. Keep the measured 20-core path and experimental warning unchanged in scope. The reused 20-core buffer offsets and context switching are not qualified. Generated-by: LLM Assisted-by: LLM Signed-off-by: Michal S. (cherry picked from commit d6b05d7cc0842f0eb4bd6861ff2da6d8b5005eda in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 12 +++++++----- drivers/gpu/drm/asahi/hw/t6040.rs | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 06f70507699eea..db668b7b6f973c 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -209,8 +209,8 @@ pub(crate) struct Config { pub(crate) struct CoreLayout { /// The measured cluster and enabled core counts; `None` admits every configuration. pub(crate) cores: Option<(u32, u32)>, - /// An additional observed layout: core slots per cluster and enabled masks. - pub(crate) alternate_core_masks: Option<(u32, &'static [u32])>, + /// An experimental layout: core slots and enabled cores per cluster. + pub(crate) alternate_core_masks: Option<(u32, u32)>, /// A compute queue's operand-state buffer. pub(crate) operand_state: fw::compute::OperandState, /// A render's deflake buffer. @@ -1212,8 +1212,10 @@ impl Gpu { let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { clusters == id.num_clusters && cores != active_cores - && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, masks)| { - id.num_cores == slots && id.core_masks.as_slice() == masks + && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, enabled)| { + id.num_cores == slots + && id.core_masks.len() == clusters as usize + && id.core_masks.iter().all(|mask| mask.count_ones() == enabled) }) }); if cfg @@ -1233,7 +1235,7 @@ impl Gpu { if alternate { dev_warn!( dev.as_ref(), - "Experimental GPU layout: command-buffer offsets are not yet validated\n" + "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" ); } let shared = Arc::pin_init( diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 74d1fb7b4338e2..c2e7c06f67b7fe 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -102,9 +102,9 @@ pub(crate) const CONFIG: Config = Config { // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { cores: Some((2, 20)), - // The observed 16-core M4 Pro reports two clusters with ten slots each. + // Admit two clusters of ten slots with eight enabled cores in each cluster. // Reusing the measured 20-core buffers on it remains experimental. - alternate_core_masks: Some((10, &[0x3db, 0x3db])), + alternate_core_masks: Some((10, 8)), operand_state: fw::compute::OperandState::TWO_BY_TEN, deflake: fw::render::Deflake::TWO_CLUSTERS, }, From 6b3a5a20154bb146bb8badc507e6fbda4338f820 Mon Sep 17 00:00:00 2001 From: Artyom Astafurov Date: Wed, 28 Oct 2026 17:12:01 -0400 Subject: [PATCH 050/171] drm/asahi: share T604x register-window constructors Move identical register-window constructors into a family helper. The T6040 and T6041 initialization-data checks retain their exact values. Assisted-by: LLM Signed-off-by: Artyom Astafurov (cherry picked from commit 8913bab42e7dc63eae2584112c720202d06ef158 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/hw/mod.rs | 1 + drivers/gpu/drm/asahi/hw/t6040.rs | 33 +------------------------------ drivers/gpu/drm/asahi/hw/t604x.rs | 33 +++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 drivers/gpu/drm/asahi/hw/t604x.rs diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 2650131a4a2936..5149f87a4ea1d8 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -14,6 +14,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t6040; +mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index c2e7c06f67b7fe..35771d38d710ce 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -11,6 +11,7 @@ use crate::g17::fw::{ }; use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; +use super::t604x::{instances, window}; use crate::mmu::UatQuirks; /// Static configuration of the T6040 GPU. @@ -113,38 +114,6 @@ pub(crate) const CONFIG: Config = Config { /// Chip identifier written to the hardware data. pub(crate) const CHIP_ID: u32 = 0x6040; -/// Base of the firmware MMIO aperture in the kernel window. -const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; - -const fn window( - slot: usize, - phys: u64, - offset: u64, - size: u32, - unk_18: u64, - unk_20: u32, -) -> RegisterWindow { - RegisterWindow { - slot, - phys, - va: MMIO_APERTURE + offset, - size, - unk_18, - unk_20, - instances: 1, - phys_stride: 0, - } -} - -/// `window` repeated `instances` times, `phys_stride` apart in physical memory. -const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { - RegisterWindow { - instances, - phys_stride, - ..window - } -} - /// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory /// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ diff --git a/drivers/gpu/drm/asahi/hw/t604x.rs b/drivers/gpu/drm/asahi/hw/t604x.rs new file mode 100644 index 00000000000000..961cef76cf4aa8 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t604x.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use crate::g17::fw::initdata::RegisterWindow; + +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +pub(super) const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +pub(super) const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} From d0dd1d3173d6c70d4a47c5f64396004a3dfbe504 Mon Sep 17 00:00:00 2001 From: Artyom Astafurov Date: Wed, 28 Oct 2026 17:12:09 -0400 Subject: [PATCH 051/171] drm/asahi: add T6041 and the measured 25F84 firmware profile Select a static 25F84 profile and admit the normalized loaded image measured in three qualification boots. Remove exploratory overrides and unknown-image admission from the support path. Assisted-by: LLM Signed-off-by: Artyom Astafurov [Eduard Moldovan: integration/aurora's 7.2.11 OF table is kept, with the apple,agx-t6041 entry added] (cherry picked from commit e86ac69ffe1a70e6e8fe984450e8e6ac0f46e6fd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 4 + drivers/gpu/drm/asahi/g17/fw/compute.rs | 119 ++++- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 7 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 9 + .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 51 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 61 ++- drivers/gpu/drm/asahi/g17/fw/render.rs | 4 + drivers/gpu/drm/asahi/g17/fw/variant.rs | 51 +- drivers/gpu/drm/asahi/g17/initdata.rs | 2 +- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 85 ++- drivers/gpu/drm/asahi/g17/mod.rs | 43 +- drivers/gpu/drm/asahi/g17/port.rs | 31 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 19 +- drivers/gpu/drm/asahi/g17/runtime.rs | 17 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 12 +- drivers/gpu/drm/asahi/hw/mod.rs | 1 + drivers/gpu/drm/asahi/hw/t6040.rs | 2 + drivers/gpu/drm/asahi/hw/t6041.rs | 491 ++++++++++++++++++ drivers/gpu/drm/asahi/hw/t8140.rs | 2 + drivers/gpu/drm/asahi/regs.rs | 56 ++ 21 files changed, 987 insertions(+), 82 deletions(-) create mode 100644 drivers/gpu/drm/asahi/hw/t6041.rs diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index e2ecd5193bd625..b11ca814203a99 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -169,6 +169,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t6040"), ProbeConfig::G17(&hw::t6040::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6041"), + ProbeConfig::G17(&hw::t6041::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index b1afdad816b6b7..91b3a2f58334c3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -26,6 +26,16 @@ pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; const COMPUTE_REGISTERS: usize = 41; const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ComputeProgram { + G17P, + V25F84, +} + +const V25F84_REGISTERS: usize = 36; +const V25F84_CHECKPOINTS: [u16; 4] = [34, 35, 35, V25F84_REGISTERS as u16]; +const V25F84_UNK_0A599_FIRST: u64 = 0x0000_0132_0040_0020; + /// Validated per-command shader-context scratch request. /// /// The layout word is Apple's packed SCS layout: depth (bits 0:15), levels @@ -133,8 +143,11 @@ impl ScratchRequest { self.tail } - fn checkpoints(self) -> [u16; 4] { - CHECKPOINTS.map(|count| count + u16::from(self.enabled())) + fn checkpoints(self, program: ComputeProgram) -> [u16; 4] { + match program { + ComputeProgram::G17P => CHECKPOINTS.map(|count| count + u16::from(self.enabled())), + ComputeProgram::V25F84 => V25F84_CHECKPOINTS, + } } } const ALT_REGISTERS: usize = 4; @@ -187,6 +200,15 @@ impl OperandState { r0a5c9: 0x1e000, size: 0x44000, }; + pub(crate) const FOUR_BY_TEN: OperandState = OperandState { + r10229: 0x78800, + r140a8: 0x7a800, + r10099: 0x3c405, + r10091: 0x78400, + r0a5c1: 0x0005, + r0a5c9: 0x3c000, + size: 0x80000, + }; } /// Checkpoints into the main register array, as entry counts: after the @@ -279,6 +301,7 @@ impl IntersectionTables { /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + pub(crate) program: ComputeProgram, pub(crate) intersection: IntersectionTables, /// Scratch request validated with the userspace command. pub(crate) scratch: ScratchRequest, @@ -392,6 +415,61 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS ]) } +fn compute_registers_25f84(args: &ComputeArgs) -> Result<[RegisterWrite; V25F84_REGISTERS]> { + let preempt = args.preempt_va; + let operand = args.operand_state_va; + let state = args.operand; + let key = work_key(args.context_id, args.work_state); + let ids = OBJECT_IDS.word(); + let intersection = args.intersection.words(); + + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, PREEMPT_STATE[1])?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, PREEMPT_STATE[2])?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, PREEMPT_STATE[3])?), + RegisterWrite::new(0x1a440, UNK_1A440), + RegisterWrite::new(0x1a458, UNK_1A458), + RegisterWrite::new(0x101d9, UNK_101D9), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), + RegisterWrite::new(0x101d1, args.scratch.size_word()), + RegisterWrite::new( + 0x0d479, + if args.intersection.auxiliary == 0 { + args.scratch.tail_word() + } else { + u64::from(args.intersection.auxiliary) + }, + ), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x107a1, args.scratch.window_word()), + RegisterWrite::new(0x0a599, V25F84_UNK_0A599_FIRST), + RegisterWrite::new(0x0d411, UNK_0D411), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x1a440, UNK_1A440_FINAL), + RegisterWrite::new(0x0a599, UNK_0A599), + ]) +} + /// The second array exchanges the operand-state regions of registers /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { @@ -506,6 +584,8 @@ impl ComputeDescriptor { const UNK_FB0: u16 = 0x1a; const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; + const UNK_FBA_25F84: u32 = 0x6c00_0001; + const UNK_FC5_25F84: u8 = 0x5f; /// Fills a descriptor cleared by `Object::initialize`. The unused firmware /// fields and padding retain that zero initialization. @@ -516,7 +596,6 @@ impl ComputeDescriptor { { return Err(EINVAL); } - let registers = compute_registers(args)?; let alt_registers = alt_registers(args)?; let registers_va = offset_va(args.descriptor_va, Self::REGISTERS_OFFSET)?; let alt_registers_va = offset_va(args.descriptor_va, Self::ALT_REGISTERS_OFFSET)?; @@ -526,11 +605,18 @@ impl ComputeDescriptor { self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; - self.checkpoints = args.scratch.checkpoints(); - // The enabled program prepends one register; disabled descriptors keep - // the same forty writes, trailer count and resume checkpoints. - let first = usize::from(!args.scratch.enabled()); - self.registers.set(registers_va, ®isters[first..])?; + self.checkpoints = args.scratch.checkpoints(args.program); + match args.program { + ComputeProgram::G17P => { + let registers = compute_registers(args)?; + let first = usize::from(!args.scratch.enabled()); + self.registers.set(registers_va, ®isters[first..])?; + } + ComputeProgram::V25F84 => { + self.registers + .set(registers_va, &compute_registers_25f84(args)?)?; + } + } self.alt_registers.set(alt_registers_va, &alt_registers)?; self.preempt_va = args.preempt_va; self.cdm_end_va = args.cdm_end_va; @@ -552,8 +638,18 @@ impl ComputeDescriptor { self.timestamp_end_va = timestamp_end_va; self.unk_fb0 = Self::UNK_FB0; self.free_list_control_va = args.free_list_control_va; - self.unk_fba = Self::UNK_FBA; - self.unk_fc5 = Self::UNK_FC5; + match args.program { + ComputeProgram::G17P => { + self.unk_fba = Self::UNK_FBA; + self.unk_fc5 = Self::UNK_FC5; + } + ComputeProgram::V25F84 => { + self.unk_fba = Self::UNK_FBA_25F84; + self.unk_fbe[0] = 0x06; + self.unk_fc5 = Self::UNK_FC5_25F84; + self.unk_fc6[0] = 0x04; + } + } self.usage_va = args.usage_fw_va; self.context_generation = args.context_generation; Ok(()) @@ -564,10 +660,11 @@ impl ComputeDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scratch: ScratchRequest, + program: ComputeProgram, ) -> Result<[Option; 4]> { let registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, - scratch.checkpoints()[0] as u8, + scratch.checkpoints(program)[0] as u8, )?; let alt_registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs index 0f0614f6155a17..b056fe5aff2301 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -3,7 +3,7 @@ use kernel::prelude::*; use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; -use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; +use super::freelist::PAGE_LIST_SIZE; pub(crate) fn extra_bytes(active_cores: u32, scs_pages: u16) -> Result { if active_cores == 0 { return Err(EINVAL); } @@ -18,9 +18,10 @@ pub(crate) fn extra_bytes(active_cores: u32, scs_pages: u16) -> Result { Ok(extra) } -pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { +pub(crate) fn required_blocks(base: usize, active_cores: u32, scs_pages: u16) -> Result { let extra = extra_bytes(active_cores, scs_pages)?; - let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + let bytes = extra + .checked_add((base * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) .map_err(|_| EOVERFLOW)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 3092e599caa846..387cebb28a3680 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -781,6 +781,15 @@ impl PerfGroup { } } } + + pub(crate) fn replicate_first_lane(&mut self, lanes: usize) { + for index in 0..PERF_STATES { + for lane in lanes.max(1)..PERF_LANES { + self.core_mv[index][lane] = self.core_mv[index][0]; + self.sram_mv[index][lane] = self.sram_mv[index][0]; + } + } + } } /// Returns the performance states selected by `map` from `opps`. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index f4c3ee5db22e29..863a90fc52a346 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -107,6 +107,7 @@ pub(crate) struct HwDataArgs25G83<'a> { pub(crate) cs_opps: &'a [Opp], /// Operating points of the AFR clock domain. pub(crate) afr_opps: &'a [Opp], + pub(crate) chip: Option<&'a Chip>, } /// The hardware data of the 25G83 build, at the base of the bundle. @@ -300,8 +301,12 @@ impl HwData25G83 { } self.register_map[window.slot] = RegisterMapEntry::window(window); } - if !args.reserved_slots.is_empty() { - return Err(EINVAL); + for &slot in args.reserved_slots { + match used.get_mut(slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[slot] = RegisterMapEntry::reserved(); } self.chip_id = args.chip_id; @@ -336,6 +341,9 @@ impl HwData25G83 { } self.perf_a.fill(&states_a, 1); self.perf_b.fill(&states_b, 1); + if ext.chip.is_some_and(|chip| chip.perf_a_replicate_lanes) { + self.perf_a.replicate_first_lane(states_a[0].cells); + } for (index, state) in states_b.iter().enumerate() { self.perf_b_freq_mhz[index] = state.freq_mhz; self.perf_a_states[index] = perf.states_a[index].into(); @@ -344,7 +352,9 @@ impl HwData25G83 { self.perf_a_relative = perf.relative_a; self.perf_b_relative = perf.relative_b; self.unk_1848 = [0x3f82_8f5c; 11]; - self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + if ext.chip.is_none_or(|chip| chip.cs_perf_table) { + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + } self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; self.unk_2544 = 5; @@ -375,15 +385,46 @@ impl HwData25G83 { self.unk_26c0 = 1; self.unk_26c4 = 1; self.unk_26f4 = 1; + // SAFETY: `HwData25G83` consists of integers only and has no padding, so any bytes are a + // valid value. + let bytes = unsafe { + core::slice::from_raw_parts_mut((self as *mut Self).cast::(), size_of::()) + }; + if let Some(chip) = ext.chip { + overlay_words(bytes, chip.hw_data_words)?; + } Ok(()) } } +fn overlay_words(bytes: &mut [u8], words: Words) -> Result { + for &(offset, value, repeat) in words { + for index in 0..repeat { + let at = offset + 4 * index; + bytes + .get_mut(at..at + 4) + .ok_or(EINVAL)? + .copy_from_slice(&value.to_le_bytes()); + } + } + Ok(()) +} + /// Power configuration control period, in 24 MHz clocks (8 ms). const PERIOD_CLOCKS: u32 = 200_000; /// Temperature sensors read by the die and MTR temperature limiters. const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; +pub(crate) struct Chip { + pub(crate) max_power_mw: u32, + pub(crate) leakage: [u32; 56], + pub(crate) aon_table: &'static [u32], + pub(crate) hw_data_words: Words, + pub(crate) power_config_words: Words, + pub(crate) perf_a_replicate_lanes: bool, + pub(crate) cs_perf_table: bool, +} + /// Offset of the per-chip leakage coefficients in the power configuration. const LEAKAGE_OFFSET: usize = 0x642c; /// Offset in the power configuration of the GPU address of the configuration's status (the @@ -597,6 +638,10 @@ impl PowerConfig25G83 { self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); } + + pub(crate) fn apply_chip(&mut self, chip: &Chip) -> Result { + overlay_words(&mut self.0, chip.power_config_words) + } } /// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index ca2b6604a302b5..2a035524bdffbd 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -507,10 +507,46 @@ impl KickEntry { } } +pub(crate) fn ksm_slot_descriptor( + ring_va: u64, + data_master: super::queue::DataMaster, + priority: u8, +) -> Result { + if ring_va & !RenderRegistration::ADDRESS_MASK != 0 || priority >= Policy::PRIORITIES { + return Err(EINVAL); + } + let term = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); + Ok(((data_master as u64 + 1) << 60) | ring_va | term) +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct KsmQueue { + pub(crate) qid: u8, + pub(crate) descriptor: u64, +} + +impl KsmQueue { + pub(crate) fn new( + qid: u8, + ring_va: u64, + data_master: super::queue::DataMaster, + priority: u8, + ) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + qid, + descriptor: ksm_slot_descriptor(ring_va, data_master, priority)?, + }) + } +} + /// The complete write-only installation of a fragment/tiling queue pair. /// Firmware must see both queues configured before either outer command. pub(crate) struct RenderRegistration { writes: [(usize, u64); 8], + queues: [KsmQueue; 2], } impl RenderRegistration { @@ -540,8 +576,7 @@ impl RenderRegistration { if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { return Err(EINVAL); } - let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); - let payload = ((engine as u64 + 1) << 60) | va | completion; + let payload = ksm_slot_descriptor(va, engine, priority)?; let (low, high) = if qid < 64 { (1u64 << qid, 0) } else { @@ -554,10 +589,30 @@ impl RenderRegistration { (Self::VALID_LOW, low), ]); } - Ok(Self { writes }) + Ok(Self { + writes, + queues: [ + KsmQueue::new( + fragment.0, + fragment.1, + super::queue::DataMaster::Fragment, + priority, + )?, + KsmQueue::new( + tiling.0, + tiling.1, + super::queue::DataMaster::Tiling, + priority, + )?, + ], + }) } pub(crate) fn writes(&self) -> &[(usize, u64); 8] { &self.writes } + + pub(crate) fn queues(&self) -> &[KsmQueue; 2] { + &self.queues + } } diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index d0000282cef826..dd66da538f0fb2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -70,6 +70,10 @@ impl Deflake { sub: [0xa0, 0x820], viewport: false, }; + pub(crate) const FOUR_CLUSTERS: Deflake = Deflake { + sub: [0x120, 0xda0], + viewport: false, + }; } /// The register programs of a firmware build's render descriptors. diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 03d5bfccafeca2..3e89d510fc83cb 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -36,7 +36,22 @@ pub(crate) enum InitGraph { V25G83, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum QueueRegistration { + Firmware, + PoweredRenderPairs, + Mailbox, +} + +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum FirmwareVersion { + V26A428, + V25G83, + V25F84, +} + /// The layouts and rules of one G17 firmware build. +#[derive(Copy, Clone)] pub(crate) struct Variant { /// The initialization graph. pub(crate) graph: InitGraph, @@ -49,8 +64,7 @@ pub(crate) struct Variant { pub(crate) qid_floor: usize, /// Host words of a queue's pointer block. pub(crate) pointers: PointerLayout, - /// The host registers each render pair through the GPU's MMIO queue ports. - pub(crate) mmio_pair_registration: bool, + pub(crate) registration: QueueRegistration, /// Render kick entries may name an MCache table. pub(crate) mcache: bool, /// Renders may set the few-primitives auxiliary framebuffer flag. @@ -65,6 +79,10 @@ pub(crate) struct Variant { pub(crate) lead_commands: bool, /// Register programs of the render descriptors. pub(crate) programs: super::render::RegisterPrograms, + pub(crate) compute: super::compute::ComputeProgram, + pub(crate) opening_record: bool, + pub(crate) split_firmware_area: bool, + pub(crate) version: FirmwareVersion, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -98,13 +116,17 @@ pub(crate) const V26A428: Variant = Variant { producer: 0x40, recovery_target: Some(0x50), }, - mmio_pair_registration: true, + registration: QueueRegistration::PoweredRenderPairs, mcache: true, few_primitives: true, ray_traversal: true, fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, + compute: super::compute::ComputeProgram::G17P, + opening_record: true, + split_firmware_area: false, + version: FirmwareVersion::V26A428, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -122,7 +144,7 @@ pub(crate) const V25G83: Variant = Variant { }, // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run // without it. - mmio_pair_registration: false, + registration: QueueRegistration::Firmware, mcache: false, // A fragment stage with the few-primitives flag hangs. few_primitives: false, @@ -132,9 +154,30 @@ pub(crate) const V25G83: Variant = Variant { fragment_busy: false, lead_commands: true, programs: super::render::RegisterPrograms::V25G83, + compute: super::compute::ComputeProgram::G17P, + opening_record: true, + split_firmware_area: false, + version: FirmwareVersion::V25G83, }; +pub(crate) const V25F84: Variant = Variant { + version: FirmwareVersion::V25F84, + qid_floor: 0, + registration: QueueRegistration::Mailbox, + compute: super::compute::ComputeProgram::V25F84, + opening_record: false, + split_firmware_area: true, + ..V25G83 +}; + +impl Variant { + pub(crate) const fn is_25f84(&self) -> bool { + matches!(self.version, FirmwareVersion::V25F84) + } +} + const _: () = { assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25F84.pointers)); }; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 578bb7533f3c92..56e7335a23bd24 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -862,7 +862,7 @@ impl InitData { clusters, }; if variant.graph == super::fw::variant::InitGraph::V25G83 { - return self.build_25g83(&env, cfg, metrics, active_cores); + return self.build_25g83(&env, cfg, metrics, active_cores, variant); } self.map_register_windows(&env, cfg.windows)?; let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index 670f4b96dcb76a..f7b1952685761a 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -18,6 +18,7 @@ use kernel::{c_str, prelude::*}; struct Objects { views: [usize; 5], firmware_area: usize, + firmware_area_2: Option, power_state: usize, aon: usize, } @@ -30,6 +31,7 @@ impl InitData { cfg: &Config, metrics: &KernelObject, active_cores: u32, + variant: &crate::g17::fw::Variant, ) -> Result { self.status_a = v::private::STATUS_A; self.status_b = v::private::STATUS_B; @@ -39,9 +41,9 @@ impl InitData { let first_block = self.build_render_pool(env)?; self.build_compute_pools(env, first_block)?; let bundle = self.build_bundle(env, cfg, metrics)?; - let objects = self.build_25g83_objects(env)?; + let objects = self.build_25g83_objects(env, variant.split_firmware_area)?; self.views = Some(objects.views); - self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects, variant)?; if cfg.uat_quirks.clean_tables { // Write back every host-built byte before the firmware may read it. for object in self.objects.iter_mut() { @@ -53,8 +55,7 @@ impl InitData { Ok(()) } - /// The views, the firmware area, the power-state object and the AON table. - fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>, split: bool) -> Result { use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; use CpuMap::WriteCombined as WC; let mut views = [0; 5]; @@ -63,12 +64,17 @@ impl InitData { } let firmware_area = self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let firmware_area_2 = match split { + true => Some(self.add(env.dynamic(mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, FP, WC)?)?), + false => None, + }; let power_state = self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; Ok(Objects { views, firmware_area, + firmware_area_2, power_state, aon, }) @@ -84,15 +90,25 @@ impl InitData { tables: &DescriptorTables, bundle: &BundleObjects, objects: &Objects, + variant: &crate::g17::fw::Variant, ) -> Result { let bundle_va = self.object(self.bundle)?.gpu_va(); let cluster_va = self.object(self.cluster)?.gpu_va(); let config_va = cluster_va + v::private::CONFIG as u64; let status_a_va = cluster_va + v::private::STATUS_A as u64; let status_b_va = cluster_va + v::private::STATUS_B as u64; - let aon_va = self.object(objects.aon)?.gpu_va(); + let chip = cfg.chip; + let aon_table = chip.map_or(v::AON_TABLE.as_slice(), |chip| chip.aon_table); + let aon_va = match aon_table.is_empty() { + true => 0, + false => self.object(objects.aon)?.gpu_va(), + }; let power_state_va = self.object(objects.power_state)?.gpu_va(); let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let firmware_area_2_va = match objects.firmware_area_2 { + Some(index) => self.object(index)?.gpu_va(), + None => firmware_area_va, + }; let mut view_va = [0; 5]; for (va, index) in view_va.iter_mut().zip(objects.views) { *va = self.object(index)?.gpu_va(); @@ -101,14 +117,28 @@ impl InitData { let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; - let calibration = Calibration::read(env.dev)?; - calibration.check_opps(&opps)?; + let calibration = if chip.is_none() { + let calibration = Calibration::read(env.dev)?; + calibration.check_opps(&opps)?; + Some(calibration) + } else { + None + }; + let (perf, max_power_mw, leakage) = match (calibration.as_ref(), chip) { + (Some(calibration), None) => ( + &calibration.perf, + calibration.max_power_mw, + &calibration.leakage, + ), + (None, Some(chip)) => (cfg.perf, chip.max_power_mw, &chip.leakage), + _ => return Err(EINVAL), + }; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &calibration.perf, + perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -116,6 +146,7 @@ impl InitData { active_cores, cs_opps: &cs_opps, afr_opps: &afr_opps, + chip, }; // SAFETY: Plain C entropy source. let entropy = unsafe { kernel::bindings::get_random_u32() }; @@ -126,22 +157,19 @@ impl InitData { })?; self.object_mut(self.bundle)? .initialize::(v::POWER_CONFIG, |power| { - power.init( - config_va, - aon_va, - entropy, - calibration.max_power_mw, - &calibration.leakage, - ); + power.init(config_va, aon_va, entropy, max_power_mw, leakage); + if let Some(chip) = chip { + power.apply_chip(chip)?; + } Ok(()) })?; self.object_mut(self.region_c)? .initialize::(0, |region| { region.init(cfg.region_c); - region.max_power_mw = calibration.max_power_mw; + region.max_power_mw = max_power_mw; Ok(()) })?; - for (index, word) in v::AON_TABLE.iter().enumerate() { + for (index, word) in aon_table.iter().enumerate() { self.object_mut(objects.aon)?.write(index * 4, *word)?; } @@ -180,7 +208,7 @@ impl InitData { .initialize::(main_offset, |main| { main.init_primary(&args, &views); main.view_va = view_va; - main.firmware_area_va = [firmware_area_va; 2]; + main.firmware_area_va = [firmware_area_va, firmware_area_2_va]; main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); // The secondary role's words, on the single instance. main.unk_300 = 4; @@ -189,10 +217,12 @@ impl InitData { main.unk_344 = 0; Ok(()) })?; - self.object_mut(self.bundle)?.write( - main_offset + abi::MainConfig::CONTROL_RING, - abi::OpeningRecord::new(abi::Role::Primary), - )?; + if variant.opening_record { + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + } // The merged root. let root = abi::RootArgs { @@ -214,12 +244,11 @@ impl InitData { } } - // The opening control record is staged in slot 0. - for (offset, value) in [ - (0, 0), - (0x10, 0), - (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), - ] { + let producer = match variant.opening_record { + true => abi::OPENING_PRODUCER, + false => 0, + }; + for (offset, value) in [(0, 0), (0x10, 0), (abi::ChannelState::PRODUCER, producer)] { self.object(self.cluster)? .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index db668b7b6f973c..d615cf3fa59085 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -193,6 +193,7 @@ pub(crate) struct Config { pub(crate) fabric_bridge: bool, /// The SoC's values of region C. pub(crate) region_c: &'static fw::initdata::RegionCConfig, + pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -215,6 +216,7 @@ pub(crate) struct CoreLayout { pub(crate) operand_state: fw::compute::OperandState, /// A render's deflake buffer. pub(crate) deflake: fw::render::Deflake, + pub(crate) compute_blocks: usize, } /// Where a G17 target's crash log buffers live. @@ -867,6 +869,7 @@ struct Shared { operand_state: &'static fw::compute::OperandState, /// Layout of renders' deflake buffers for this GPU's cores. deflake: fw::render::Deflake, + compute_blocks: usize, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -1125,8 +1128,9 @@ impl Shared { self.crashed.store(true, Ordering::Release); dev_err!( self.dev.as_ref(), - "GPU firmware ({:?}) crashed, device lost\n", - role + "GPU firmware ({:?}) crashed, device lost (crash log {} bytes)\n", + role, + crashlog.map_or(0, |log| log.len()) ); // The GPU dev coredump support this used is gone; the crash log is not published. @@ -1247,6 +1251,7 @@ impl Gpu { variant, operand_state: &cfg.core_layout.operand_state, deflake: cfg.core_layout.deflake, + compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -1510,18 +1515,24 @@ impl Gpu { session.port.attach( session.init.port_views(mailbox)?, session.bridge.regs.clone(), + shared.variant.registration, )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session - .primary - .init(session.init.root_va(Role::Primary)?, ack)?; + let root_va = session.init.root_va(Role::Primary)?; + dev_dbg!( + dev.as_ref(), + "bring-up: initialization data built, publishing root {:#x}\n", + root_va + ); + session.primary.init(root_va, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } + dev_dbg!(dev.as_ref(), "bring-up: initialization data accepted\n"); if let Some(secondary) = session.secondary.as_mut() { secondary.init(session.init.root_va(Role::Secondary)?, ack)?; } @@ -1531,15 +1542,25 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening( - dev, - &session.init, - &session.primary, - session.secondary.as_ref(), - )?; + if shared.variant.opening_record { + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; + dev_dbg!(dev.as_ref(), "bring-up: opening record retired\n"); + } else { + dev_dbg!( + dev.as_ref(), + "bring-up: no opening record staged; device-control counters {:?}\n", + session.init.control_counters()? + ); + } session .uat .activate_render_context(&session._render_global)?; + dev_dbg!(dev.as_ref(), "bring-up: firmware running\n"); Ok(firmware) } diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 6f9d1c7d956c05..5e759bb2b7d18a 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -91,8 +91,8 @@ unsafe impl Sync for Views {} #[pin_data] pub(super) struct Port { views: SetOnce, - /// Register access for tracing the core power state at the doorbell. regs: SetOnce, + registration: SetOnce, /// Compute work-ring slots reserved by publishers that have not exposed them yet. #[pin] rings: SpinLock<[u32; PRIORITIES]>, @@ -163,6 +163,7 @@ impl Port { pin_init!(Self { views: SetOnce::new(), regs: SetOnce::new(), + registration: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), rx: SetOnce::new(), @@ -182,17 +183,37 @@ impl Port { } /// Called once while the session is built, before any publisher can see the port. - pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { - // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port, and - // with it every use of the handle. + pub(super) fn attach( + &self, + views: Views, + regs: crate::regs::G17Resources, + registration: super::fw::variant::QueueRegistration, + ) -> Result { + // SAFETY: The session owns the mailbox and outlives every port user. let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; - if self.views.populate(views) && self.regs.populate(regs) && self.rx.populate(rx) { + if self.views.populate(views) + && self.regs.populate(regs) + && self.registration.populate(registration) + && self.rx.populate(rx) + { Ok(()) } else { Err(EBUSY) } } + pub(super) fn register_queues(&self, queues: &[super::fw::kick::KsmQueue]) -> Result { + use super::fw::variant::QueueRegistration; + if self.registration.as_ref() != Some(&QueueRegistration::Mailbox) { + return Ok(()); + } + let regs = self.regs.as_ref().ok_or(ENODEV)?; + for queue in queues { + regs.register_ksm_queue(queue.qid, queue.descriptor)?; + } + Ok(()) + } + fn views(&self) -> Result<&Views> { self.views.as_ref().ok_or(ENODEV) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 55994066825f81..9501ec86785d22 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -11,7 +11,7 @@ use crate::g17::{ context::Context, fw::{ channels::ControlRecord, - compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + compute::{ComputeProgram, OperandState, COMPUTE_DESCRIPTOR_SIZE}, queue::*, variant::PointerLayout, }, @@ -145,6 +145,7 @@ struct Graph { pointers: PointerLayout, /// Layout of the operand-state buffer for the GPU's cores. operand_state: &'static OperandState, + program: ComputeProgram, } impl Graph { @@ -154,6 +155,7 @@ impl Graph { context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, + program: ComputeProgram, ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; @@ -220,6 +222,7 @@ impl Graph { reclaimed: 0, pointers, operand_state, + program, }) } @@ -614,9 +617,10 @@ impl Queue { context: Arc, pointers: PointerLayout, operand_state: &'static OperandState, + program: ComputeProgram, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -2149,6 +2153,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; + fn register_queue(&mut self, queue: &crate::g17::fw::kick::KsmQueue) -> Result; fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; @@ -2237,6 +2242,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + program: self.graph.program, intersection: *intersection, scratch: *scratch, usc_base: *usc_base, @@ -2296,7 +2302,7 @@ impl Queue { }, mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)?, scs_enabled: scratch.enabled(), priority: policy.priority(), }; @@ -2364,9 +2370,16 @@ impl Queue { )?; wait_vas.push(address, GFP_KERNEL)?; } + let kick_ring_va = self.kick.low_va(); let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { + host.register_queue(&crate::g17::fw::kick::KsmQueue::new( + qid, + kick_ring_va, + DataMaster::Compute, + policy.priority(), + )?)?; producer = self.graph.first_descriptor(descriptor_high)?; host.prepare_compute_shared()?; packet.mark_published(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d391fbb3001bbe..af76e8b62f0403 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1160,6 +1160,7 @@ impl Backend { self.context.clone(), self.shared.variant.pointers, self.shared.operand_state, + self.shared.variant.compute, )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); @@ -1399,6 +1400,9 @@ impl compute::Host for ComputeHost<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } + fn register_queue(&mut self, queue: &super::fw::kick::KsmQueue) -> Result { + self.port.register_queues(core::slice::from_ref(queue)) + } fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } @@ -1443,6 +1447,9 @@ impl compute::Host for DirectHost<'_> { // An installed queue published its identity with its first command. Ok(()) } + fn register_queue(&mut self, _queue: &super::fw::kick::KsmQueue) -> Result { + Err(EAGAIN) + } fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } @@ -1734,7 +1741,7 @@ impl super::Firmware { impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; + let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)?; let mut deferred = DeferredBatch::publication()?; loop { admission.check()?; @@ -1833,7 +1840,7 @@ impl Backend { return None; } let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()).ok()?; loop { let channels = self.shared.channels.observe(); let progress = self.shared.progress.observe(); @@ -1976,7 +1983,7 @@ impl Backend { let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Some(Err(EINVAL)); }; - let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + let required = match super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) { Ok(required) => required, Err(error) => return Some(Err(error)), }; @@ -1991,7 +1998,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { return Err(render::FALLBACK); } @@ -2068,7 +2075,7 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; if let Some(result) = self.compute_direct(packet, required, deferred) { return result; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index bf0f989f042a50..10085c3fcb922d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -177,7 +177,7 @@ impl job::Backend for Backend { // A reverse timeline chain must be submitted before the host signal. // Validate here, but do not consume physical capacity behind unsignalled // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages) .map(|_| ()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 6657f83e9e738a..fc2e01be761aeb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -572,10 +572,14 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { - // Firmware builds without MMIO registration install the pair from its queue - // configuration alone. - if !self.firmware.variant.mmio_pair_registration { - return Ok(()); + use crate::g17::fw::variant::QueueRegistration; + match self.firmware.variant.registration { + QueueRegistration::Firmware => return Ok(()), + QueueRegistration::Mailbox => { + self.firmware.port.register_queues(registration.queues())?; + return Ok(()); + } + QueueRegistration::PoweredRenderPairs => {} } if !self.registration_powered { self.firmware.acquire_registration_power()?; diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 5149f87a4ea1d8..c54ffb5c208fe8 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -14,6 +14,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t6040; +pub(crate) mod t6041; mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 35771d38d710ce..d531e20c7e9f94 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -52,6 +52,7 @@ pub(crate) const CONFIG: Config = Config { native_gpu_permissions: true, }, region_c: ®ION_C, + chip: None, adopt_root_entries: false, fabric_bridge: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on @@ -108,6 +109,7 @@ pub(crate) const CONFIG: Config = Config { alternate_core_masks: Some((10, 8)), operand_state: fw::compute::OperandState::TWO_BY_TEN, deflake: fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: fw::freelist::COMPUTE_BLOCKS, }, }; diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs new file mode 100644 index 00000000000000..956778038d1d48 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use super::t604x::{instances, window}; +use crate::mmu::UatQuirks; + +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + variant: 4, + num_dies: 1, + gpu_variant: GpuVariant::C, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Unverified placement; use the M4 Pro layout. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + chip: Some(&CHIP), + adopt_root_entries: false, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G16, + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 5, 2], + name: "macOS 26.5.2 (25F84, iBoot mBoot-18000.121.3)", + images: &[crate::fwid::FirmwareBuild { + name: "Apple GFX 25F84 for T6041 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x5983a, 0x5991d], + text_sha256: [ + 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, + 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, + 0xa3, 0xc1, 0x83, 0xe8, + ], + }], + variant: &fw::variant::V25F84, + }]), + core_layout: CoreLayout { + cores: Some((4, 40)), + alternate_core_masks: None, + operand_state: fw::compute::OperandState::FOUR_BY_TEN, + deflake: fw::render::Deflake::FOUR_CLUSTERS, + compute_blocks: 160, + }, +}; + +pub(crate) const CHIP_ID: u32 = 0x6041; + +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x0, 0x4000, 0, 2), + instances( + window(3, 0x2_201c_4000, 0x8000, 0x1_8000, 0, 2), + 8, + 0x200_0000, + ), + window(4, 0x5_0244_c000, 0xd_0000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0xd_8000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0xe_0000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0xe_8000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0xf_0000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0xf_8000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x11_0000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x11_8000, 0x2_1400, 0, 2), + instances( + window(20, 0x2_8b00_0000, 0x14_0000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), + window(22, 0x2_8900_0000, 0x24_8000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x25_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x26_8000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x27_8000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x28_0000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x28_8000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x2b_0000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x2b_8000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x2c_0000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x2c_8000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x2d_0000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x2e_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x2e_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x2f_8000, 0x58, 0xe6_0000, 2), +]; + +const RESERVED_SLOTS: [usize; 14] = [2, 5, 6, 7, 8, 27, 30, 38, 42, 46, 47, 48, 49, 52]; + +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 10, 18, 25, 31, 40, 50, 62, 70, 78, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +const MAX_POWER_MW: u32 = 129_702; + +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; + +const CHIP: v25g83::Chip = v25g83::Chip { + max_power_mw: MAX_POWER_MW, + leakage: { + let mut words = [0; 56]; + words[0] = 0x44f9_c000; + words[1] = 0x44f7_2000; + words[2] = 0x44f4_b000; + words[3] = 0x44f1_5000; + words[32] = 0x43e4_0000; + words[33] = 0x43e4_0000; + words[34] = 0x43e4_0000; + words[35] = 0x43e4_0000; + words[52] = 0x4404_0000; + words[54] = 0x43df_8000; + words + }, + aon_table: &[], + hw_data_words: HW_DATA_WORDS, + power_config_words: POWER_CONFIG_WORDS, + perf_a_replicate_lanes: true, + cs_perf_table: false, +}; + +const HW_DATA_WORDS: &[(usize, u32, usize)] = &[ + (0xe0, 0x2008, 1), + (0xe8, 0x20080000, 1), + (0xf4, 0x2008, 1), + (0xf8, 0x24cb, 1), + (0xfc, 0xc98a2cfa, 1), + (0x100, 0xf4f624cb, 1), + (0x104, 0x1877e917, 1), + (0x108, 0x38d924cb, 1), + (0x10c, 0xbdab0000, 1), + (0x2d8, 0xfff82000, 1), + (0x2dc, 0xd32d2cdb, 1), + (0x2e0, 0xf5002000, 1), + (0x2e4, 0x21dae926, 1), + (0x2e8, 0x38b62000, 1), + (0x2ec, 0xc7420008, 1), + (0x3e0, 0x7fe0, 1), + (0x3e8, 0x7fe00000, 1), + (0x3f4, 0x7fe0, 1), + (0x3f8, 0x8000, 1), + (0x400, 0x80000000, 1), + (0x40c, 0x8000, 1), + (0x5d8, 0x4b232645, 1), + (0x5dc, 0xe98, 1), + (0x5e0, 0xd5a2ea5f, 1), + (0x5e4, 0x40003fff, 1), + (0x5e8, 0xca5e4000, 1), + (0x5ec, 0x4000f5a2, 1), + (0xf6c, 0x1, 1), + (0xf8c, 0xff, 1), + (0xfac, 0x1, 1), + (0xfb8, 0x1d, 1), + (0x2594, 0x2e0000, 1), + (0x2610, 0x1, 1), +]; + +const POWER_CONFIG_WORDS: &[(usize, u32, usize)] = &[ + (0x0, 0x1, 1), + (0x44, 0xc, 1), + (0x9c, 0x43a90000, 1), + (0x200, 0x1, 1), + (0x208, 0x162, 1), + (0x210, 0xd89165f9, 1), + (0x218, 0x1254c, 1), + (0x698, 0xd8917807, 1), + (0x69c, 0x13, 1), + (0x6a8, 0x129a8, 1), + (0x728, 0x4a714, 1), + (0x7a8, 0x4a714, 1), + (0x920, 0x3dd00000, 1), + (0x9ec, 0xbbfca403, 1), + (0x9f0, 0x28, 1), + (0xa70, 0x4, 1), + (0xa7c, 0x0, 1), + (0xa84, 0x3f800000, 1), + (0xa8c, 0x3e0b4396, 1), + (0xa9c, 0x3e99999a, 1), + (0xaa8, 0x28, 1), + (0xab8, 0x43b10000, 1), + (0xac0, 0x162, 1), + (0xad0, 0x0, 1), + (0xae0, 0xc7fca200, 1), + (0xaf0, 0x3e8, 1), + (0xafc, 0x3f800000, 1), + (0xb00, 0x3e0b4396, 1), + (0xb04, 0x3e99999a, 1), + (0xb08, 0x162, 1), + (0xb2c, 0x2, 1), + (0xb88, 0x42a00000, 1), + (0xb90, 0x7, 1), + (0xbc8, 0x3f7eb852, 1), + (0xbcc, 0x3ba3d70a, 1), + (0xbd0, 0x3f64c986, 1), + (0xbd4, 0x403ea7f0, 1), + (0x21f4, 0x42200000, 1), + (0x2210, 0x28, 1), + (0x2264, 0x1020, 1), + (0x226c, 0x2a222aa2, 1), + (0x2294, 0x40baaaaa, 1), + (0x22a4, 0x42080000, 1), + (0x22b0, 0x28, 1), + (0x22c8, 0x1020, 1), + (0x2868, 0x389d8, 1), + (0x2874, 0x154, 1), + (0x28d8, 0x47fd5300, 1), + (0x29bc, 0x74, 1), + (0x29c8, 0x3f772c23, 1), + (0x29d0, 0x3d0d3dcb, 1), + (0x29d8, 0x3cf5c290, 1), + (0x29e8, 0x3f19999a, 1), + (0x2a04, 0x4304ad79, 1), + (0x2a0c, 0x162, 1), + (0x2a2c, 0xc7fd10a9, 1), + (0x2a44, 0x3f772c23, 1), + (0x2a48, 0x3d0d3dcb, 1), + (0x2a4c, 0x3cf5c290, 1), + (0x2a50, 0x3f19999a, 1), + (0x2a54, 0x162, 1), + (0x2b18, 0x2ef94, 2), + (0x2b20, 0x2e89, 1), + (0x2b24, 0xe36, 1), + (0x2b40, 0x1, 1), + (0x2b94, 0x46023400, 1), + (0x2b9c, 0x208d, 1), + (0x2bc4, 0x47800000, 1), + (0x2bcc, 0x3e8, 1), + (0x2be8, 0x208d, 1), + (0x2c40, 0x28, 1), + (0x2c88, 0x3e8, 1), + (0x2cec, 0x1, 1), + (0x2cf0, 0xf, 1), + (0x2cf4, 0x3, 1), + (0x2cfc, 0x6, 1), + (0x2f0c, 0x2a222aa2, 1), + (0x3140, 0x29, 1), + (0x3144, 0x24dd, 1), + (0x3148, 0x4, 1), + (0x314c, 0x20, 1), + (0x3150, 0x29, 1), + (0x3154, 0x10d2c39d, 1), + (0x315c, 0x1d441, 1), + (0x31e0, 0x29, 1), + (0x31e4, 0x24a0, 1), + (0x31e8, 0x4, 1), + (0x31ec, 0x20, 1), + (0x31f0, 0x29, 1), + (0x31f4, 0x10f5cfb4, 1), + (0x31fc, 0x1d4d8, 1), + (0x3230, 0x29, 1), + (0x3234, 0x24ba, 1), + (0x3238, 0x4, 1), + (0x323c, 0x20, 1), + (0x3240, 0x29, 1), + (0x3244, 0x10fe2d62, 1), + (0x324c, 0x1d554, 1), + (0x3280, 0x28, 1), + (0x3284, 0x245c, 1), + (0x3288, 0x4, 1), + (0x328c, 0x20, 1), + (0x3290, 0x28, 1), + (0x3294, 0x10ec3bb1, 1), + (0x329c, 0x1d5bc, 1), + (0x32d0, 0x28, 1), + (0x32d4, 0x246e, 1), + (0x32d8, 0x4, 1), + (0x32dc, 0x20, 1), + (0x32e0, 0x29, 1), + (0x32e4, 0x1101cc20, 1), + (0x32ec, 0x1d640, 1), + (0x3320, 0x28, 1), + (0x3324, 0x2439, 1), + (0x3328, 0x4, 1), + (0x332c, 0x20, 1), + (0x3330, 0x28, 1), + (0x3334, 0x10f01dbd, 1), + (0x333c, 0x1d6dd, 1), + (0x33c0, 0x29, 1), + (0x33c4, 0x2497, 1), + (0x33c8, 0x4, 1), + (0x33cc, 0x20, 1), + (0x33d0, 0x29, 1), + (0x33d4, 0x111077fe, 1), + (0x33dc, 0x1d847, 1), + (0x3460, 0x29, 1), + (0x3464, 0x2480, 1), + (0x3468, 0x4, 1), + (0x346c, 0x20, 1), + (0x3470, 0x29, 1), + (0x3474, 0x110f91ba, 1), + (0x347c, 0x1d8fa, 1), + (0x3500, 0x28, 1), + (0x3504, 0x243a, 1), + (0x3508, 0x4, 1), + (0x350c, 0x20, 1), + (0x3510, 0x28, 1), + (0x3514, 0x10f2bfe8, 1), + (0x351c, 0x1d9a7, 1), + (0x3550, 0x28, 1), + (0x3554, 0x246d, 1), + (0x3558, 0x4, 1), + (0x355c, 0x20, 1), + (0x3560, 0x28, 1), + (0x3564, 0x113b3172, 1), + (0x356c, 0x1da43, 1), + (0x35a0, 0x28, 1), + (0x35a4, 0x2483, 1), + (0x35a8, 0x4, 1), + (0x35ac, 0x20, 1), + (0x35b0, 0x29, 1), + (0x35b4, 0x114027af, 1), + (0x35bc, 0x1dace, 1), + (0x3640, 0x28, 1), + (0x3644, 0x2486, 1), + (0x3648, 0x4, 1), + (0x364c, 0x20, 1), + (0x3650, 0x29, 1), + (0x3654, 0x112421ab, 1), + (0x365c, 0x1db07, 1), + (0x36e0, 0x29, 1), + (0x36e4, 0x24ce, 1), + (0x36e8, 0x4, 1), + (0x36ec, 0x20, 1), + (0x36f0, 0x29, 1), + (0x36f4, 0x114de7b5, 1), + (0x36fc, 0x1dbaa, 1), + (0x3730, 0x29, 1), + (0x3734, 0x24a1, 1), + (0x3738, 0x4, 1), + (0x373c, 0x20, 1), + (0x3740, 0x29, 1), + (0x3744, 0x11342213, 1), + (0x374c, 0x1dc2e, 1), + (0x3780, 0x28, 1), + (0x3784, 0x2446, 1), + (0x3788, 0x4, 1), + (0x378c, 0x20, 1), + (0x3790, 0x28, 1), + (0x3794, 0x111ec867, 1), + (0x379c, 0x1dcb7, 1), + (0x37d0, 0x28, 1), + (0x37d4, 0x2453, 1), + (0x37d8, 0x4, 1), + (0x37dc, 0x20, 1), + (0x37e0, 0x29, 1), + (0x37e4, 0x11273f2d, 1), + (0x37ec, 0x1dd49, 1), + (0x3820, 0x28, 1), + (0x3824, 0x2470, 1), + (0x3828, 0x4, 1), + (0x382c, 0x20, 1), + (0x3830, 0x28, 1), + (0x3834, 0x1131becf, 1), + (0x383c, 0x1ddc8, 1), + (0x38c0, 0x29, 1), + (0x38c4, 0x249c, 1), + (0x38c8, 0x4, 1), + (0x38cc, 0x20, 1), + (0x38d0, 0x29, 1), + (0x38d4, 0x1153d78e, 1), + (0x38dc, 0x1df60, 1), + (0x3960, 0x29, 1), + (0x3964, 0x2484, 1), + (0x3968, 0x4, 1), + (0x396c, 0x20, 1), + (0x3970, 0x29, 1), + (0x3974, 0x114b8bd5, 1), + (0x397c, 0x1dfe3, 1), + (0x3a00, 0x29, 1), + (0x3a04, 0x2522, 1), + (0x3a08, 0x4, 1), + (0x3a0c, 0x20, 1), + (0x3a10, 0x29, 1), + (0x3a14, 0x116ebefa, 1), + (0x3a1c, 0x1e077, 1), + (0x3a50, 0x29, 1), + (0x3a54, 0x2494, 1), + (0x3a58, 0x4, 1), + (0x3a5c, 0x20, 1), + (0x3a60, 0x29, 1), + (0x3a64, 0x11455238, 1), + (0x3a6c, 0x1e0d8, 1), + (0x3aa0, 0x29, 1), + (0x3aa4, 0x24a1, 1), + (0x3aa8, 0x4, 1), + (0x3aac, 0x20, 1), + (0x3ab0, 0x29, 1), + (0x3ab4, 0x1157064b, 1), + (0x3abc, 0x1e165, 1), + (0x3b1c, 0x2522, 1), + (0x3b20, 0x4, 1), + (0x3b24, 0x20, 1), + (0x3b28, 0x29, 1), + (0x3b2c, 0x116cd390, 1), + (0x3b34, 0x1dbf1, 1), + (0x3b40, 0x29, 1), + (0x3b44, 0x34, 1), + (0x3b5c, 0x2a222aa2, 1), + (0x62c4, 0x1000000, 1), + (0x62cc, 0x1010202, 1), + (0x62d0, 0x1010a28, 1), + (0x62d4, 0xa280201, 1), + (0x62d8, 0x2010101, 1), + (0x62dc, 0x1010a28, 1), + (0x62e0, 0xa280101, 1), + (0x62e4, 0x45000101, 1), + (0x62e8, 0x4000200, 1), + (0x62f4, 0x4a000000, 1), + (0x62f8, 0x100, 1), + (0x6304, 0xd7000000, 1), + (0x6308, 0x29, 1), + (0x6328, 0x67325452, 1), + (0x632c, 0x67335452, 1), + (0x6360, 0xf, 1), + (0x63c8, 0x1, 1), + (0x63cc, 0xd88be35c, 1), + (0x63d0, 0x13, 1), + (0x63d8, 0x13, 1), + (0x63dc, 0xd891fc7b, 1), + (0x63e0, 0x13, 1), + (0x63e4, 0xd8633257, 1), + (0x63e8, 0x13, 1), + (0x63ec, 0xd863f8ef, 1), + (0x63f0, 0x13, 1), + (0x63f4, 0xd8903ed2, 1), + (0x63f8, 0x13, 1), + (0x63fc, 0xd89165f9, 1), + (0x6400, 0x13, 1), + (0x6404, 0xd891fc7b, 1), + (0x6408, 0x13, 1), + (0x6414, 0xab2ab62, 1), + (0x6418, 0x7, 1), + (0x641c, 0x30d40, 1), + (0x6428, 0x1, 1), + (0x650c, 0x89778246, 1), +]; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index ea0f9cd782e81d..93d1c86e3425d2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -51,6 +51,7 @@ pub(crate) const CONFIG: Config = Config { opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, region_c: &crate::g17::fw::initdata::REGION_C_26A428, + chip: None, adopt_root_entries: true, fabric_bridge: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), @@ -60,6 +61,7 @@ pub(crate) const CONFIG: Config = Config { alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, }, }; diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 6a7bd5057fa543..a984b9d2520387 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -44,6 +44,16 @@ const FAULT_INFO: usize = 0x17030; const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; const G17_HOST_POWERED: u32 = 0xf; +const KSM_MAILBOX_TAG: usize = 0x4090; +const KSM_MAILBOX_DATA: usize = 0x4098; +const KSM_MAILBOX_FLAGS: u64 = (1 << 42) | (1 << 53); +const KSM_SLOT_DESCRIPTOR: u64 = 0x21008; +const KSM_SLOT_TIMESTAMP: u64 = 0x21020; +const KSM_SELECT: usize = 0x103a8; +const KSM_RUN_SET: usize = 0x21068; +const KSM_RUN_CLEAR: usize = 0x21078; +const KSM_RUNNING: [usize; 2] = [0x21038, 0x21040]; + const ID_VERSION: usize = 0xd04000; const ID_UNK08: usize = 0xd04008; const ID_COUNTS_1: usize = 0xd04010; @@ -600,6 +610,52 @@ impl G17Resources { Ok(true) } + /// Registers a queue KSM slot through the firmware mailbox. + pub(crate) fn register_ksm_queue(&self, qid: u8, descriptor: u64) -> Result { + if qid > 127 { + return Err(EINVAL); + } + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let tag = |reg: u64| reg | KSM_MAILBOX_FLAGS | (u64::from(qid) << 43); + let bit = 1u64 << (qid & 63); + let (low, high) = if qid < 64 { (bit, 0) } else { (0, bit) }; + let running = KSM_RUNNING[usize::from(qid >= 64)]; + if sgx.try_read64(running)? & bit != 0 { + sgx.try_write64(high, KSM_RUN_CLEAR + 8)?; + sgx.try_write64(low, KSM_RUN_CLEAR)?; + } + sgx.try_write64(tag(KSM_SLOT_DESCRIPTOR), KSM_MAILBOX_TAG)?; + sgx.try_write64(descriptor, KSM_MAILBOX_DATA)?; + sgx.try_write64(tag(KSM_SLOT_TIMESTAMP), KSM_MAILBOX_TAG)?; + sgx.try_read64(KSM_MAILBOX_DATA)?; + sgx.try_write64(tag(KSM_SLOT_DESCRIPTOR), KSM_MAILBOX_TAG)?; + let mut readback = sgx.try_read64(KSM_MAILBOX_DATA)?; + let direct = readback != descriptor; + if direct { + sgx.try_write64(u64::from(qid), KSM_SELECT)?; + sgx.try_write64(descriptor, KSM_SLOT_DESCRIPTOR as usize)?; + readback = sgx.try_read64(KSM_SLOT_DESCRIPTOR as usize)?; + } + if readback != descriptor { + dev_err!( + self.dev.as_ref(), + "KSM queue {} slot descriptor {:#x} did not take (reads {:#x})\n", + qid, + descriptor, + readback + ); + return Err(EIO); + } + if sgx.try_read64(running)? & bit == 0 { + sgx.try_write64(high, KSM_RUN_SET + 8)?; + sgx.try_write64(low, KSM_RUN_SET)?; + } + sgx.try_write64(tag(KSM_SLOT_TIMESTAMP), KSM_MAILBOX_TAG)?; + sgx.try_read64(KSM_MAILBOX_DATA)?; + Ok(()) + } + fn sgx_read32(&self) -> u32 { if let Some(sgx) = self.sgx.try_access() { (&*sgx).as_view().relaxed().read32(OFF) From 1784b459d11195fcd0c6c8d2beae2df0e173c650 Mon Sep 17 00:00:00 2001 From: Artyom Astafurov Date: Tue, 3 Nov 2026 12:57:29 -0500 Subject: [PATCH 052/171] drm/asahi: write T6041's tiler segment control on 25F84 macOS on J616c writes 0x1c890 = 1 and 0x1c918 = 0 for its tiling passes, where the shared firmware profiles write 0x200440 and 4. Do the same in T6041's 25F84 build and leave the shared profiles unchanged. 0x1c9a8 stays at the shared 0x3c; macOS writes 0x5c here. This does not fix the 25F84 reason-0 restarts under concurrent rendering. They still occurred with these values on an earlier base, also with 0x1c9a8 = 0x5c. Ran on J616c with 25F84, with PRs 732, 749 and 920 and Mesa ca90d8c1919, serial rendering off and the firmware log on: three boots of niri, DMS and 20 Whisper runs each, with no GPU recovery. Add KUnit coverage for the policy boundary and constructor output. Link: https://github.com/aurora-silicon/linux/pull/525#issuecomment-6486668315 Assisted-by: LLM Signed-off-by: Artyom Astafurov (cherry picked from commit 18ca0a651ad2a482dd43fc3b4d38ef1d605ee49c in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 162 +++++++++++++++++++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 4 + drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 6 +- 4 files changed, 170 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index dd66da538f0fb2..7933828961b781 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -376,6 +376,8 @@ pub(crate) struct RenderArgs<'a> { pub(crate) fragment: RenderStage, /// Register programs of the firmware build. pub(crate) programs: RegisterPrograms, + /// Device/build policy for registers 0x1c890 and 0x1c918 in V25G83-format arrays. + pub(crate) ta_mode_1: bool, /// Deflake-buffer layout of the GPU. pub(crate) deflake: Deflake, } @@ -666,6 +668,8 @@ fn ta_registers( /// Tiler segment control of a two-cluster pass, what macOS writes. const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; const VDM_SEGMENT_INDEX: u64 = 4; +/// Tiler segment control of T6041 on 25F84, what macOS writes, with 0x1c918 = 0. +const T6041_TA_MODE_1: u64 = 1; /// Pages the tiler keeps out of a partial render's segments: without them the pages a pass /// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. const MIN_FREE_TVB_PAGES: u64 = 0x3c; @@ -720,8 +724,13 @@ fn ta_registers_25g83( push(find(0x1c880, 0)?); push(RegisterWrite::new(0x1c948, gtp(4)?)); push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); - push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); + let (segment_control, segment_index) = if args.ta_mode_1 { + (T6041_TA_MODE_1, 0) + } else { + (VDM_SEGMENT_CONTROL, VDM_SEGMENT_INDEX) + }; + push(RegisterWrite::new(0x1c890, segment_control)); + push(RegisterWrite::new(0x1c918, segment_index)); push(find(0x1c079, 0)?); push(find(0x1c9d8, 0)?); push(RegisterWrite::new(0x1c089, gtp2)); @@ -1600,3 +1609,152 @@ impl FragmentSlot { Ok(Some(table)) } } + +#[kunit_tests(asahi_g17_ta_mode)] +mod ta_mode_tests { + use super::*; + use crate::g17::fw::{ + compute::{IntersectionTables, ScratchRequest}, + render_rt::RayQueryStage, + variant::{Firmware, Variant, V25F84, V25G83, V26A428}, + }; + + fn selected(firmware: &Firmware) -> &'static Variant { + match firmware { + Firmware::Fixed(variant) => variant, + Firmware::Identified(builds) => { + assert_eq!(builds.len(), 1); + builds[0].variant + } + } + } + + #[test] + fn policy_is_local_to_the_admitted_t6041_build() { + let max = selected(&crate::hw::t6041::CONFIG.firmware); + let pro = selected(&crate::hw::t6040::CONFIG.firmware); + assert!(max.is_25f84()); + assert_eq!(max.programs, RegisterPrograms::V25G83); + assert!(max.ta_mode_1); + assert!(!pro.ta_mode_1); + assert!(!V25F84.ta_mode_1); + assert!(!V25G83.ta_mode_1); + assert!(!V26A428.ta_mode_1); + } + + fn fixture() -> (RenderPass, RenderResources) { + // SAFETY: These two types contain only integers, bools and structs of + // those types, for which all-zero bytes are valid. Required render + // dimensions and user-window addresses are filled below. + let mut pass: RenderPass = unsafe { core::mem::zeroed() }; + // SAFETY: RenderResources contains only integers and integer structs. + let mut resources: RenderResources = unsafe { core::mem::zeroed() }; + pass.width = 64; + pass.height = 64; + pass.layers = 1; + pass.utile_width = 32; + pass.utile_height = 32; + pass.sample_size = 4; + pass.vdm_va = USER_BASE + 0x10000; + resources.tilemap_va = USER_BASE + 0x20000; + resources.layermeta_va = USER_BASE + 0x30000; + resources.heapmeta_va = USER_BASE + 0x40000; + resources.tpc_va = USER_BASE + 0x50000; + resources.deflake_va = USER_BASE + 0x60000; + resources.gtp_va = core::array::from_fn(|i| USER_BASE + 0x70000 + i as u64 * 0x1000); + (pass, resources) + } + + fn args<'a>(pass: &'a RenderPass, resources: &'a RenderResources, scs: bool) -> RenderArgs<'a> { + let stage = RenderStage { + qid: 1, + descriptor_va: USER_BASE + 0x80000, + state_word: 1, + stamp_va: USER_BASE + 0x90000, + aux_stamp_va: USER_BASE + 0x90008, + }; + // Exercise the array with and without its optional SCS entry. + let ray = RayQueryStage { + intersection: IntersectionTables::NONE, + scratch: ScratchRequest::new(if scs { (1 << 32) | (1 << 24) | 8 } else { 0 }).unwrap(), + }; + RenderArgs { + pass, + resources, + ray_query: [ray; 2], + ordinal: 0, + ta: stage, + fragment: stage, + programs: RegisterPrograms::V25G83, + ta_mode_1: false, + deflake: Deflake::FOUR_CLUSTERS, + } + } + + #[test] + fn ta_mode_changes_only_the_segment_writes_with_or_without_scs() { + let (mut pass, resources) = fixture(); + for (width, height, layers) in [(64, 64, 1), (3456, 2234, 2)] { + pass.width = width; + pass.height = height; + pass.layers = layers; + let geometry = Geometry::new(&pass).unwrap(); + let words = PassWords::new(&pass).unwrap(); + for scs in [false, true] { + let mut args = args(&pass, &resources, scs); + let (before, count) = ta_registers_25g83(&args, &geometry, &words).unwrap(); + args.ta_mode_1 = selected(&crate::hw::t6041::CONFIG.firmware).ta_mode_1; + let (after, new_count) = ta_registers_25g83(&args, &geometry, &words).unwrap(); + assert_eq!(count, 83 + usize::from(scs)); + assert_eq!(new_count, count); + let slot = after[79 + usize::from(scs)].reg; + assert_eq!(slot, 0x14308); + let mut changed = 0; + for (old, new) in before.iter().zip(&after).take(count) { + let old = (old.reg, old.value); + let new = (new.reg, new.value); + if old.0 == 0x1c890 { + assert_eq!(old.1, VDM_SEGMENT_CONTROL); + assert_eq!(new, (0x1c890, T6041_TA_MODE_1)); + changed += 1; + } else if old.0 == 0x1c918 { + assert_eq!(old.1, VDM_SEGMENT_INDEX); + assert_eq!(new, (0x1c918, 0)); + changed += 1; + } else { + // Includes every merge-buffer write and 0x1c9a8. + assert_eq!(old, new); + } + } + assert_eq!(changed, 2); + } + } + } + + #[test] + fn mode_does_not_change_fragment_or_g17p_arrays() { + let (pass, resources) = fixture(); + let geometry = Geometry::new(&pass).unwrap(); + let words = PassWords::new(&pass).unwrap(); + for scs in [false, true] { + let mut args = args(&pass, &resources, scs); + let (before, count) = fragment_registers_25g83(&args, &geometry, &words).unwrap(); + args.ta_mode_1 = true; + let (after, new_count) = fragment_registers_25g83(&args, &geometry, &words).unwrap(); + assert_eq!(count, 91 + usize::from(scs)); + assert_eq!(new_count, count); + for (old, new) in before.iter().zip(&after).take(count) { + assert_eq!((old.reg, old.value), (new.reg, new.value)); + } + args.programs = RegisterPrograms::G17P; + args.ta_mode_1 = false; + let before = ta_registers(&args, &geometry, &words).unwrap(); + args.ta_mode_1 = true; + let after = ta_registers(&args, &geometry, &words).unwrap(); + for (old, new) in before.iter().zip(&after) { + assert_eq!((old.reg, old.value), (new.reg, new.value)); + assert_ne!({ new.reg }, 0x1c890); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 3e89d510fc83cb..4a32d585d41902 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -79,6 +79,8 @@ pub(crate) struct Variant { pub(crate) lead_commands: bool, /// Register programs of the render descriptors. pub(crate) programs: super::render::RegisterPrograms, + /// Use 0x1c890 = 1, 0x1c918 = 0 in V25G83-format tiling arrays (device/build policy). + pub(crate) ta_mode_1: bool, pub(crate) compute: super::compute::ComputeProgram, pub(crate) opening_record: bool, pub(crate) split_firmware_area: bool, @@ -123,6 +125,7 @@ pub(crate) const V26A428: Variant = Variant { fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, + ta_mode_1: false, compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, @@ -154,6 +157,7 @@ pub(crate) const V25G83: Variant = Variant { fragment_busy: false, lead_commands: true, programs: super::render::RegisterPrograms::V25G83, + ta_mode_1: false, compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2436cfd33ed35b..9019271b239d0d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -466,6 +466,7 @@ impl Pair { aux_stamp_va: addresses.stamps[1][1], }, programs: self.variant.programs, + ta_mode_1: self.variant.ta_mode_1, deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 956778038d1d48..8be466f7842ab9 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -60,7 +60,11 @@ pub(crate) const CONFIG: Config = Config { 0xa3, 0xc1, 0x83, 0xe8, ], }], - variant: &fw::variant::V25F84, + // macOS writes these tiler segment values on this machine, not the two-cluster ones. + variant: &fw::variant::Variant { + ta_mode_1: true, + ..fw::variant::V25F84 + }, }]), core_layout: CoreLayout { cores: Some((4, 40)), From 187e26b5f59cfe7c5c6f231f9d1ddb3295c4aa79 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:27:22 +1100 Subject: [PATCH 053/171] drm/asahi: drop the unread G17 firmware version tag Variant::version was only read by is_25f84(), which only the TA mode KUnit test calls, so every build without KUnit warned that the field is never read and the method never used. The variant already carries what differs between the builds. Check the T6041 build in that test by its 25F84 compute program and split firmware area instead. Assisted-by: LLM Signed-off-by: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> (cherry picked from commit a9a68c6f3843a29da6d50c2f08dc6db7ec4a564d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/render.rs | 3 ++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 17 ----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 7933828961b781..8fcc25500ab2d0 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1633,7 +1633,8 @@ mod ta_mode_tests { fn policy_is_local_to_the_admitted_t6041_build() { let max = selected(&crate::hw::t6041::CONFIG.firmware); let pro = selected(&crate::hw::t6040::CONFIG.firmware); - assert!(max.is_25f84()); + assert_eq!(max.compute, crate::g17::fw::compute::ComputeProgram::V25F84); + assert!(max.split_firmware_area); assert_eq!(max.programs, RegisterPrograms::V25G83); assert!(max.ta_mode_1); assert!(!pro.ta_mode_1); diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 4a32d585d41902..c5287e66a641d0 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -43,13 +43,6 @@ pub(crate) enum QueueRegistration { Mailbox, } -#[derive(Copy, Clone, PartialEq, Eq)] -pub(crate) enum FirmwareVersion { - V26A428, - V25G83, - V25F84, -} - /// The layouts and rules of one G17 firmware build. #[derive(Copy, Clone)] pub(crate) struct Variant { @@ -84,7 +77,6 @@ pub(crate) struct Variant { pub(crate) compute: super::compute::ComputeProgram, pub(crate) opening_record: bool, pub(crate) split_firmware_area: bool, - pub(crate) version: FirmwareVersion, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -129,7 +121,6 @@ pub(crate) const V26A428: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, - version: FirmwareVersion::V26A428, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -161,11 +152,9 @@ pub(crate) const V25G83: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, - version: FirmwareVersion::V25G83, }; pub(crate) const V25F84: Variant = Variant { - version: FirmwareVersion::V25F84, qid_floor: 0, registration: QueueRegistration::Mailbox, compute: super::compute::ComputeProgram::V25F84, @@ -174,12 +163,6 @@ pub(crate) const V25F84: Variant = Variant { ..V25G83 }; -impl Variant { - pub(crate) const fn is_25f84(&self) -> bool { - matches!(self.version, FirmwareVersion::V25F84) - } -} - const _: () = { assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); From 399c38f1d7b63afabfa5af7b55a5ec0f9db5fb38 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 12:32:20 -0300 Subject: [PATCH 054/171] drm/asahi: g17: blame the running VM, not a starved one, for a keyless timeout On the M4 Pro (25G83) every recovery slot key reads 0, so recovery attribution cannot check the firmware's blame and always keeps it. A fragment shader spinning on a lock then takes down unrelated clients: a restart drops only the guilty pair's started group, its next queued group spins again, and each following recovery 83 ms later blames whichever queue the spinner starved. The driver fails that queue's oldest render with EIO, which poisons its VM. On 11-03 that killed Hyprland, quickshell and T3 Code in two cascades of deqp-vk hangs. In five of the six innocent blames the blamed render had never started (tiler start and fragment end unwritten). When slot identity cannot judge a timeout (not a page fault), read the host's own pass records at the same halted instant: if the blamed pair's oldest published render has not started, and exactly one VM has a published render that started and did not finish, and no other VM holds compute work, blame that VM's queues instead. Groups an earlier recovery listed or blamed are skipped; the restart dropped them, so their records stay half-written. In every other case the firmware's blame stands, as before. A blamed compute queue keeps the firmware's blame: only render pairs have the pass records this reads. This does not stop the guilty pair's queued groups from running after the restart, so a client with many of them can still reach the restart storm limit. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 3ff3dc1e03aed9c2efb2545f2190da2b3d4f25af in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 24 ++++++++++++ drivers/gpu/drm/asahi/g17/recovery.rs | 10 +++++ .../gpu/drm/asahi/g17/recovery/attribution.rs | 39 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 ++ .../drm/asahi/g17/runtime/render/recovery.rs | 30 +++++++++++++- 5 files changed, 104 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 9019271b239d0d..c8833311d8cd7e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1298,6 +1298,30 @@ impl Pair { Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } + /// The oldest published group has neither its tiler start nor its fragment end written. + pub(crate) fn front_untouched(&self) -> Option { + let front = self.active.iter().find(|active| active.outer_published)?; + Some(observation::snapshot(&self.memory, front.ticket.ordinal).ok()?.untouched()) + } + + /// The owner of a published group whose tiler started and whose fragment end is unwritten. + /// Groups an earlier recovery listed or blamed are skipped: a restart drops started groups, + /// so their records stay half-written without anything running. + pub(crate) fn running_owner(&self) -> Option<&Arc> { + self.active + .iter() + .filter(|active| { + active.outer_published + && !active.packet.completion.interrupted() + && !active.packet.completion.blamed() + }) + .find(|active| { + observation::snapshot(&self.memory, active.ticket.ordinal) + .is_ok_and(|snapshot| snapshot.pass[0] != 0 && snapshot.pass[1] == 0) + }) + .map(|active| active.packet.completion.status()) + } + /// Firmware is halted and all graph borrowers have joined before this copy. pub(crate) fn publish_recovery_targets(&self) -> Result { self.memory.graph.publish_recovery_targets() diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 8e46bd467fe1c8..33e4a785351963 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -92,6 +92,11 @@ pub(super) enum Class { pub(super) trait Queues { fn render_scheduler_active(&self) -> bool; fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; + /// Read while firmware is halted, like `render_pass_started`. When the oldest published + /// render of the pair holding `blamed` had not started, the QIDs of every pair of the one VM + /// with a render that started and did not finish. None if no such VM, several, or any other + /// VM holds compute work, which these records cannot see. + fn render_culprit(&self, _blamed: u8) -> Option { None } /// Copy every retained queue's live producer to its recovery target, then /// issue a full barrier. A failed graph must not skip subsequent queues. fn publish_recovery_targets(&self) -> Option; @@ -252,6 +257,11 @@ pub(super) fn service( // acknowledged; sampling them during classification can blame a bystander. let attribution = sources.attribute(request.blamed, |qid, stamp| host.render_pass_started(qid, stamp)); + let attribution = sources.host_fallback(attribution, + || host.render_culprit(request.blamed?)); + if let Attribution::Queues(qids) = attribution { + pr_warn!("G17: recovery gen {} blames queues {:#x}\n", request.generation, qids); + } let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index 38ad8612a0b99c..fdf0cd62cbd075 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -122,6 +122,21 @@ impl Sources { Attribution::Queues(guilty) } } + + /// When slot identity cannot check a timeout's blame (25G83 reads every key as 0, and a + /// single-slot mask has no peer), the host's own pass records still can: a blamed render + /// that never started was starved, not hung. `culprit` names the queues of the one VM whose + /// render started and did not finish; without one the firmware's blame stands. + pub(super) fn host_fallback( + &self, + attribution: Attribution, + culprit: impl FnOnce() -> Option, + ) -> Attribution { + if attribution != Attribution::Firmware || self.reason.is_none_or(|reason| reason == PAGE_FAULT) { + return attribution; + } + culprit().map_or(Attribution::Firmware, Attribution::Queues) + } } #[kernel::macros::kunit_tests(asahi_g17_recovery_sources)] @@ -173,4 +188,28 @@ mod tests { assert_eq!(sources.attribute(Some(38), |qid, _| Some(qid == 39)), Attribution::Firmware); } + + #[test] + fn host_records_move_a_keyless_timeout_blame_to_the_running_vm() { + let mut sources = Sources::sample(0b10, |_| Some((0, 2))); + sources.reason = Some(0); + let attribution = sources.attribute(Some(36), |_, _| panic!("unknown key")); + assert_eq!(attribution, Attribution::Firmware); + assert_eq!(sources.host_fallback(attribution, || Some(3 << 60)), Attribution::Queues(3 << 60)); + assert_eq!(sources.host_fallback(attribution, || None), Attribution::Firmware); + } + + #[test] + fn host_records_never_override_a_page_fault_or_slot_evidence() { + let mut sources = Sources::sample(0b10, |_| Some((0, 2))); + sources.reason = Some(PAGE_FAULT); + assert_eq!(sources.host_fallback(Attribution::Firmware, || panic!("page fault")), + Attribution::Firmware); + sources.reason = None; + assert_eq!(sources.host_fallback(Attribution::Firmware, || panic!("no reason")), + Attribution::Firmware); + sources.reason = Some(0); + assert_eq!(sources.host_fallback(Attribution::NoCulprit, || panic!("slot evidence")), + Attribution::NoCulprit); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3e779bba4fbbfb..ba76bb3735062e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -158,6 +158,9 @@ impl recovery::Queues for Service<'_> { fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { self.firmware.queues.render_pass_started(qid, stamp) } + fn render_culprit(&self, blamed: u8) -> Option { + self.firmware.queues.render_culprit(blamed) + } fn publish_recovery_targets(&self) -> Option { let mut failure = self.firmware.queues.publish_render_recovery_targets(); for queue in self diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs index 7a6d987591c764..e597f63f15b659 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -2,8 +2,8 @@ //! Recovery targets for retained render graphs while firmware is halted. -use crate::g17::runtime::Registry; -use kernel::prelude::*; +use crate::g17::{runtime::Registry, status::VmStatus}; +use kernel::{prelude::*, sync::Arc}; impl Registry { pub(in crate::g17::runtime) fn report_render_recovery(&self, qid: u8, port: &crate::g17::port::Port) { @@ -29,6 +29,32 @@ impl Registry { .pass_started(stamp) } + /// See `recovery::Queues::render_culprit`. + pub(in crate::g17::runtime) fn render_culprit(&self, blamed: u8) -> Option { + let pairs = || self.render.entries.iter().flatten().filter_map(|entry| entry.pair.as_ref()); + if pairs().find(|pair| pair.qids().contains(&blamed))?.front_untouched() != Some(true) { + return None; + } + let mut culprit: Option<&Arc> = None; + let mut qids = 0u128; + for pair in pairs() { + let Some(owner) = pair.running_owner() else { + continue; + }; + if culprit.is_some_and(|known| !Arc::ptr_eq(known, owner)) { + return None; + } + culprit = Some(owner); + qids |= pair.qids().into_iter().fold(0, |mask, qid| mask | 1u128 << qid); + } + let culprit = culprit?; + let mut foreign = false; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.visit_owners(&mut |owner| foreign |= !Arc::ptr_eq(owner, culprit)); + } + (!foreign).then_some(qids) + } + pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { let mut failure = None; for pair in self.render.entries.iter().flatten() From 6dbc5728adeafe09ee6114b13e49070eabb9ad4c Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 13:00:47 -0300 Subject: [PATCH 055/171] drm/asahi: g17: keep another VM's recovery out of an innocent VM's status A command a GPU restart interrupted fails with ECANCELED. Unless a recovery found its VM guilty, it does not poison the VM, but it was still passed to VmStatus::report_failure, which stores it in the VM status that GET_PARAMS VM_STATUS and the status mirror return. Honeykrisp's check_status treats any nonzero status as a lost device, so every Vulkan client with work in flight lost its device in another client's recovery, while GL clients, which never read the status that way, survived. The UAPI documents the opposite: a job cancelled because another VM's recovery discarded it is not a failure of this VM, and its error stays 0. Report a non-poisoning ECANCELED to nobody, as was already done for work cancelled by its owner. The fence still carries ECANCELED, the restart is still logged ("interrupted by a GPU recovery after it started"), and a guilty VM is still poisoned with it. Under Vulkan such a submission's output is undefined, as the UAPI says, instead of the device being lost. A restart that finds nobody guilty (no blamed queue and no guilty table entries, or owners it cannot resolve) now fails all interrupted work this way, the culprit's too, so a culprit that keeps hanging is stopped only by the restart storm limit. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit b8cb1a94ded4fb4120b35c5bdfe601ed9c52eb8a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/completion.rs | 11 ++++++----- drivers/gpu/drm/asahi/g17/queue/render.rs | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 90f6d77780bff0..9e285b81c2cf2f 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -547,7 +547,7 @@ impl Completion { /// A firmware restart listed this command in its recovery table: it was in /// flight, so it may have run partly. Its stamps and completion record may /// still look like success. Unless a replay republishes it from the start, - /// it fails and the failure is always reported to the client. + /// it fails: its fence carries ECANCELED, and only a guilty VM is poisoned. pub(crate) fn note_interrupted(&self) { self.interrupted.store(true, Ordering::Release); } @@ -563,12 +563,13 @@ impl Completion { (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) } - /// ECANCELED stays silent only for work that was never interrupted (closed - /// or cancelled by its owner); every other failure reaches the client. + /// A non-poisoning ECANCELED stays out of the VM status, as the UAPI documents for work + /// another VM's recovery discarded: Honeykrisp reads any status error as a lost device. + /// Every other failure reaches the client. fn report(&self, error: Error) { if self.poisons(error) { self.status.record(error); - } else if error != ECANCELED || self.interrupted() { + } else if error != ECANCELED { self.status.report_failure(error); } } @@ -715,7 +716,7 @@ impl Completion { guard.quarantine(); } if self.spared() { - if error != ECANCELED || self.interrupted() { + if error != ECANCELED { self.status.report_failure(error); } } else { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c8833311d8cd7e..25995472b55287 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1697,9 +1697,10 @@ impl Pair { /// A front group a restart interrupted after its tiler started, which firmware /// then retired without writing its fragment end, may have run partly. It is - /// not replayed (blending or storage writes could apply twice) and fails now, - /// reported to its owner, instead of waiting out the render timeout. Returns - /// None when the front is not such a group. + /// not replayed (blending or storage writes could apply twice) and fails now + /// instead of waiting out the render timeout: its fence carries ECANCELED, and + /// only a guilty VM is poisoned. Returns None when the front is not such a + /// group. fn settle_interrupted_front( &mut self, host: &mut impl Host, From 8619dc4ac4f08379afc8425f9ac24bb0f7a2f13d Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 13:00:47 -0300 Subject: [PATCH 056/171] drm/asahi: g17: do not count a failed VM's leftover hangs as a restart storm A firmware restart drops the groups that had started, but a hung client's other published groups run as usual afterwards. A client that queued the same spinning draw many times (deqp-vk on 11-03 had 17 renders retained on one pair) therefore hangs the GPU once per group, 83 ms apart. Once the blame lands on that client, its VM is poisoned and it can publish nothing new, so the cascade ends with its queued work. Yet the eighth restart within 30 s reaches the storm limit and stops the GPU for every client. When every VM a recovery finds guilty had already failed before it, and the recovery table matched one of their commands by its exact stamp (so the restart dropped it), take that restart back from the storm count. At most RENDER_DEPTH restarts are forgiven per window, so an unknown firmware loop still reaches the limit. A first failure always counts. The host cannot stop the queued groups themselves: 25G83 has no recovery target word, and a ContextKill is only sent for closed contexts. Releasing a killed live context's commands would depend on the kill acknowledgement waiting for a running spinner, which is unverified. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 109dac13643f33c22d1bdea88ad508e3ffa15a81 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 78 ++++++++++++++++++-- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 33 ++++++++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 + 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 33e4a785351963..8095a1e13da9f6 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -45,6 +45,8 @@ pub(super) struct Entries { /// Listed commands found by their exact stamp, and by the oldest-work fallback. pub(super) exact: u32, pub(super) fallback: u32, + /// QIDs with a listed command found by its exact stamp: that command was dropped. + exact_qids: u128, } impl Entries { @@ -149,6 +151,8 @@ pub(super) struct State { pub(super) struct ServiceResult { pub(super) scan_error: Option, pub(super) target_error: Option, + /// Every VM found guilty had failed before, and the restart dropped one of its commands. + pub(super) leftover: bool, } impl State { @@ -164,6 +168,9 @@ impl State { self.timeouts.note_unless_progress(now, safety::retired()) } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } + /// A restart that only dropped leftover work of failed VMs (`ServiceResult::leftover`) + /// does not count toward the storm limit. Returns whether it was forgiven. + pub(super) fn forgive_restart(&mut self) -> bool { self.restarts.forgive() } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -292,7 +299,7 @@ pub(super) fn service( host.recovery_mut().pending = None; let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); - let (cancelled, blame) = classify(host, entries, request.blamed, attribution); + let (cancelled, blame, leftover) = classify(host, entries, request.blamed, attribution); if listed != 0 { pr_warn!( "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", @@ -317,7 +324,7 @@ pub(super) fn service( } let scan_error = host.rescan_compute().err(); host.release_retired_compute(); - Ok(ServiceResult { scan_error, target_error }) + Ok(ServiceResult { scan_error, target_error, leftover }) } /// Walks the packed recovery table up to its first invalid record and marks @@ -331,8 +338,12 @@ fn read_entries(host: &mut H) -> Entries { if !entries.include(flags) { break; } - match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { - Interrupted::Exact => entries.exact += 1, + let qid = ((flags >> 4) & 0x7f) as u8; + match host.note_interrupted(qid, stamp) { + Interrupted::Exact => { + entries.exact += 1; + entries.exact_qids |= 1u128 << qid; + } Interrupted::Oldest => entries.fallback += 1, Interrupted::None => {} } @@ -346,7 +357,9 @@ fn read_entries(host: &mut H) -> Entries { /// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed /// render that had not started while others had (or with no culprit at all) is innocent, and its /// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { +/// Last, whether every guilty VM had failed before this restart and it dropped one of their +/// commands: a failed VM publishes nothing new, so such restarts end with its queued work. +fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; match attribution { @@ -357,7 +370,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att let no_culprit = attribution == Attribution::NoCulprit; let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return (0, blame); + return (0, blame, false); } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -393,8 +406,14 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return (0, blame); + return (0, blame, false); + } + let known = |status: &Arc| guilty[..count].iter().flatten().any(|known| Arc::ptr_eq(known, status)); + let mut dropped = false; + for qid in (0..QIDS).filter(|qid| entries.exact_qids & (1u128 << qid) != 0) { + queues.visit_owners(qid, &mut |status| dropped |= known(status)); } + let leftover = count != 0 && dropped && guilty[..count].iter().flatten().all(|status| status.get() != 0); let spared = queues.classify_work(&|status| { guilty[..count] .iter() @@ -406,7 +425,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att }) .unwrap_or(Class::Spared) }); - (cancelled & !(guilty_qids & !collateral) & spared, blame) + (cancelled & !(guilty_qids & !collateral) & spared, blame, leftover) } @@ -498,6 +517,49 @@ mod tests { assert!(!host.blamed); } + /// Two VMs, each owning one render pair: `a` queues 40/41, `b` queues 60/61. + struct Owners([(u8, Arc); 4]); + + impl Queues for Owners { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, _qid: u8, _stamp: u64) -> Option { None } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, qid: u8) -> Option { + self.0.iter().any(|(own, _)| *own == qid).then_some(false) + } + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { + self.0.iter().filter(|(own, _)| *own == qid).for_each(|(_, status)| visit(status)); + } + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } + fn settle_blamed_render(&mut self, _qid: u8, _error: Error) {} + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + } + + fn owners() -> Owners { + let a = Arc::pin_init(VmStatus::new(), GFP_KERNEL).unwrap(); + let b = Arc::pin_init(VmStatus::new(), GFP_KERNEL).unwrap(); + Owners([(40, a.clone()), (41, a), (60, b.clone()), (61, b)]) + } + + fn leftover(queues: &mut Owners, exact_qid: u8) -> bool { + let entries = Entries { exact_qids: 1u128 << exact_qid, ..Entries::default() }; + classify(queues, entries, Some(40), Attribution::Queues(3u128 << 60)).2 + } + + #[test] + fn only_dropped_work_of_an_already_failed_vm_is_leftover() { + let mut queues = owners(); + // A first failure counts toward the storm. + assert!(!leftover(&mut queues, 60)); + queues.0[2].1.record(EIO); + assert!(leftover(&mut queues, 60)); + // The restart dropped nothing of the failed VM. + assert!(!leftover(&mut queues, 40)); + } + #[test] fn unknown_25g83_keys_still_preserve_firmware_blame() { let mut host = Halted::new(false); diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index d71d3cc81553f2..5c2ad346c98904 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -6,6 +6,9 @@ use core::sync::atomic::{AtomicU64, Ordering}; const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Restarts forgiven per window: one per group a render pair holds. Leftover work of a failed +/// VM ends on its own, but an unknown firmware loop must still reach the limit. +const FORGIVE_LIMIT: u8 = crate::hw::t8140::queues::RENDER_DEPTH; /// Commands retired successfully. A watchdog storm means the GPU stopped making progress; /// timeouts of overloaded clients while other work keeps retiring are not one. @@ -34,12 +37,13 @@ pub(crate) fn since_retired(now: u64) -> Option { pub(super) struct Storm { start: Option, count: u8, + forgiven: u8, stopped: bool, progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false, progress: 0 } + Self { start: None, count: 0, forgiven: 0, stopped: false, progress: 0 } } /// Like [`Self::note`], but work retired since the previous event starts a new window. pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { @@ -54,11 +58,21 @@ impl Storm { if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { self.start = Some(now); self.count = 0; + self.forgiven = 0; } self.count += 1; self.stopped = self.count >= LIMIT; self.stopped } + /// Takes back the latest event, at most [`FORGIVE_LIMIT`] times per window. A stop stays. + pub(super) fn forgive(&mut self) -> bool { + if self.stopped || self.count == 0 || self.forgiven >= FORGIVE_LIMIT { + return false; + } + self.count -= 1; + self.forgiven += 1; + true + } } #[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] @@ -109,6 +123,23 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn forgiven_restarts_are_bounded_per_window() { + let mut storm = Storm::new(); + for _ in 0..FORGIVE_LIMIT { + assert!(!storm.note(0)); + assert!(storm.forgive()); + } + assert!(!storm.note(0)); + assert!(!storm.forgive()); + for _ in 2..LIMIT { assert!(!storm.note(0)); } + assert!(storm.note(0)); + assert!(!storm.forgive()); + let mut storm = Storm::new(); + for _ in 0..FORGIVE_LIMIT { storm.note(0); storm.forgive(); } + assert!(!storm.note(WINDOW_NS)); + assert!(storm.forgive()); + } + #[test] fn recovery_table_ends_at_its_first_invalid_record() { use crate::g17::recovery::Entries; let mut entries = Entries::default(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index ba76bb3735062e..d6d811b410e89f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -326,6 +326,9 @@ impl Firmware { .set_generation(service.firmware.recovery.generation()); service.firmware.rings.invalidate_statistics(); let result = result?; + if result.leftover && service.firmware.recovery.forgive_restart() { + pr_warn!("G17: recovery dropped queued work of an already failed VM; not counted as a restart storm\n"); + } if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), From b650ddc7775d4b3ac000d90ab4c40bbd97652ede Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 14:15:03 -0300 Subject: [PATCH 057/171] drm/asahi: g17: keep work a recovery dropped out of the VM status When a GPU restart drops an innocent pair's groups and they cannot run again in order, fail_dropped fails them with ENODATA without poisoning the VM. The failure was still passed to VmStatus::report_failure, which stores it in the status that GET_PARAMS VM_STATUS and the status mirror return, and Honeykrisp's check_status treats any nonzero status as a lost device. In the 16-group spin repro on m4.119, vkmark lost its device this way twice ("failing 2 groups ... ENODATA ... the client sees a lost device") although no recovery blamed it. Treat a non-poisoning ENODATA like a non-poisoning ECANCELED: the fence carries the error, the VM status stays 0. Every other ENODATA source (compute replay and spared quarantine) is likewise work a recovery dropped, and a poisoning ENODATA is still recorded. The VM_STATUS UAPI comment now names ENODATA beside ECANCELED. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 93bdf2867955e49e6765b69bdbb745f28248e165 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/completion.rs | 15 ++++++++++----- drivers/gpu/drm/asahi/g17/queue/render.rs | 4 ++-- include/uapi/drm/asahi_drm.h | 5 +++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 9e285b81c2cf2f..4d3d5e5a7904d2 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -563,13 +563,13 @@ impl Completion { (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) } - /// A non-poisoning ECANCELED stays out of the VM status, as the UAPI documents for work - /// another VM's recovery discarded: Honeykrisp reads any status error as a lost device. - /// Every other failure reaches the client. + /// A non-poisoning ECANCELED or ENODATA (work a recovery discarded or dropped) stays out of + /// the VM status, as the UAPI documents for work another VM's recovery discarded: + /// Honeykrisp reads any status error as a lost device. Every other failure reaches the client. fn report(&self, error: Error) { if self.poisons(error) { self.status.record(error); - } else if error != ECANCELED { + } else if !discarded(error) { self.status.report_failure(error); } } @@ -716,7 +716,7 @@ impl Completion { guard.quarantine(); } if self.spared() { - if error != ECANCELED { + if !discarded(error) { self.status.report_failure(error); } } else { @@ -727,6 +727,11 @@ impl Completion { } } +/// The errors of work a recovery discarded (ECANCELED) or dropped before it ran (ENODATA). +fn discarded(error: Error) -> bool { + error == ECANCELED || error == ENODATA +} + /// Collected under the device mutex and executed only after releasing it. /// Runtime queues establish the ownership witness before constructing `Retired` /// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 25995472b55287..a32a86f1f6976d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1659,8 +1659,8 @@ impl Pair { /// Firmware retired every group of the pair after a restart, but some were dropped /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, /// the ones that ran too: they consumed output the dropped ones never wrote. The work is - /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, - /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + /// innocent, so it fails without poisoning its VM: its fences carry ENODATA, which stays out + /// of the VM status, instead of waiting for the render timeout to fail it with ETIMEDOUT. fn fail_dropped( &mut self, host: &mut impl Host, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 418522205ee6f8..5641384b51bfee 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -447,8 +447,9 @@ enum drm_asahi_feature { * treating that fence as successful execution. Syncobj signal alone is not * an execution-success indication. A job cancelled (ECANCELED) because * another VM's recovery discarded it, an input did not signal in time, or - * its queue was closed, is not a failure of this VM: its fence carries the - * error, error stays 0, and its output is undefined. + * its queue was closed, or dropped (ENODATA) because a recovery could not + * run it again, is not a failure of this VM: its fence carries the error, + * error stays 0, and its output is undefined. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. From c7284af1c57be05319e77fa305313572d0057ffe Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 14:15:04 -0300 Subject: [PATCH 058/171] drm/asahi: g17: fail exhausted replays without poisoning their VM A pair whose groups a restart drops untouched has them republished, up to MAX_REPLAYS times. In a cascade of restarts another client causes, the same innocent groups are dropped on every restart, and the fourth drop failed them with EIO recorded on their VM: T3 Code lost its device in the 16-group spin repro on m4.119 ("12 groups retired without execution; replay limit reached, failing them"). replay::decide() only replays groups that are innocent, not blamed and never ran, so none of them can have caused the hang. Fail them like fail_dropped does: spared, with ENODATA, which stays out of the VM status. The replay limit still bounds how often they are republished. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 616a0678a847097b578ef094653a11549bdb1c99 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index a32a86f1f6976d..76447a297c5ec9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1636,13 +1636,10 @@ impl Pair { let packet = front.packet.clone(); failing = failing || !packet.completion.begin_replay(); if failing { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); - } - } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + // decide() replays only innocent groups that never ran: like `fail_dropped`, + // they fail without poisoning their VM. Restarts other VMs caused dropped them. + packet.completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; continue; From 1ccbcab1d23071584de310731d4b8321b8ef4100 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 14:15:04 -0300 Subject: [PATCH 059/171] drm/asahi: g17: keep a VM a recovery found guilty out of later spared work Each recovery classifies every VM it does not find guilty as spared. A hung client's VM that an earlier recovery found guilty therefore turned spared as soon as one later recovery blamed someone else. The next recovery that blamed it again then found its oldest work spared, took the blame as collateral to that recovery, found no guilty VM, and so did not treat the restart as leftover work of a failed VM: it counted toward the restart storm limit. In the 16-group spin repro on m4.119, gens 9 and 10 blamed glmark2 and quickshell, and the four correct blames on the spinning client after them all counted; the next restart reached the limit and stopped the GPU for everyone. Remember on the VM status that a recovery found it guilty, and keep such a VM's work guilty in every later recovery. A VM that failed only for its own client is still spared. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/status.rs] (cherry picked from commit 92f6396a4476ca5fbd1943353cb52f2c6cf11ce6 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 30 ++++++++++++++++++++--- drivers/gpu/drm/asahi/g17/status.rs | 35 ++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 8095a1e13da9f6..43636e3e051489 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -414,6 +414,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att queues.visit_owners(qid, &mut |status| dropped |= known(status)); } let leftover = count != 0 && dropped && guilty[..count].iter().flatten().all(|status| status.get() != 0); + guilty[..count].iter().flatten().for_each(|status| status.note_recovery_guilty()); let spared = queues.classify_work(&|status| { guilty[..count] .iter() @@ -423,7 +424,9 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att .filter(|known| Arc::ptr_eq(known, status)) .map(|_| Class::Guilty) }) - .unwrap_or(Class::Spared) + // A VM an earlier recovery found guilty stays guilty. Spared, its leftover work would + // make the next blame on it collateral, and that restart would count as a storm. + .unwrap_or(if status.recovery_guilty() { Class::Guilty } else { Class::Spared }) }); (cancelled & !(guilty_qids & !collateral) & spared, blame, leftover) } @@ -518,7 +521,7 @@ mod tests { } /// Two VMs, each owning one render pair: `a` queues 40/41, `b` queues 60/61. - struct Owners([(u8, Arc); 4]); + struct Owners([(u8, Arc); 4], [Option; 4]); impl Queues for Owners { fn render_scheduler_active(&self) -> bool { true } @@ -530,7 +533,12 @@ mod tests { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { self.0.iter().filter(|(own, _)| *own == qid).for_each(|(_, status)| visit(status)); } - fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128 { + for (index, (_, status)) in self.0.iter().enumerate() { + self.1[index] = Some(class(status)); + } + 0 + } fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } fn settle_blamed_render(&mut self, _qid: u8, _error: Error) {} fn settle_cancelled_renders(&mut self, _qids: u128) {} @@ -541,7 +549,7 @@ mod tests { fn owners() -> Owners { let a = Arc::pin_init(VmStatus::new(), GFP_KERNEL).unwrap(); let b = Arc::pin_init(VmStatus::new(), GFP_KERNEL).unwrap(); - Owners([(40, a.clone()), (41, a), (60, b.clone()), (61, b)]) + Owners([(40, a.clone()), (41, a), (60, b.clone()), (61, b)], [None; 4]) } fn leftover(queues: &mut Owners, exact_qid: u8) -> bool { @@ -560,6 +568,20 @@ mod tests { assert!(!leftover(&mut queues, 40)); } + #[test] + fn a_guilty_vm_is_not_spared_by_another_vms_recovery() { + let mut queues = owners(); + let entries = || Entries { exact_qids: 1u128 << 40, ..Entries::default() }; + // b failing for its own client does not make it guilty in a's recovery. + queues.0[2].1.record(EIO); + classify(&mut queues, entries(), Some(40), Attribution::Queues(3u128 << 40)); + assert_eq!(queues.1, [Some(Class::Guilty), Some(Class::Guilty), Some(Class::Spared), Some(Class::Spared)]); + // Once a recovery finds b guilty, a later recovery that blames a keeps b guilty. + classify(&mut queues, entries(), Some(60), Attribution::Queues(3u128 << 60)); + classify(&mut queues, entries(), Some(40), Attribution::Queues(3u128 << 40)); + assert_eq!(queues.1, [Some(Class::Guilty); 4]); + } + #[test] fn unknown_25g83_keys_still_preserve_firmware_blame() { let mut host = Halted::new(false); diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index a065c5030b1b8d..75df8ad0480ae7 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -5,7 +5,7 @@ use crate::debug::*; use crate::gem; use core::sync::atomic::{AtomicBool, AtomicI32, Ordering}; -use kernel::{new_mutex, prelude::*, sync::Mutex}; +use kernel::{new_mutex, prelude::*, str::BStr, sync::Mutex, task::Pid}; const DEBUG_CLASS: DebugFlags = DebugFlags::Errors; @@ -22,6 +22,11 @@ pub(crate) struct VmStatus { client_closed: AtomicBool, /// A failure of this VM has been logged as a warning. warned: AtomicBool, + /// A GPU recovery found this VM's work guilty. + recovery_guilty: AtomicBool, + /// The process that created the VM, named in the failure report. + tgid: Pid, + comm: [u8; 16], /// The client's mapped status word, retained with its object only while the /// client's VM handle exists; released by `release_mirror` at VM close. #[pin] @@ -38,12 +43,23 @@ struct Mirror { unsafe impl Send for Mirror {} impl VmStatus { + /// Called from the creating client's ioctl: the VM records who it belongs to. pub(crate) fn new() -> impl PinInit { + let task = kernel::current!(); + // SAFETY: `comm` is a fixed array inside the live current task. A concurrent rename by + // another thread of the process may tear the copy, as with C's unlocked `current->comm`. + let comm: [u8; 16] = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*task.as_ptr()).comm).cast()) + }; + let tgid = task.tgid(); pin_init!(Self { error: AtomicI32::new(0), reported_error: AtomicI32::new(0), client_closed: AtomicBool::new(false), warned: AtomicBool::new(false), + recovery_guilty: AtomicBool::new(false), + tgid, + comm, mirror <- new_mutex!(None, "Asahi VM status mirror"), }) } @@ -52,6 +68,16 @@ impl VmStatus { self.error.load(Ordering::Acquire) } + pub(crate) fn note_recovery_guilty(&self) { + self.recovery_guilty.store(true, Ordering::Release); + } + + /// A GPU recovery found this VM guilty: it hung or faulted the GPU, unlike a VM that + /// failed only for its own client. + pub(crate) fn recovery_guilty(&self) -> bool { + self.recovery_guilty.load(Ordering::Acquire) + } + /// Record permanent admission failure and report it to the client. #[track_caller] pub(crate) fn record(&self, error: Error) { @@ -98,11 +124,14 @@ impl VmStatus { ); } } else if !self.warned.swap(true, Ordering::AcqRel) { + let len = self.comm.iter().position(|&b| b == 0).unwrap_or(self.comm.len()); pr_warn!( - "GPU work of a client failed ({:?}, reported at {}:{}); the client sees a lost device\n", + "GPU work of a client failed ({:?}, reported at {}:{}, client {}[{}]); the client sees a lost device\n", error, file, - at.line() + at.line(), + BStr::from_bytes(&self.comm[..len]), + self.tgid ); } let error = match first { From a6dc07a08dae1cb1b035f430a26fba7a53cb4760 Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Tue, 3 Nov 2026 15:10:23 -0300 Subject: [PATCH 060/171] drm/asahi: g17: blame a guilty VM's running render over a started one render_culprit moved a keyless timeout's blame only when the blamed pair's oldest render had not started. A bystander's render whose tiler started and whose fragment pass then waits behind the spinning one is just as starved. On m4.120, in the 16-group spin repro, 15 of the 16 recoveries landed on the spinning client, 3 of them through render_culprit. In the 16th, the firmware blamed T3 Code, whose render had its tiler start written and no fragment end, while the spinner's last group ran. T3 Code lost its device. When a VM an earlier recovery found guilty still has a render that started and did not finish, blame it whether or not the blamed render had started: that VM has already failed, and its queued groups end with the restarts they cause. The one-running-VM rule, which can blame a VM that has not failed, still needs a blamed render that never started. Known limit: if a second VM hangs while a guilty VM still has started renders, each restart blames the guilty VM until those renders are gone, and those restarts do not count toward the restart storm limit. The evidence here cannot tell that second hang from a starved bystander, which is the case this change is for. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/runtime/render/recovery.rs] (cherry picked from commit 6295b0256c1c1de0d02f056548b62c290c93b3bd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 9 +- .../drm/asahi/g17/runtime/render/recovery.rs | 92 +++++++++++++++---- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 43636e3e051489..81e2d1d81d6bb4 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -94,10 +94,11 @@ pub(super) enum Class { pub(super) trait Queues { fn render_scheduler_active(&self) -> bool; fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; - /// Read while firmware is halted, like `render_pass_started`. When the oldest published - /// render of the pair holding `blamed` had not started, the QIDs of every pair of the one VM - /// with a render that started and did not finish. None if no such VM, several, or any other - /// VM holds compute work, which these records cannot see. + /// Read while firmware is halted, like `render_pass_started`. The QIDs of every pair of a VM + /// an earlier recovery found guilty with a render that started and did not finish. Without + /// one, and only when the oldest published render of the pair holding `blamed` had not + /// started, those of the one VM with such a render. None if no such VM, several, or any + /// other VM holds compute work, which these records cannot see. fn render_culprit(&self, _blamed: u8) -> Option { None } /// Copy every retained queue's live producer to its recovery target, then /// issue a full barrier. A failed graph must not skip subsequent queues. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs index e597f63f15b659..02ece8cf4025b7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -32,27 +32,18 @@ impl Registry { /// See `recovery::Queues::render_culprit`. pub(in crate::g17::runtime) fn render_culprit(&self, blamed: u8) -> Option { let pairs = || self.render.entries.iter().flatten().filter_map(|entry| entry.pair.as_ref()); - if pairs().find(|pair| pair.qids().contains(&blamed))?.front_untouched() != Some(true) { - return None; - } - let mut culprit: Option<&Arc> = None; - let mut qids = 0u128; - for pair in pairs() { - let Some(owner) = pair.running_owner() else { - continue; - }; - if culprit.is_some_and(|known| !Arc::ptr_eq(known, owner)) { - return None; + let untouched = pairs().find(|pair| pair.qids().contains(&blamed))?.front_untouched() == Some(true); + let running = pairs().filter_map(|pair| { + let qids = pair.qids().into_iter().fold(0, |mask, qid| mask | 1u128 << qid); + Some((qids, pair.running_owner()?)) + }); + culprit(running, untouched, |culprit| { + let mut foreign = false; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.visit_owners(&mut |owner| foreign |= !Arc::ptr_eq(owner, culprit)); } - culprit = Some(owner); - qids |= pair.qids().into_iter().fold(0, |mask, qid| mask | 1u128 << qid); - } - let culprit = culprit?; - let mut foreign = false; - for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { - queue.visit_owners(&mut |owner| foreign |= !Arc::ptr_eq(owner, culprit)); - } - (!foreign).then_some(qids) + foreign + }) } pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { @@ -67,3 +58,64 @@ impl Registry { failure } } + +/// Picks the QIDs to blame from each pair's running owner. A VM an earlier recovery found guilty +/// comes first, even when the blamed render had started: its groups queued before that keep +/// hanging the GPU one per restart, while bystanders' renders whose tiler started wait behind +/// them, and that recovery failed its interrupted work. Otherwise the blamed render must be +/// `untouched`, exactly one VM may be running, and `foreign` must find no other VM's compute work. +fn culprit<'a>( + running: impl Iterator)>, + untouched: bool, + foreign: impl FnOnce(&Arc) -> bool, +) -> Option { + let (mut guilty, mut qids, mut several) = (0u128, 0u128, false); + let mut culprit: Option<&Arc> = None; + for (pair, owner) in running { + if owner.recovery_guilty() { + guilty |= pair; + } + several |= culprit.is_some_and(|known| !Arc::ptr_eq(known, owner)); + culprit = Some(owner); + qids |= pair; + } + if guilty != 0 { + return Some(guilty); + } + let culprit = culprit.filter(|_| untouched && !several)?; + (!foreign(culprit)).then_some(qids) +} + +#[kernel::macros::kunit_tests(asahi_g17_render_culprit)] +mod tests { + use super::*; + + fn vm() -> Arc { + Arc::pin_init(VmStatus::new(), GFP_KERNEL).unwrap() + } + + #[test] + fn a_guilty_vm_still_running_is_blamed_before_running_bystanders() { + let (spinner, bystander) = (vm(), vm()); + let running = || [(0b11, &bystander), (0b1100, &spinner)].into_iter(); + // Two VMs running and no earlier guilt: the firmware's blame stands, even when one of + // them failed only for its own client. + bystander.record(EIO); + assert_eq!(culprit(running(), true, |_| false), None); + spinner.note_recovery_guilty(); + assert_eq!(culprit(running(), true, |_| panic!("compute is not consulted")), Some(0b1100)); + // The bystander's blamed render whose tiler started is starved too. + assert_eq!(culprit(running(), false, |_| panic!("compute is not consulted")), Some(0b1100)); + } + + #[test] + fn one_running_vm_is_blamed_only_without_foreign_compute() { + let (a, b) = (vm(), vm()); + assert_eq!(culprit([(0b11, &a), (0b1100, &a)].into_iter(), true, |_| false), Some(0b1111)); + // A blamed render that had started may be the hang itself. + assert_eq!(culprit([(0b11, &a), (0b1100, &a)].into_iter(), false, |_| false), None); + assert_eq!(culprit([(0b11, &a)].into_iter(), true, |_| true), None); + assert_eq!(culprit([(0b11, &a), (0b1100, &b)].into_iter(), true, |_| false), None); + assert_eq!(culprit(core::iter::empty(), true, |_| false), None); + } +} From e7fa52b4e7f07a66f4024860ffd2755e1d820249 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:14:48 -0800 Subject: [PATCH 061/171] drm/asahi: g17: M5: describe the M5 boards and refuse those without bare-metal GPU tables Add the M5 Pro/Max configuration (T6050) and its OF match, selected by the ID registers. A G17 configuration now names its firmware protocol, and the 26A428/25G83 graph tables become optional, since the M5 graph carries its own. Probe refuses an M5 board that has no bare-metal GPU page tables, and the M5 boot entry point stays closed until one does. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..15e1b3e4fa44 (3 commits); SPTM/APIF removed; adapted to integration's g17::Config, ProbeConfig and Firmware::Identified; original 6613f5ab7b7e, 1a021061f226, 15e1b3e4fa44 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser [Eduard Moldovan: integration/aurora's 7.2.11 OF table is kept, with the two T6050 entries added] (cherry picked from commit a2797fa0626e879263ce0febd9dfa3a426629a26 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 21 ++++++ drivers/gpu/drm/asahi/g17/initdata.rs | 8 +-- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 4 +- drivers/gpu/drm/asahi/g17/mod.rs | 72 +++++++++++++++++--- drivers/gpu/drm/asahi/hw/mod.rs | 1 + drivers/gpu/drm/asahi/hw/t6040.rs | 9 +-- drivers/gpu/drm/asahi/hw/t6041.rs | 9 +-- drivers/gpu/drm/asahi/hw/t6050.rs | 53 ++++++++++++++ drivers/gpu/drm/asahi/hw/t8140.rs | 9 +-- 9 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 drivers/gpu/drm/asahi/hw/t6050.rs diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index b11ca814203a99..fb0058fae4007e 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -152,6 +152,8 @@ impl drm::driver::Driver for AsahiDrmDriver { pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), + /// M5 Pro, Max and Ultra: the ID registers select the configuration. + M5, M3(crate::m3::Chip), } @@ -173,6 +175,14 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t6041"), ProbeConfig::G17(&hw::t6041::CONFIG) ), + ( + of::DeviceId::new(c"gpu,t6050"), + ProbeConfig::M5 + ), + ( + of::DeviceId::new(c"apple,agx-t6050"), + ProbeConfig::M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -423,6 +433,17 @@ impl AsahiPlatformDriver { }) } + fn probe_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + let cfg = &hw::t6050::CONFIG; + if let g17::FirmwareProfile::M5NoBareMetalTables = cfg.profile { + dev_err!(pdev.as_ref(), "M5 Pro/Max bare-metal GPU tables not yet available\n"); + return Err(ENODEV); + } + Self::probe_g17(pdev, cfg) + } + fn probe_g17<'bound>( pdev: &'bound platform::Device>, cfg: &'static g17::Config, diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 56e7335a23bd24..7e77feb2d3ee7e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -849,7 +849,7 @@ impl InitData { active_cores: u32, variant: &super::fw::Variant, ) -> Result { - self.pm_metrics_va = cfg.pm_metrics.va; + self.pm_metrics_va = cfg.pm_metrics()?.va; self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { @@ -1137,7 +1137,7 @@ impl InitData { self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics()?.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); @@ -1178,7 +1178,7 @@ impl InitData { windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: cfg.perf()?, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1186,7 +1186,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); Ok(()) })?; self.object_mut(self.cluster)? diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index f7b1952685761a..c0485c4932e48f 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -130,7 +130,7 @@ impl InitData { calibration.max_power_mw, &calibration.leakage, ), - (None, Some(chip)) => (cfg.perf, chip.max_power_mw, &chip.leakage), + (None, Some(chip)) => (cfg.perf()?, chip.max_power_mw, &chip.leakage), _ => return Err(EINVAL), }; let hw_args = abi::HwDataArgs { @@ -165,7 +165,7 @@ impl InitData { })?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(cfg.region_c()?); region.max_power_mw = max_power_mw; Ok(()) })?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index d615cf3fa59085..a0b130a3f4e4c7 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -157,6 +157,40 @@ pub(crate) struct FixedBuffer { pub(crate) size: usize, } +/// Firmware protocol of a G17 GPU. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum FirmwareProfile { + /// The firmware publishes its kernel roots and the driver owns the page tables (A18 Pro, + /// M4 Pro). + PublishedRoots, + /// An M5 board without bare-metal GPU tables yet: probe refuses it. + M5NoBareMetalTables, +} + +impl Config { + /// The fixed parameter-buffer metrics buffer of a 26A428 or 25G83 graph. + pub(crate) fn pm_metrics(&self) -> Result<&FixedBuffer> { + self.pm_metrics.as_ref().ok_or(EINVAL) + } + + /// The performance tables of a 26A428 or 25G83 graph. + pub(crate) fn perf(&self) -> Result<&'static fw::initdata::PerfConfig> { + self.perf.ok_or(EINVAL) + } + + /// Region C of a 26A428 or 25G83 graph. + pub(crate) fn region_c(&self) -> Result<&'static fw::initdata::RegionCConfig> { + self.region_c.ok_or(EINVAL) + } +} + +impl FirmwareProfile { + /// The M5 firmware's protocol (26A428 InitData graph, queues and submission). + pub(crate) const fn is_m5(self) -> bool { + matches!(self, Self::M5NoBareMetalTables) + } +} + /// Static configuration of a SoC with a G17 GPU. pub(crate) struct Config { /// SoC identifier. @@ -172,14 +206,18 @@ pub(crate) struct Config { pub(crate) secondary_root_offset: Option, /// Where the coprocessors' crash log buffers live. pub(crate) crash_buffers: CrashBuffers, - /// Parameter buffer page metrics shared with the firmware. - pub(crate) pm_metrics: FixedBuffer, + /// Firmware protocol. + pub(crate) profile: FirmwareProfile, + /// Parameter buffer page metrics shared with the firmware; `None` where the firmware graph + /// places its own (M5). + pub(crate) pm_metrics: Option, /// Register windows the firmware maps through the hardware data. pub(crate) windows: &'static [fw::initdata::RegisterWindow], /// Register-map slots that are reserved but have no window. pub(crate) reserved_slots: &'static [usize], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Per-SoC parts of the performance tables; `None` where the firmware graph has its own + /// (M5, [`fw::m5::initdata`]). + pub(crate) perf: Option<&'static fw::initdata::PerfConfig>, /// Number of operating points the device tree lists. pub(crate) opps: usize, /// UAT policy where it differs from the A18 Pro's. @@ -191,8 +229,8 @@ pub(crate) struct Config { /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware /// starts (A18 Pro). The M4 Pro has only run with neither written. pub(crate) fabric_bridge: bool, - /// The SoC's values of region C. - pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// The SoC's values of region C; `None` where the firmware graph has its own (M5). + pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the @@ -1394,6 +1432,19 @@ impl Gpu { Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) } + /// Starts the M5 firmware. It runs only on page tables the driver owns, which no M5 board + /// has yet. + fn boot_m5( + _pdev: &platform::Device>, + _dev: &AsahiDrmDevice, + _cfg: &'static Config, + _regs: regs::G17Resources, + _geometry: mmu::UatGeometry, + _shared: &Arc, + ) -> Result> { + Err(ENODEV) + } + /// Starts both coprocessors and initializes their firmware. fn boot( pdev: &platform::Device>, @@ -1403,6 +1454,9 @@ impl Gpu { geometry: mmu::UatGeometry, shared: &Arc, ) -> Result> { + if cfg.profile.is_m5() { + return Self::boot_m5(pdev, dev, cfg, regs, geometry, shared); + } let uat = Arc::new( mmu::Uat::new( dev.into(), @@ -1421,8 +1475,8 @@ impl Gpu { let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), - Placement::At(cfg.pm_metrics.va), - cfg.pm_metrics.size, + Placement::At(cfg.pm_metrics()?.va), + cfg.pm_metrics()?.size, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; @@ -1744,7 +1798,7 @@ impl gpu::Gpu for Gpu { .with_status()?; let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; let ranges = - hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics()?.va).ok_or(EINVAL)?; let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; for range in ranges { reserved.push(range, GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index c54ffb5c208fe8..ebb9a423378463 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -19,6 +19,7 @@ mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; +pub(crate) mod t6050; /// GPU generation enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index d531e20c7e9f94..29aff6ef4b5f54 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -33,13 +33,14 @@ pub(crate) const CONFIG: Config = Config { crash_buffers: CrashBuffers::FirmwareData, // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &[], - perf: &PERF, + perf: Some(&PERF), opps: OPPS, // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The // rest is the policy the M4 Pro has run with; none of it was tested otherwise. @@ -51,7 +52,7 @@ pub(crate) const CONFIG: Config = Config { handoff_slot: 0xffff_ffff, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: None, adopt_root_entries: false, fabric_bridge: false, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 8be466f7842ab9..7938ee0248c4bd 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -24,13 +24,14 @@ pub(crate) const CONFIG: Config = Config { secondary_root_offset: None, crash_buffers: CrashBuffers::FirmwareData, // Unverified placement; use the M4 Pro layout. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: UatQuirks { reserved_tables: true, @@ -40,7 +41,7 @@ pub(crate) const CONFIG: Config = Config { handoff_slot: 0xffff_ffff, native_gpu_permissions: true, }, - region_c: ®ION_C, + region_c: Some(®ION_C), chip: Some(&CHIP), adopt_root_entries: false, fabric_bridge: false, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs new file mode 100644 index 00000000000000..dde03d20fcdac6 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 Pro GPU firmware and address-space configuration. +//! +//! The M5 firmware builds its HW data, performance tables and region C in its own graph +//! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. +use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; + +pub(crate) const CONFIG: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Unknown until the board has bare-metal page tables. + secondary_root_offset: None, + // The coprocessors request a crash log buffer at these addresses (primary first) when they + // start and hand its contents back when they crash; outside the canonical range of dynamic + // objects. + crash_buffers: CrashBuffers::Fixed([ + FixedBuffer { va: 0x100_025b_4000, size: 0x8000 }, + FixedBuffer { va: 0x100_0267_0000, size: 0x8000 }, + ]), + profile: FirmwareProfile::M5NoBareMetalTables, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: None, + chip: None, + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // No loaded image is admitted: the board has no bare-metal page tables yet. + firmware: crate::g17::fw::variant::Firmware::Identified(&[]), + // Two clusters of ten cores. + core_layout: CoreLayout { + cores: Some((2, 20)), + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::TWO_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 93d1c86e3425d2..6c2c2a173630e0 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -39,18 +39,19 @@ pub(crate) const CONFIG: Config = Config { ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. - pm_metrics: FixedBuffer { + profile: crate::g17::FirmwareProfile::PublishedRoots, + pm_metrics: Some(FixedBuffer { va: 0x10_0008_0000, size: 0x8000, - }, + }), recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_REGISTER_SLOTS, - perf: &PERF, + perf: Some(&PERF), opps: OPPS, uat_quirks: crate::mmu::UatQuirks::DEFAULT, - region_c: &crate::g17::fw::initdata::REGION_C_26A428, + region_c: Some(&crate::g17::fw::initdata::REGION_C_26A428), chip: None, adopt_root_entries: true, fabric_bridge: true, From ba77be3edcd143eb11ae7f17f2f784b7d0705b26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:14:57 -0800 Subject: [PATCH 062/171] drm/asahi: g17: M5: give M5 VMs sparse tiles, render pool mappings and context pins Add the UAT pieces the M5 queues use: sparse-tile binds and their zeroed and per-VM sink tiles, render pool mappings that move with a pool identity, the metrics alias, execution-context pins and the M5 dynamic kernel ranges. On M5, a healthy last-job release skips the VM drain. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..692d42fb35d7 (5 commits); SPTM/APIF removed; adapted to integration's mmu.rs (7.3 GPUVM, UatQuirks, ExecutionRoot, SG leases); comments and names reworded; M5 paths restored where the first rewrite lost them: one prepare_bind_object, with the sparse-tile bind; the G15/G16 adopted UAT (Uat::new_adopted_inner) sets m5: false, for integration's own config (defconfig + asahi.config), which builds G16 beside G17 and failed with E0063 without it; original 6613f5ab7b7e, e25ba8a6fb2b, 1a021061f226, f49e91b3f364, 692d42fb35d7 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit a9b2acd57d0327a49562309be50590524972c7ff in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/file.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/object.rs | 5 +- drivers/gpu/drm/asahi/mmu.rs | 103 ++++++++++++++++++++- drivers/gpu/drm/asahi/mmu/bind.rs | 16 +++- drivers/gpu/drm/asahi/mmu/layout.rs | 9 ++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 113 ++++++++++++++++++++++- drivers/gpu/drm/asahi/mmu/residency.rs | 8 ++ drivers/gpu/drm/asahi/mmu/sparse_tile.rs | 49 ++++++++++ drivers/gpu/drm/asahi/mmu/table.rs | 23 +++++ drivers/gpu/drm/asahi/pgtable.rs | 2 +- 11 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/asahi/mmu/layout.rs create mode 100644 drivers/gpu/drm/asahi/mmu/sparse_tile.rs create mode 100644 drivers/gpu/drm/asahi/mmu/table.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index b135a558eeee32..dcc5fdc590fce8 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -956,6 +956,7 @@ impl File { op.offset, prot, single_page, + false, )?)?; } vm.reserve_deferred_user_unmaps(unmaps)?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a0b130a3f4e4c7..dda1f1564fdf23 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -21,7 +21,7 @@ //! nests inside nothing. Capacity waits block on an availability domain signalled by the //! release that provides the capacity, never on a sleep. -mod availability; +pub(crate) mod availability; mod buffer; mod calibration; mod channel; diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 87fdb0e8a62ad1..716e5452d4bae7 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -338,8 +338,7 @@ impl Allocator<'_> { prot: mmu::Prot, cpu: CpuMap, ) -> Result { - let mut range = self.uat.geometry().kernel_range(); - range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + let range = self.uat.kernel_dynamic_range(); KernelObject::new( self.dev, self.uat.kernel_vm(), @@ -362,7 +361,7 @@ impl Allocator<'_> { KernelObject::new( self.dev, self.uat.kernel_lower_vm(), - Placement::In(crate::hw::t8140::dynamic::LOWER, align), + Placement::In(self.uat.kernel_lower_dynamic_range(), align), size, prot, cpu, diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ca224008dc573a..a2820df7d314cc 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -112,14 +112,18 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod layout; + mod lifetime; mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; mod shared; +mod sparse_tile; +mod table; pub(crate) use bind::PreparedUserBindBatch; -pub(crate) use lifetime::VmJobGuard; +pub(crate) use lifetime::{RenderPoolMappings, VmJobGuard}; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region @@ -374,6 +378,17 @@ struct VmInner { uat_inner: Arc, binding: Arc>, id: u64, + /// Per-VM sparse tiles (DRM_ASAHI_BIND_SPARSE_TILE): [read-only zero, + /// writable sink], allocated on first use. + sparse_tiles: [Option>; 2], +} + +/// GPUVA filled from a kernel sparse tile instead of its GEM's first page. +pub(crate) const GPUVA_SPARSE_TILE: u32 = kernel::bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS; + +/// Whether a GPUVA carries [`GPUVA_SPARSE_TILE`]. +fn is_sparse_tile(flags: GpuVaFlags) -> bool { + u32::from(flags) & GPUVA_SPARSE_TILE != 0 } /// Exec-locked access to the [`VmInner`] of a GPUVM. @@ -583,6 +598,19 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + if is_sparse_tile(op.flags()) { + // Kernel tile fill: the GEM only anchors the GPUVA. + let result = if one_page { + self.map_sparse_tile(iova..iova + left as u64, ctx.prot) + } else { + Err(EINVAL) + }; + if let Err(error) = result { + self.rollback_failed_map(map_addr, map_length as u64); + return Err(error); + } + left = 0; + } // Bind preparation retained the table's backing before taking the VM reservation. // Never acquire the object reservation from inside the GPUVM map callback. let segments = op.vm_bo().data().segments.lock(); @@ -764,6 +792,23 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Fill `range` from the kernel sparse tile for `prot`: the global zero + /// tile for read-only fills, this VM's own sink for writable ones (never + /// shared, so discarded writes cannot be read by another VM). The caller + /// holds the mapping mutation guard. + fn map_sparse_tile(&mut self, range: Range, prot: Prot) -> Result { + let kind = usize::from(prot != PROT_GPU_SHARED_RO); + let tile = match &self.sparse_tiles[kind] { + Some(tile) => tile.clone(), + None => { + let tile = sparse_tile::SparseTile::new()?; + self.sparse_tiles[kind] = Some(tile.clone()); + tile + } + }; + self.page_table.map_tiled(range, &tile, prot) + } + /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { mod_dev_dbg!( @@ -1036,6 +1081,8 @@ pub(crate) struct Vm { dev: ARef, id: u64, mapping_epoch: Option>, + /// Created by an M5 GPU's UAT ([`Uat::is_m5`]). + m5: bool, bind_sequence: Arc, lifetime: Option>, residency: Option>, @@ -1058,6 +1105,14 @@ pub(crate) struct ExecutionContext { vm: Vm, } +/// One published command's claim on its context's hardware identity, held +/// until the command retires exactly. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ContextPin { + pub(crate) id: u8, + pub(crate) generation: u64, +} + struct ExecutionRoot { released: AtomicBool, inner: Arc, @@ -1074,6 +1129,17 @@ impl ExecutionContext { pub(crate) fn generation(&self) -> u8 { self.root.generation } + /// Claim the context's hardware identity for one published command until + /// it retires exactly. + pub(crate) fn pin(&self) -> Result { + Ok(ContextPin { + id: self.root.id, + generation: self.root.generation.into(), + }) + } + + pub(crate) fn unpin(&self, _pin: ContextPin) {} + pub(crate) fn vm(&self) -> &Vm { &self.vm } @@ -1935,6 +2001,8 @@ pub(crate) struct Uat { kernel_vm: Vm, kernel_lower_vm: Vm, + /// The GPU behind this UAT runs the M5 firmware's protocol ([`Uat::is_m5`]). + m5: bool, } impl Handoff { @@ -2174,6 +2242,7 @@ impl Vm { uat_inner, binding: binding.clone(), id, + sparse_tiles: [None, None], }, )?; let gpuvm = ARef::from(&*unique_gpuvm); @@ -2185,6 +2254,7 @@ impl Vm { id, gpuvm, mapping_epoch, + m5: false, bind_sequence, lifetime: None, residency: None, @@ -3202,6 +3272,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, @@ -3498,7 +3569,7 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { self.inner.check_m4()?; - let vm = Vm::new( + let mut vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, @@ -3506,10 +3577,37 @@ impl Uat { None, id, )?; + vm.m5 = self.m5; self.inner.check_m4()?; Ok(vm) } + /// The GPU behind this UAT runs the M5 firmware's protocol. + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + + /// Keep dynamic objects outside the M5 boot graph, its unmapped sentinel, + /// and the firmware register aperture. Client mappings have their own VA. + pub(crate) fn kernel_dynamic_range(&self) -> Range { + let mut range = self.geometry.kernel_range(); + range.start += if self.is_m5() { + 0x2_0000_0000 + } else { + crate::hw::t8140::dynamic::KERNEL_OFFSET + }; + range + } + + /// Canonical lower objects are aliased into every owning client VM. + /// Sparse VA reservation has no physical-memory cost. + pub(crate) fn kernel_lower_dynamic_range(&self) -> Range { + if self.is_m5() { + layout::CANONICAL_RANGE + } else { + crate::hw::t8140::dynamic::LOWER + } + } /// Returns the address space geometry of this UAT. pub(crate) fn geometry(&self) -> UatGeometry { self.geometry @@ -3761,6 +3859,7 @@ impl Uat { geometry, kernel_vm, kernel_lower_vm, + m5: false, inner, slots: slotalloc::SlotAllocator::new( UAT_USER_CTX as u32, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index acce5677b9f923..a792cdd1bec08b 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -15,6 +15,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + sparse_tile: bool, context: context::PreparedBinding, _residency: Option, _sg_lease: kernel::drm::gem::shmem::SGLease, @@ -59,6 +60,9 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state before taking the GPUVM exec lock or changing any leaf. + /// With `sparse_tile` (DRM_ASAHI_BIND_SPARSE_TILE) the range is filled from a kernel + /// sparse tile and `gem` only anchors it; it requires `single_page`. + #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -67,7 +71,11 @@ impl Vm { offset: u64, prot: Prot, single_page: bool, + sparse_tile: bool, ) -> Result { + if sparse_tile && !single_page { + return Err(EINVAL); + } gem::validate_vm_binding(gem, self)?; let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; @@ -103,6 +111,7 @@ impl Vm { size, offset, single_page, + sparse_tile, context, }) } @@ -138,7 +147,12 @@ impl Vm { let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { - let (flags, repeat) = if map.single_page { + let (flags, repeat) = if map.sparse_tile { + ( + GpuVaFlags::from(GpuVaFlag::REPEAT) | GpuVaFlags::from(super::GPUVA_SPARSE_TILE), + UAT_PGSZ as u32, + ) + } else if map.single_page { (GpuVaFlag::REPEAT.into(), UAT_PGSZ as u32) } else { (GpuVaFlags::empty(), 0) diff --git a/drivers/gpu/drm/asahi/mmu/layout.rs b/drivers/gpu/drm/asahi/mmu/layout.rs new file mode 100644 index 00000000000000..66c49f1189a308 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/layout.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Fixed GPU VA arenas of the M5 lower address space. + +use core::ops::Range; + +/// Canonical queue backing has its own sparse arena. It must not overlap +/// the client-private arena or retain the A18 Pro's sub-2-GiB limit. +pub(super) const CANONICAL_RANGE: Range = 0x80_0000_0000..0x90_0000_0000; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 66d507c2d76949..07155e7881e9fc 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -18,6 +18,7 @@ const ADMISSION_WAIT_MS: u32 = 1000; pub(super) struct DriverMappings { mappings: KVec, reserved: KVec>, + metrics: Option<(u64, KernelMapping)>, compute: Option, render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], } @@ -28,11 +29,15 @@ struct PoolMappings { mappings: KVec, } +/// A graph's pool mappings moved out of a VM slot (see +/// Vm::take_render_pool_mappings). Dropping it unmaps them. +pub(crate) struct RenderPoolMappings(PoolMappings); + impl DriverMappings { fn ranges(&self) -> impl Iterator> + '_ { Iterator::chain( Iterator::chain( - self.mappings.iter(), + Iterator::chain(self.mappings.iter(), self.metrics.iter().map(|(_, mapping)| mapping)), Iterator::chain( self.compute.iter().flat_map(|pool| pool.mappings.iter()), self.render @@ -152,12 +157,48 @@ impl Vm { *driver = Some(DriverMappings { mappings, reserved, + metrics: None, compute: None, render: core::array::from_fn(|_| None), }); Ok(()) } + /// Cache a single metrics backing per VM. Mapping allocation runs outside + /// the driver lock; a racing equivalent installation reuses the winner. + pub(crate) fn metrics_alias( + &self, + key: u64, + make: impl FnOnce() -> Result, + ) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + { + let driver = lifetime.driver.lock(); + if let Some((id, mapping)) = driver.as_ref().and_then(|d| d.metrics.as_ref()) { + return if *id == key { Ok(mapping.iova()) } else { Err(EINVAL) }; + } + } + let mapping = make()?; + let mut driver = lifetime.driver.lock(); + if let Some((id, existing)) = driver.as_ref().and_then(|d| d.metrics.as_ref()) { + let result = if *id == key { Ok(existing.iova()) } else { Err(EINVAL) }; + drop(driver); + drop(mapping); + return result; + } + let driver = driver.get_or_insert_with(|| DriverMappings { + mappings: KVec::new(), + reserved: KVec::new(), + metrics: None, + compute: None, + render: core::array::from_fn(|_| None), + }); + let address = mapping.iova(); + driver.metrics = Some((key, mapping)); + Ok(address) + } + + /// Reserves the complete append before moving aliases into VM ownership. pub(crate) fn append_driver_mappings(&self, mappings: KVec) -> Result { let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; @@ -170,6 +211,51 @@ impl Vm { Ok(()) } + /// Move a drained graph's pool mappings for `slot` out of this VM + /// (so another graph of the same VM can install there); `put` returns them + /// if that install fails. Only the exact owner identity is taken. + pub(crate) fn take_render_pool_mappings(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + let lifetime = self.lifetime.as_ref()?; + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver)? + .render + .get_mut(usize::from(slot))? + .take_if(|pool| pool.id == id) + .map(RenderPoolMappings) + } + pub(crate) fn put_render_pool_mappings(&self, slot: u8, mappings: RenderPoolMappings) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(mappings.0); + Ok(()) + } + + /// The graph identity whose pool mappings occupy `slot` in this + /// VM (0 when free). A second graph cannot install there until it leaves. + pub(crate) fn render_pool_owner(&self, slot: u8) -> u64 { + self.lifetime + .as_ref() + .and_then(|lifetime| { + lifetime + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .map(|pool| pool.id) + }) + .unwrap_or(0) + } + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { if id == 0 { return None; @@ -652,6 +738,31 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + // Healthy last-job release need not close mapping/job admission when + // there is no accepted cleanup to drain. Keep every recovery, close, + // in-progress commit and failed-cleanup path on the existing drain. + // M5 only: other G17 GPUs keep the drain on every last release. + if self.vm.m5 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed + && state.commits == 0 + && state.close_ranges.is_none() + && state.unmaps.is_empty() + && state.objects.is_empty() + && self.vm.status.as_ref().is_some_and(|status| { + status.get() == 0 && status.reported_error() == 0 + }) + { + drop(state); + // The C deferred list is independent of the queues above. Its + // GEM puts may sleep/reenter; run after unlocking, while this + // job guard still retains the VM and its residency lease. + self.vm.bo_deferred_cleanup(); + return; + } state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 49c2241bc399fa..4fb1a7b9f324e4 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -616,6 +616,14 @@ impl Vm { inner.page_table.discard_partial_map(map.addr..end)?; inner.tlbi_contexts_naming_root(map.addr, map.range as usize); let prot = map.inner.prot.ok_or(EINVAL)?; + if super::is_sparse_tile(map.flags) { + // A tile fill has no GEM pages to restore: reinstall it. + let _change = inner.mapping_mutation(); + inner.map_sparse_tile(map.addr..end, prot)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + return Ok(()); + } let repeat = map.flags.contains(GpuVaFlag::REPEAT); let mut skip = map .offset diff --git a/drivers/gpu/drm/asahi/mmu/sparse_tile.rs b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs new file mode 100644 index 00000000000000..55b3807e9af9f1 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Kernel-owned scratch for sparse fills (DRM_ASAHI_BIND_SPARSE_TILE). + +use kernel::{prelude::*, sync::Arc}; + +/// A zeroed, physically contiguous kernel allocation. The page tables repeat +/// its first page across a filled range, as a SINGLE_PAGE bind repeats its +/// GEM's first page. +pub(super) struct SparseTile { + page: *mut kernel::bindings::page, + physical: u64, +} + +// SAFETY: The pages are exclusively owned until Drop; only their physical +// address is shared, and it is immutable. +unsafe impl Send for SparseTile {} +// SAFETY: See Send. No interior mutability. +unsafe impl Sync for SparseTile {} + +impl SparseTile { + /// 64 pages. The page tables read only the first. + const ORDER: u32 = 6; + + pub(super) fn new() -> Result> { + // SAFETY: A plain zeroed higher-order page allocation. + let page = unsafe { kernel::bindings::alloc_pages( + kernel::bindings::GFP_KERNEL | kernel::bindings::__GFP_ZERO, Self::ORDER) }; + if page.is_null() { return Err(ENOMEM); } + // SAFETY: page is a live allocation returned above. + let physical = unsafe { kernel::bindings::page_to_phys(page) } as u64; + let tile = Arc::new(Self { page, physical }, GFP_KERNEL); + if tile.is_err() { + // SAFETY: The allocation above was not published anywhere. + unsafe { kernel::bindings::__free_pages(page, Self::ORDER) }; + } + Ok(tile?) + } + + pub(super) fn physical(&self) -> u64 { self.physical } +} + +impl Drop for SparseTile { + fn drop(&mut self) { + // SAFETY: The VM that maps the tile owns it and drops it after its + // page tables. + unsafe { kernel::bindings::__free_pages(self.page, Self::ORDER) }; + } +} diff --git a/drivers/gpu/drm/asahi/mmu/table.rs b/drivers/gpu/drm/asahi/mmu/table.rs new file mode 100644 index 00000000000000..3414c99774bd0e --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/table.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Page-table operations the M5 paths use beyond the common ones. + +use super::{sparse_tile::SparseTile, Prot, UatPageTable}; +use core::ops::Range; +use kernel::{prelude::*, sync::Arc}; + +impl UatPageTable { + pub(super) fn visit_runs( + &self, + range: Range, + visit: impl FnMut(u64, u64, u8) -> Result, + ) -> Result { + let _ = (range, visit); + Err(EOPNOTSUPP) + } + + pub(super) fn map_tiled(&mut self, range: Range, tile: &Arc, prot: Prot) -> Result { + let _ = (range, tile, prot); + Err(EOPNOTSUPP) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 215c19bf49ef08..a996a47ffe8735 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -289,7 +289,7 @@ const HIGH_BITS_PXN: u16 = 1 << 1; const HIGH_BITS_UXN: u16 = 1 << 2; const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Prot { memattr: u8, ap: u8, From 0f51320f81ec57fd90b73a1a4ba6de1b8ce55edc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:03 -0800 Subject: [PATCH 063/171] drm/asahi: g17: M5: build the M5 firmware's InitData graph Add the M5 firmware's InitData layout and its graph builder, and let InitData address the M5 bundle, private state, recovery views, effort word and control ring at their M5 locations. Firmware events stay off on M5 until its queues take submission, and the M5 effort word is published as zero. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..77dc6eb34210 (5 commits); SPTM/APIF removed; adapted to integration's InitData (status_b and views fields) and fw::Variant; comments and names reworded; M5 paths restored where the first rewrite lost them: the M5 metrics binding; original 6613f5ab7b7e, e25ba8a6fb2b, 1a021061f226, f49e91b3f364, 77dc6eb34210 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit 3e98334daa511130cfa0d513f4256d350ff03f3c in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/channels.rs | 6 +- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 2361 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 21 + drivers/gpu/drm/asahi/g17/fw/m5/power.rs | 12 + drivers/gpu/drm/asahi/g17/fw/mod.rs | 3 + drivers/gpu/drm/asahi/g17/initdata.rs | 145 +- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 197 ++ drivers/gpu/drm/asahi/g17/mod.rs | 22 +- 8 files changed, 2731 insertions(+), 36 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/power.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/m5.rs diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 53e899085a0e45..81fee87f406213 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -263,7 +263,9 @@ impl ControlRecord { match self { Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, - Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::SchedulerStateRelease(record) => if record.opcode == DeviceControlOp::M5SchedulerStateRelease as u32 { + DeviceControlOp::M5SchedulerStateRelease + } else { DeviceControlOp::SchedulerStateRelease }, Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, Self::ContextKill(_) => DeviceControlOp::ContextKill, } @@ -295,6 +297,8 @@ pub(crate) enum DeviceControlOp { IdlePowerOff = 0x0a, /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). SchedulerStateRelease = 0x14, + /// M5 encoding of the same scheduler-state release. + M5SchedulerStateRelease = 0x13, /// Release a free list's buffer slot ([`FreeListRelease`]). FreeListRelease = 0x2e, /// Stop the queues named by a closed logical scheduler. diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs new file mode 100644 index 00000000000000..1ad2413151823a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -0,0 +1,2361 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware initialization encodings and allocation layout. +//! Opaque initialized words have no inferred meaning. Runtime ownership is +//! provided by the common G17 KernelObject and GPUVM paths. + +#![allow(dead_code)] + +use core::ops::Range; +use core::result::Result; + +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const G17P_QOS_RESOURCE_SIZE: usize = 0xe40; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + + J813G17G25G76, + + J873G18G26A428, + + J714G17S26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => COMMON26_INITDATA_VERSION_WORD, + Self::J714G17S26A428 => M5_INITDATA_VERSION_WORD, + } + } + + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_ROOT_SIZE, + Self::J714G17S26A428 => M5_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + COMMON26_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY + } + (Self::J714G17S26A428, InstanceRole::Primary) => M5_CONTROL_OPENING_OPCODE_PRIMARY, + (Self::J714G17S26A428, InstanceRole::Secondary) => { + M5_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + pub(crate) const fn control_register_opcode(self) -> Option { + match self { + Self::J873G18G26A428 | Self::J714G17S26A428 => None, + Self::J700G17P | Self::J813G17G25G76 => Some(CONTROL_OPENING_PRIMARY_REGISTER_OPCODE), + } + } + + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_OFFSETS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => COMMON26_BUNDLE_VIEW_EXTENTS, + Self::J714G17S26A428 => M5_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => COMMON26_SHARED_LAYOUT.block_88, + Self::J714G17S26A428 => M5_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + + pub(crate) const fn work_ring_bundle_offsets(self) -> [usize; WORK_CHANNEL_COUNT] { + match self { + Self::J873G18G26A428 => COMMON26_WORK_RING_BUNDLE_OFFSETS, + Self::J714G17S26A428 => M5_WORK_RING_BUNDLE_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => WORK_RING_BUNDLE_OFFSETS, + } + } +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_FIXED_WORD: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +const ROOT_STATUS_A: usize = 0xa8; + +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +const MAIN_REGION_VIEWS: usize = 0x2d0; + +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SENTINEL_A: usize = 0x32c; + +const MAIN_PRIMARY_SENTINEL_B: usize = 0x344; + +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const CONTROL_RECORD_SIZE: usize = 0x40; + +pub(crate) const CONTROL_OPENING_PRIMARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_SECONDARY_PRODUCER: u32 = 1; + +pub(crate) const CONTROL_OPENING_PRIMARY_REGISTER_OPCODE: u32 = 0x20; + +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +pub(crate) const CHANNEL_STATE_PRODUCER: usize = 2; + +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, + 0x080, 0x180, 0x280, + 0x040, 0x140, 0x240, + 0x000, 0x100, 0x200, + 0x300, +]; + +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +pub(crate) const HW_REGISTER_MAP_OFFSET: usize = 0x640; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_APERTURE: Range = 0xffff_fc21_8000_0000..0xffff_fc21_9000_0000; + +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE: usize = 0x20; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + + Primary = 0, + + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + + pub(crate) region_a: u64, + + pub(crate) region_c: u64, + + pub(crate) hw_data_bundle: u64, + + pub(crate) hw_data_bundle_alloc: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + + pub(crate) status_b: u64, + + pub(crate) secondary_extras: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x0000000c), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x0000000f), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairRootSeparation, + PairAliasedObject, + SharedRegionSeparation, + InterfaceMismatch, + HostMappedAllocationsDisabled, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_FIXED_WORD, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +pub(crate) fn derive_channel_table_for_profile( + profile: RootProfile, + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = derive_channel_table(role, state_grid, status_a, main_config, bundle); + if role == InstanceRole::Primary { + let rings = profile.work_ring_bundle_offsets(); + for (index, entry) in table.iter_mut().take(WORK_CHANNEL_COUNT).enumerate() { + entry.ring = bundle + rings[index] as u64; + } + } + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + // J714's firmware takes both primary sentinels too; they are not + // exclusive to the older J700 ABI. + if matches!(profile, RootProfile::J700G17P | RootProfile::J714G17S26A428) { + put_u32(out, MAIN_PRIMARY_SENTINEL_A, 1); + put_u32(out, MAIN_PRIMARY_SENTINEL_B, 0xabcd_abcd); + } + + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + + RootProfile::J873G18G26A428 | RootProfile::J714G17S26A428 => { + COMMON26_MAIN_SECONDARY_KIND + } + }); + if profile == RootProfile::J714G17S26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + M5_SHARED_LAYOUT.policy as u64); + } else if profile == RootProfile::J873G18G26A428 { + + put_u64(out, COMMON26_MAIN_SECONDARY_POLICY_PTR, + bundle + COMMON26_SHARED_LAYOUT.policy as u64); + } else { + + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + + return encode_common26_region_c(out); + } + if profile == RootProfile::J714G17S26A428 { + + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + + if 1 == 0 { + put_u32(out, 0x9cc, 0); + } + + if 0 != 0 { + out[0xe21] = 1; + } + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +pub(crate) const COMMON26_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +pub(crate) const COMMON26_ROOT_SIZE: usize = 0xc8; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Evidence { + FirmwareProven, + ProducerChecked, + AdtProven, + Inferred, + Guessed, +} + +pub(crate) const ABI_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const ALLOC_HW_DATA: usize = 0; + +pub(crate) const ALLOC_BLOCK_C18: usize = 1; + +pub(crate) const ALLOC_BLOCK_1048: usize = 2; + +pub(crate) const ALLOC_BLOCK_E10: usize = 3; + +pub(crate) const ALLOC_POLICY: usize = 4; + +pub(crate) const ALLOC_BLOCK_68: usize = 5; + +pub(crate) const ALLOC_BLOCK_800: usize = 6; + +pub(crate) const ALLOC_BLOCK_88: usize = 7; + +pub(crate) const ALLOC_DM_RING: usize = 8; + +pub(crate) const ALLOC_MAIN_CONFIG: usize = 9; + +pub(crate) const ALLOC_CONTROL_RING: usize = 10; + +pub(crate) const ALLOC_RING_STATE: usize = 11; + +pub(crate) const ALLOC_STATUS_B: usize = 12; + +pub(crate) const ALLOC_STATUS_CONFIG: usize = 13; + +pub(crate) const ALLOC_STATUS_A: usize = 14; + +pub(crate) const ALLOC_LOG_STATE: usize = 15; + +pub(crate) const ALLOC_RING_4800: usize = 16; + +pub(crate) const ALLOC_BUFFER_28800: usize = 17; + +pub(crate) const ALLOC_BUFFER_79800: usize = 18; + +pub(crate) const ALLOC_RING_9000: usize = 19; + +pub(crate) const ALLOC_SECONDARY_EXTRA_1: usize = 20; + +pub(crate) const ALLOC_POLICY_STATE: usize = 21; + +pub(crate) const ALLOC_REGION_C: usize = 22; + +pub(crate) const ALLOC_AON_PTD: usize = 23; + +pub(crate) const ABI_BRN_TABLE_SIZE: usize = 0; + +pub(crate) const ABI_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +pub(crate) const PRIVATE_ALLOCATION_ENTRIES: usize = 38; + +const ROLE_SET_ENTRIES: usize = 10; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[ALLOC_HW_DATA]; + sizes[1] = table[ALLOC_BLOCK_C18]; + sizes[2] = table[ALLOC_BLOCK_1048]; + sizes[3] = table[ALLOC_BLOCK_E10]; + sizes[4] = table[ALLOC_POLICY]; + sizes[5] = table[ALLOC_BLOCK_68]; + sizes[6] = table[ALLOC_BLOCK_800]; + sizes[7] = ABI_BRN_TABLE_SIZE; + sizes[8] = table[ALLOC_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[ALLOC_DM_RING]; + ring += 1; + } + sizes[21] = table[ALLOC_MAIN_CONFIG]; + sizes[22] = table[ALLOC_CONTROL_RING]; + sizes[23] = table[ALLOC_MAIN_CONFIG]; + sizes[24] = table[ALLOC_CONTROL_RING]; + sizes +} + +pub(crate) const fn private_allocation_sizes(table: &[usize; 24]) -> [usize; PRIVATE_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; PRIVATE_ALLOCATION_ENTRIES]; + let mut state = 0; + while state <= WORK_CHANNEL_COUNT { + sizes[state] = table[ALLOC_RING_STATE]; + state += 1; + } + sizes[13] = table[ALLOC_STATUS_B]; + sizes[14] = table[ALLOC_STATUS_CONFIG]; + let role_set = [ + table[ALLOC_STATUS_A], table[ALLOC_RING_STATE], table[ALLOC_LOG_STATE], table[ALLOC_RING_STATE], + table[ALLOC_RING_STATE], table[ALLOC_RING_4800], table[ALLOC_BUFFER_28800], + table[ALLOC_BUFFER_79800], table[ALLOC_RING_9000], table[ALLOC_RING_4800], + ]; + let mut role = 0; + while role < 2 { + let mut entry = 0; + while entry < ROLE_SET_ENTRIES { + sizes[15 + role * ROLE_SET_ENTRIES + entry] = role_set[entry]; + entry += 1; + } + role += 1; + } + sizes[35] = table[ALLOC_RING_STATE]; + sizes[36] = table[ALLOC_SECONDARY_EXTRA_1]; + sizes[37] = table[ALLOC_POLICY_STATE]; + sizes +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RoleSetLayout { + + pub(crate) status_a: usize, + + pub(crate) event_state: usize, + + pub(crate) log_state: usize, + + pub(crate) trace_state: usize, + pub(crate) state_30: usize, + + pub(crate) event_ring: usize, + pub(crate) buffer_28800: usize, + + pub(crate) buffer_79800: usize, + + pub(crate) trace_ring: usize, + pub(crate) buffer_4800: usize, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrivateClusterLayout { + pub(crate) dm_ring_states: [usize; WORK_CHANNEL_COUNT], + + pub(crate) control_state: usize, + + pub(crate) status_b: usize, + + pub(crate) status_config: usize, + pub(crate) roles: [RoleSetLayout; 2], + + pub(crate) secondary_control_state: usize, + + pub(crate) secondary_extra_1: usize, + + pub(crate) policy_state: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +const fn role_set_layout(offsets: &[usize; PRIVATE_ALLOCATION_ENTRIES], first: usize) -> RoleSetLayout { + RoleSetLayout { + status_a: offsets[first], + event_state: offsets[first + 1], + log_state: offsets[first + 2], + trace_state: offsets[first + 3], + state_30: offsets[first + 4], + event_ring: offsets[first + 5], + buffer_28800: offsets[first + 6], + buffer_79800: offsets[first + 7], + trace_ring: offsets[first + 8], + buffer_4800: offsets[first + 9], + } +} + +pub(crate) const fn private_cluster_layout(table: &[usize; 24]) -> PrivateClusterLayout { + let sizes = private_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_ring_states = [0usize; WORK_CHANNEL_COUNT]; + let mut state = 0; + while state < WORK_CHANNEL_COUNT { + dm_ring_states[state] = offsets[state]; + state += 1; + } + let end = offsets[37] + sizes[37]; + PrivateClusterLayout { + dm_ring_states, + control_state: offsets[12], + status_b: offsets[13], + status_config: offsets[14], + roles: [role_set_layout(&offsets, 15), role_set_layout(&offsets, 25)], + secondary_control_state: offsets[35], + secondary_extra_1: offsets[36], + policy_state: offsets[37], + end, + total: round_up_usize(end, ABI_ALLOCATION_PAGE), + } +} + +pub(crate) const WORK_CHANNEL_DM_INDEX: [usize; WORK_CHANNEL_COUNT] = + [3, 7, 11, 2, 6, 10, 1, 5, 9, 0, 4, 8]; + +pub(crate) const fn work_ring_offsets(layout: &SharedClusterLayout) -> [usize; WORK_CHANNEL_COUNT] { + let mut rings = [0usize; WORK_CHANNEL_COUNT]; + let mut channel = 0; + while channel < WORK_CHANNEL_COUNT { + rings[channel] = layout.dm_rings[WORK_CHANNEL_DM_INDEX[channel]]; + channel += 1; + } + rings +} + +pub(crate) const M5_25G76_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_25G76); + +pub(crate) const COMMON26_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428); + +pub(crate) const COMMON26_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428[ALLOC_HW_DATA]; + +pub(crate) const COMMON26_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + COMMON26_SHARED_LAYOUT.block_c18, + COMMON26_SHARED_LAYOUT.block_1048, + COMMON26_SHARED_LAYOUT.block_e10, + COMMON26_SHARED_LAYOUT.block_68, + COMMON26_SHARED_LAYOUT.block_800, +]; + +pub(crate) const COMMON26_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_C18], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_1048], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_E10], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_68], + ABI_ALLOCATION_SIZES_26A428[ALLOC_BLOCK_800], +]; + +pub(crate) const COMMON26_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&COMMON26_SHARED_LAYOUT); + +pub(crate) const COMMON26_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +pub(crate) const COMMON26_MAIN_SECONDARY_KIND: u32 = 0xc; + +pub(crate) const COMMON26_FWCTL_RING_OFFSET: usize = 0x40; + +pub(crate) const COMMON26_STATUS_B_FWCTL_STATE: usize = 0x4ac0; + +pub(crate) const COMMON26_STATUS_B_FWCTL_RING: usize = 0x4ac8; + +pub(crate) const COMMON26_STATUS_B_CONFIG_HEADER: usize = 0xe714; + +pub(crate) const COMMON26_STATUS_B_LINK_WORD: usize = 0xe71c; + +pub(crate) const COMMON26_STATUS_B_LINK_PTR: usize = 0xe720; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LayoutBand { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) delta: isize, + pub(crate) evidence: Evidence, +} + +const fn band(start: usize, end: usize, delta: isize, evidence: Evidence) -> LayoutBand { + LayoutBand { start, end, delta, evidence } +} + +pub(crate) const fn relocate_by_bands(bands: &[LayoutBand], offset: usize) -> Option { + let mut index = 0; + while index < bands.len() { + let band = bands[index]; + if band.start <= offset && offset < band.end { + return Some((offset as isize + band.delta) as usize); + } + index += 1; + } + None +} + +pub(crate) const COMMON26_HW_DATA_BANDS: &[LayoutBand] = &[ + + band(0x000, 0x640, 0, Evidence::Inferred), + + band(0xe88, 0xe90, 0x118, Evidence::ProducerChecked), + + band(0xe90, 0xebc, 0x120, Evidence::FirmwareProven), + + band(0xec0, 0xf5c, 0x11c, Evidence::FirmwareProven), + band(0xf5c, 0xf60, 0x124, Evidence::ProducerChecked), + + band(0xf68, 0xf98, 0x11c, Evidence::FirmwareProven), + band(0xf98, 0xfac, 0x120, Evidence::FirmwareProven), + + band(0xfb0, 0x25f8, 0x11c, Evidence::FirmwareProven), + band(0x25f8, 0x26b4, 0x120, Evidence::FirmwareProven), + band(0x26b4, 0x26f0, 0x128, Evidence::FirmwareProven), + band(0x26f0, M5_25G76_HW_DATA_SIZE, 0x12c, Evidence::FirmwareProven), +]; + +pub(crate) const COMMON26_STATUS_CONFIG_BANDS: &[LayoutBand] = &[ + band(0x000, 0x1dc, 0, Evidence::ProducerChecked), + band(0x4e0, 0x518, 0xc, Evidence::ProducerChecked), + band(0x518, 0x528, 0x14, Evidence::ProducerChecked), + band(0x530, 0x534, 0xc, Evidence::ProducerChecked), + band(0x554, 0x584, 0xc, Evidence::ProducerChecked), + band(0x584, 0x5a8, 0x1c, Evidence::ProducerChecked), + band(0x5cc, 0x628, 0x18, Evidence::ProducerChecked), +]; + +fn write_relocated_words( + words: &[(usize, u32)], + source_base: usize, + bands: &[LayoutBand], + out: &mut [u8], +) -> Result<(), InitdataError> { + for &(offset, value) in words { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + if byte == 0 { + continue; + } + let source = offset + .checked_sub(source_base) + .ok_or(InitdataError::BufferSize)? + + index; + if let Some(target) = relocate_by_bands(bands, source) { + *out.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + } + Ok(()) +} + +pub(crate) const COMMON26_HW_REGISTER_MAP_SLOTS: usize = 60; + +pub(crate) const COMMON26_HW_REGISTER_MAP_END: usize = + HW_REGISTER_MAP_OFFSET + COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE; + +pub(crate) const COMMON26_HW_CHIP_ID_OFFSET: usize = 0xfb0; + +pub(crate) const COMMON26_HW_PERF_STATE_COUNT: usize = 0x10e0; + +pub(crate) const COMMON26_HW_PERF_FREQUENCY: usize = 0x10e4; + +pub(crate) const COMMON26_HW_PERF_CORE_VOLTAGE: usize = 0x1124; + +pub(crate) const COMMON26_HW_PERF_SRAM_VOLTAGE: usize = 0x1524; + +pub(crate) const COMMON26_HW_PERF_ROWS: usize = 16; + +pub(crate) const COMMON26_HW_PERF_ROW_STRIDE: usize = 0x40; + +pub(crate) const COMMON26_HW_PERF_LANES: usize = 16; + +pub(crate) const COMMON26_HW_PERF_SCALE: usize = 0x1964; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_A: usize = 0x19e4; + +pub(crate) const COMMON26_HW_PERF_RELATIVE_B: usize = 0x1a24; + +const M5_HW_AUX_RELATIVE_FREQUENCY: usize = 0x1aa4; +const M5_HW_AUX_PERF_STATE: usize = 0x1cac; +const M5_HW_AUX_PERF_STATE_SIZE: usize = 0x148; + +// Clock-domain 2: MHz, core mV, SRAM mV. The second voltage lane is unused. +const J714_AUX_PERF_STATES: [(u32, u32, u32); 14] = [ + (24, 125, 775), (338, 590, 775), (486, 620, 775), (636, 645, 775), + (796, 685, 775), (888, 700, 775), (988, 745, 775), (1084, 770, 775), + (1182, 795, 795), (1278, 835, 835), (1374, 860, 860), + (1470, 900, 900), (1578, 960, 960), (1620, 990, 990), +]; + +// Evaluate the frequency-only curve at compile time; no kernel FPU use. +// Conversion uses Hz before the single-precision multiply and divide. +const fn j714_relative_frequency() -> [u32; 16] { + let mut values = [0; 16]; + let base = J714_GPU_PERF_STATES[1].0 as u64 * 1_000_000; + let maximum = J714_GPU_PERF_STATES[13].0 as u64 * 1_000_000; + let denominator = (maximum - base) as f32; + let mut state = 2; + while state < 14 { + let frequency = J714_GPU_PERF_STATES[state].0 as u64 * 1_000_000; + values[state] = (((frequency - base) as f32 * 100.0) / denominator).to_bits(); + state += 1; + } + values +} + +const J714_RELATIVE_FREQUENCY: [u32; 16] = j714_relative_frequency(); + +fn encode_m5_aux_perf_states(out: &mut [u8]) { + let table = &mut out[M5_HW_AUX_PERF_STATE.. + M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; + table.fill(0); + put_u32(table, 0, 13); + put_u32(table, 4, 14); + for (state, &(frequency, core, sram)) in J714_AUX_PERF_STATES.iter().enumerate() { + put_u32(table, 8 + state * 4, frequency); + put_u32(table, 0x48 + state * 8, core); + put_u32(table, 0xc8 + state * 8, sram); + } +} + +pub(crate) const COMMON26_HW_PERF_INDEX_MAP: usize = 0x1ae4; + +pub(crate) const COMMON26_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2734; + +pub(crate) const COMMON26_QOS_POINTER_OFFSET: usize = 0x27cc; + +pub(crate) const COMMON26_HW_FIXED_WORD_OFFSET: usize = 0x2714; + +pub(crate) const COMMON26_HW_FIXED_WORD_VALUE: u32 = 0x96; + +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +const COMMON26_PERF_SCALE_VALUE: u32 = 0x3f82_8f5c; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindowSpec { + pub(crate) slot: usize, + pub(crate) phys: u64, + + pub(crate) size_10: u32, + + pub(crate) size_14: u32, + + pub(crate) unk_18: u64, + + pub(crate) flag: u32, +} + +const fn spec(slot: usize, phys: u64, size: u32, unk_18: u64, flag: u32) -> RegisterWindowSpec { + RegisterWindowSpec { slot, phys, size_10: size, size_14: size, unk_18, flag } +} + +pub(crate) const COMMON26_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +pub(crate) fn encode_common26_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + + let shifted = ABI_ALLOCATION_SIZES_25G76[ALLOC_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in COMMON26_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + Ok(()) +} + +pub(crate) fn encode_common26_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + // J714 descriptors carry device-specific AON table indices. + const RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0079_0001, 0x0000_1401], + [0x6731_5452, 0x00bc_0001, 0x0000_1401], + [0x7230_5452, 0x00ce_0001, 0x0000_1401], + ]; + out.fill(0); + for (index, record) in RECORDS.iter().enumerate() { + for (word, value) in record.iter().enumerate() { + put_u32(out, index * 12 + word * 4, *value); + } + } + Ok(()) +} + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const COMMON26_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_PRIMARY: u32 = COMMON26_CONTROL_OPENING_OPCODE_PRIMARY; + +pub(crate) const M5_CONTROL_OPENING_OPCODE_SECONDARY: u32 = + COMMON26_CONTROL_OPENING_OPCODE_SECONDARY; + +pub(crate) const M5_INITDATA_VERSION_WORD: u64 = 0x0c8e_0339_8728_04c0; + +pub(crate) const M5_ROOT_SIZE: usize = 0xc8; + +pub(crate) const ABI_ALLOCATION_SIZES_26A428_M5: [usize; 24] = [ + 0x2850, 0x0c18, 0x1048, 0x0e10, 0x126b0, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x0780, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x00a8, 0x0008, 0x1ca0, 0x00e4, +]; + +pub(crate) const M5_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&ABI_ALLOCATION_SIZES_26A428_M5); + +pub(crate) const M5_SHARED_CLUSTER_SIZE: usize = M5_SHARED_LAYOUT.total; + +pub(crate) const M5_PRIVATE_CLUSTER_SIZE: usize = M5_PRIVATE_LAYOUT.total; + +pub(crate) const M5_HW_DATA_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_HW_DATA]; + +pub(crate) const M5_KICK_SHARED_SIZE: usize = 0x2000; +pub(crate) const M5_KICK_SHARED_POINTER: usize = 0x27dc; + +pub(crate) fn set_m5_kick_shared(out: &mut [u8], address: u64) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(address)?; + let end = address.checked_add(M5_KICK_SHARED_SIZE as u64 - 1) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(end)?; + if address & (UAT_PAGE_SIZE as u64 - 1) != 0 { + return Err(InitdataError::AddressOutOfRange); + } + put_u64(out, M5_KICK_SHARED_POINTER, address); + Ok(()) +} + +pub(crate) const M5_POLICY_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_POLICY]; + +pub(crate) const M5_STATUS_CONFIG_SIZE: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_STATUS_CONFIG]; + +pub(crate) const M5_REGION_C_ALLOCATION: usize = + ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_REGION_C]; + +pub(crate) const M5_AON_PTD_SIZE: usize = ABI_ALLOCATION_SIZES_26A428_M5[ALLOC_AON_PTD]; + +pub(crate) const M5_SHARED_PREFIX_SIZE: usize = M5_SHARED_LAYOUT.brn_table; + +pub(crate) const M5_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + M5_SHARED_LAYOUT.block_c18, + M5_SHARED_LAYOUT.block_1048, + M5_SHARED_LAYOUT.block_e10, + M5_SHARED_LAYOUT.block_68, + M5_SHARED_LAYOUT.block_800, +]; + +pub(crate) const M5_BUNDLE_VIEW_EXTENTS: [usize; 5] = COMMON26_BUNDLE_VIEW_EXTENTS; + +pub(crate) const M5_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&M5_SHARED_LAYOUT); + +pub(crate) const M5_MAIN_BUNDLE_OFFSETS: [usize; 2] = M5_SHARED_LAYOUT.main_config; + +pub(crate) const M5_CONTROL_RING_BUNDLE_OFFSETS: [usize; 2] = + M5_SHARED_LAYOUT.control_ring; + +pub(crate) const M5_PRIVATE_STATUS_A: [usize; 2] = [ + M5_PRIVATE_LAYOUT.roles[0].status_a, + M5_PRIVATE_LAYOUT.roles[1].status_a, +]; + +pub(crate) const M5_PRIVATE_STATE_GRIDS: [usize; 2] = [ + M5_PRIVATE_LAYOUT.dm_ring_states[0], + M5_PRIVATE_LAYOUT.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], +]; + +pub(crate) const M5_STATUS_CONFIG_IN_STATUS_B: usize = + M5_PRIVATE_LAYOUT.status_config - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_STATUS_B_SPAN: usize = + M5_PRIVATE_LAYOUT.roles[0].status_a - M5_PRIVATE_LAYOUT.status_b; + +pub(crate) const M5_RECOVERY_STATE: usize = 0x4ae0; +pub(crate) const M5_HOST_RECOVERY: usize = 0x4af0; +pub(crate) const M5_RECOVERY_INFO: usize = 0x4298; +pub(crate) const M5_RECOVERY_SLOT_MASK: usize = 0x4ab8; + +pub(crate) const M5_STATUS_CONFIG_LINK_WORD: usize = 0x774; + +pub(crate) const M5_STATUS_CONFIG_LINK_PTR: usize = 0x778; + +pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; + +pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; + +pub(crate) const J714_CORE_REVISION: (u32, u32) = (0x18, 4); + +pub(crate) const J714_GPU_CORES: u32 = 20; + +/// Populate the enabled-USC count from the device's sampled core masks. +/// This count covers the whole device, not one cluster. +pub(crate) fn set_m5_enabled_cores(out: &mut [u8], enabled: u32) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + if enabled == 0 || enabled > 128 { + return Err(InitdataError::InterfaceMismatch); + } + put_u32(out, 0x10a4, enabled); + put_u32(out, 0x268c, enabled); + Ok(()) +} + +pub(crate) const M5_POLICY_STATE_PTR: usize = 0x8254; + +pub(crate) const M5_POLICY_AON_PTD_PTR: usize = 0x11a1c; + +pub(crate) const J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS: u32 = 200_000; + +pub(crate) const J714_GPU_PERF_BASE_PSTATE: u32 = 1; + +const J714_POWER_POLICY_MINIMUM_TABLE: usize = 0x1cab; + +const J714_POWER_POLICY_MINIMUM_ENTRIES: usize = 16 * 32; + +pub(crate) const J714_GPU_NUM_PERF_STATES: u32 = 13; + +// Frequency in MHz, two core-voltage tables, two SRAM-voltage tables (mV). +pub(crate) const J714_GPU_PERF_STATES: [(u32, u32, u32, u32, u32); 14] = [ + (0, 125, 125, 775, 775), (338, 605, 605, 775, 775), + (486, 620, 635, 775, 775), (636, 655, 655, 775, 775), + (796, 705, 705, 775, 775), (888, 715, 715, 775, 775), + (988, 745, 745, 775, 775), (1084, 770, 770, 775, 775), + (1182, 800, 800, 800, 800), (1278, 840, 840, 840, 840), + (1374, 880, 880, 880, 880), (1470, 925, 925, 925, 925), + (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), +]; + +pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; + +pub(crate) struct J714RegisterWindowSpec { + pub(crate) slot: usize, + + pub(crate) instances: &'static [u64], + pub(crate) size: u32, + + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +const fn jw(slot: usize, instances: &'static [u64], size: u32, unk_18: u64, flag: u32) + -> J714RegisterWindowSpec { + J714RegisterWindowSpec { slot, instances, size, unk_18, flag } +} + +const G: u64 = J714_GPU_BASE; + +pub(crate) const J714_REGISTER_WINDOW_SPECS: [J714RegisterWindowSpec; 29] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000], 0x4000, 0, 2), + jw(20, &[0x23_0300_0000, 0x23_0320_0000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000], 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), +]; + +pub(crate) const J714_REGISTER_WINDOW_58: J714RegisterWindowSpec = + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct J714RegisterWindow { + pub(crate) slot: usize, + pub(crate) device_va: u64, + pub(crate) count: usize, + pub(crate) size: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +pub(crate) const J714_REGISTER_WINDOW_COUNT: usize = J714_REGISTER_WINDOW_SPECS.len() + 1; + +pub(crate) fn j714_register_window_spec(index: usize) -> &'static J714RegisterWindowSpec { + J714_REGISTER_WINDOW_SPECS.get(index).unwrap_or(&J714_REGISTER_WINDOW_58) +} + +pub(crate) fn j714_register_windows() -> [J714RegisterWindow; J714_REGISTER_WINDOW_COUNT] { + let mask = UAT_PAGE_SIZE as u64 - 1; + let mut out = [J714RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; + J714_REGISTER_WINDOW_COUNT]; + let mut next: u64 = 0; + for (index, window) in out.iter_mut().enumerate() { + let spec = j714_register_window_spec(index); + let va = next + (spec.instances[0] & mask); + let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); + *window = J714RegisterWindow { + slot: spec.slot, + device_va: HW_REGISTER_APERTURE.start + va, + count: spec.instances.len(), + size: spec.size, + unk_18: spec.unk_18, + flag: spec.flag, + }; + let end = (va + span + mask) & !mask; + next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; + } + out +} + +pub(crate) fn encode_j714_register_map( + windows: &[J714RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, window) in windows.iter().enumerate() { + let spec = j714_register_window_spec(index); + if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS + || !HW_REGISTER_APERTURE.contains(&window.device_va) + { + return Err(InitdataError::RegisterWindow); + } + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, spec.instances[0]); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.count as u32 * window.size); + put_u32(out, base + 0x14, window.size); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + Ok(()) +} + +pub(crate) fn encode_m5_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + windows: &[J714RegisterWindow], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high + .checked_add(G17P_QOS_RESOURCE_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; M5_25G76_COMPLETION_QUEUE_COUNT * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + let g18 = &mut out[..COMMON26_HW_DATA_SIZE]; + g18.fill(0); + write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, COMMON26_HW_DATA_BANDS, g18)?; + encode_j714_register_map(windows, &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; + put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, 0x6050); + put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, J714_GPU_NUM_PERF_STATES); + put_u32(g18, COMMON26_HW_FIXED_WORD_OFFSET, COMMON26_HW_FIXED_WORD_VALUE); + + put_u32(g18, 0x0fb4, 1); + // J714's 26A428 HW data carries 1 here. The firmware copies this + // header at initialization. + put_u32(g18, 0x0fb8, 1); + put_u32(g18, 0x10d4, J714_CORE_REVISION.0); + put_u32(g18, 0x10d8, J714_CORE_REVISION.1); + put_u32(g18, 0x10dc, J714_GPU_CORES); + put_u32(g18, 0x283a, 1); + put_u32(g18, 0x283e, 1); + for (state, &(frequency, core0, core1, sram0, sram1)) in J714_GPU_PERF_STATES.iter().enumerate() { + put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); + put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); + + let relative = frequency * 100 / J714_GPU_PERF_STATES[13].0; + put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); + for lane in 0..COMMON26_HW_PERF_LANES { + let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; + put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, if lane == 1 { core1 } else { core0 }); + put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, if lane == 1 { sram1 } else { sram0 }); + } + } + for index in 0..COMMON26_HW_PERF_ROWS { + put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, + J714_RELATIVE_FREQUENCY[index]); + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, + J714_RELATIVE_FREQUENCY[index]); + } + encode_m5_aux_perf_states(g18); + let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; + g18[table..table + completion.len()].copy_from_slice(&completion); + put_u64(g18, COMMON26_QOS_POINTER_OFFSET, qos_high); + + let insert = M5_HW_DATA_INSERT; + let len = M5_HW_DATA_INSERT_LEN; + out.copy_within(insert..COMMON26_HW_DATA_SIZE, insert + len); + out[insert..insert + len].fill(0); + put_u32(out, 0x2660, 0); + put_u32(out, 0x2728, 0); + put_u32(out, 0x272c, 1); + // J714's 26A428 HW data carries this tiling-allocator word. The + // firmware copies it at initialization and the tiling allocator shifts + // it by 19. The inherited J813 word selected three groups instead of four. + put_u32(out, 0x26b0, 0x0020_0000); + Ok(()) +} + +pub(crate) const M5_POLICY_MAXIMUM_POWER: usize = 0xa18; + +/// Apply the calibrated maximum shared by the policy and host power input. +pub(crate) fn set_m5_region_maximum_power(out: &mut [u8], maximum: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if maximum == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x88, maximum); + Ok(()) +} + +pub(crate) fn encode_m5_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { + return Err(InitdataError::BufferSize); + } + out.fill(0); + encode_common26_region_c(&mut out[..REGION_C_SIZE])?; + + out.copy_within(0xe10..0xe78, 0x1c34); + out[0x9e4..0x1c34].fill(0); + out.copy_within(0x7f4..0x9e4, 0x7f8); + put_u32(out, 0x7f4, 1); + + put_u32(out, 0x9e0, 50); + + put_u32(out, 0x08c, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x090, J714_GPU_NUM_PERF_STATES * 100); + put_u32(out, 0x09c, (J714_GPU_NUM_PERF_STATES + 1) * 100); + put_u32(out, 0x034, 1); + put_u32(out, 0x038, 1); + put_u32(out, 0x7e4, 0x3f19_999a); + put_u32(out, 0x1c30, 250); + const FW_UTIL_TUNABLES: [u8; 32] = [ + 1, 0, 0, 0, 0, 2, 2, 1, 1, 40, 10, 1, 1, 1, 2, 40, + 10, 1, 1, 1, 2, 40, 10, 1, 1, 1, 1, 40, 10, 1, 1, 0, + ]; + out[0x9e8..0xa08].copy_from_slice(&FW_UTIL_TUNABLES); + Ok(()) +} + +pub(crate) fn encode_m5_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let config = M5_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, COMMON26_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, COMMON26_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, COMMON26_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, COMMON26_STATUS_B_LINK_WORD, 0); + put_u64(out, COMMON26_STATUS_B_LINK_PTR, + status_b + (config + M5_STATUS_CONFIG_LINK_WORD) as u64); + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, COMMON26_STATUS_CONFIG_BANDS, + &mut out[config..config + M5_STATUS_CONFIG_SIZE])?; + put_u32(out, config + M5_STATUS_CONFIG_LINK_WORD, 1); + put_u64(out, config + M5_STATUS_CONFIG_LINK_PTR, + status_b + COMMON26_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +pub(crate) fn encode_m5_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(COMMON26_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_m5_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +pub(crate) fn encode_m5_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = M5_SHARED_LAYOUT; + if out.len() < M5_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + out[layout.block_c18..M5_SHARED_PREFIX_SIZE].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } + } + let policy = &mut out[layout.policy..layout.policy + M5_POLICY_SIZE]; + put_u32(policy, 0x04, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + put_u32(policy, 0x08, J714_GPU_POWER_SAMPLE_PERIOD_AIC_CLOCKS); + + for (offset, value) in [ + (0x10, 4), (0x14, 0x3f80_0000), (0x2c, 1), (0x30, 1), + (0x40, J714_GPU_PERF_BASE_PSTATE * 100), (0x44, 1), + (0x48, J714_GPU_NUM_PERF_STATES * 100), (0x50, 100), + (0x60, 4), (0x64, 0x3f80_0000), (0x6c, 1), (0x70, 1), + (0x80, J714_GPU_PERF_BASE_PSTATE * 100), (0x84, 1), + (0x88, J714_GPU_NUM_PERF_STATES * 100), (0x90, 100), + + (0x9c, 1), (0xa0, 1), (0xa4, 1), (0xa8, 1), + ] { + put_u32(policy, offset, value); + } + + for index in 0..J714_POWER_POLICY_MINIMUM_ENTRIES { + put_u16(policy, J714_POWER_POLICY_MINIMUM_TABLE + index * 2, i16::MAX as u16); + } + + for (offset, value) in [ + (0xb58, 1), + (0xb60, 0x3f4c_cccd), + (0xb64, 0x3f7e_b852), + (0xb68, 0x3e4c_cccd), + (0xb6c, 0x3ba3_d70a), + (0xb70, 0x3f85_0e56), + (0xb74, 0x3f85_0e56), + (0xb80, 0x4060_0000), + (0xb84, 0x4060_0000), + ] { + put_u32(policy, offset, value); + } + put_u64(policy, M5_POLICY_STATE_PTR, bindings.state); + put_u64(policy, M5_POLICY_AON_PTD_PTR, bindings.aon_ptd); + Ok(()) +} + +/// Initial controller inputs for J714 with the matched calibration. +pub(crate) fn encode_m5_policy_controller_defaults(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() < M5_SHARED_PREFIX_SIZE { return Err(InitdataError::BufferSize); } + // Both calibrated hardware curves, as J714's firmware expects them. + const RELATIVE_POWER_MAIN: [u32; 16] = [ + 0x00000000, 0x4106f651, 0x413ea3b9, 0x4180c1c6, + 0x41b433fc, 0x41ccaf2d, 0x41f7a8ee, 0x42104b59, + 0x4228d2f7, 0x424956c5, 0x426d29a9, 0x428da278, + 0x42b4e595, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 16] = [ + 0x00000000, 0x4105a0ed, 0x414a36f5, 0x418b74a8, + 0x41c1510d, 0x41df97fb, 0x420be387, 0x4223193c, + 0x423cc4e6, 0x42604354, 0x427ef922, 0x42957243, + 0x42b6ec22, 0x42c80000, 0x00000000, 0x00000000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const INITIAL: &[(usize, u32)] = &[ + (0xd4, 0x3f828f5c), + (0xd8, 0x3f828f5c), + (0xdc, 0x3f828f5c), + (0xe0, 0x3f828f5c), + (0xe4, 0x3f828f5c), + (0xe8, 0x3f828f5c), + (0xec, 0x3f828f5c), + (0xf0, 0x3f828f5c), + (0xf4, 0x3f828f5c), + (0xf8, 0x3f828f5c), + (0xfc, 0x3f828f5c), + (0x100, 0x3f828f5c), + (0x104, 0x3f828f5c), + (0x108, 0x3f828f5c), + (0x9cc, 0x271), + (0x9d8, 0x3f7f2e9f), + (0x9e0, 0x3b516154), + (0x9e8, 0x3ca59586), + (0x9f4, 0x47c12200), + (0x9f8, 0x40a90fdb), + (0xa00, 0xbc576554), + (0xa04, 0x28), + (0xa08, J714_GPU_NUM_PERF_STATES * 100), + (0xa0c, J714_GPU_NUM_PERF_STATES * 100), + (M5_POLICY_MAXIMUM_POWER, 0x18244), + (0xa4c, J714_GPU_NUM_PERF_STATES * 100), + (0xa84, 0x8), + (0xa90, 0x3f000000), + (0xa98, 0x3f000000), + (0xaa0, 0x3e0b4396), + (0xaac, 0x47800000), + (0xab0, 0x3e99999a), + (0xabc, 0x28), + (0xac0, J714_GPU_NUM_PERF_STATES * 100), + (0xad0, 0x18244), + (0xadc, 0x10), + (0xae4, 0x5dc00), + (0xb3c, 0x4b), + (0xb44, 0x55), + (0xb48, 0x32), + (0xb4c, 0x6), + (0xb50, 0x1), + (0xb54, 0x6), + (0xb7c, 0x42be0000), + (0xb88, 0x414a1af3), + (0xb8c, 0x64), + (0xb90, J714_GPU_NUM_PERF_STATES * 100), + (0xb94, 0x64), + (0xba0, 0x4a), + (0xba8, 0x1), + (0xbd4, 0x64), + (0xbf4, 0x401), + (0xbf8, 0x6400), + (0xbfc, 0xa00), + (0xc00, 0x1f4000), + (0xc04, 0x12c00), + (0xc08, 0xfa000), + (0xc0c, 0x12c00), + (0xc10, 0x7d000), + (0xc14, 0x12c00), + (0xc18, 0x3e800), + (0xc1c, 0x19000), + (0xc20, 0x100), + (0xc78, 0x1), + (0x22ac, 0x47800000), + (0x22b0, 0x42c80000), + (0x22b4, 0x44a28000), + (0x22b8, 0x654), + (0x22cc, 0x64), + (0x22d4, 0x42c80000), + (0x22d8, J714_GPU_NUM_PERF_STATES * 100), + (0x22e0, 0x3f4ccccd), + (0x22e4, 0x3e4ccccd), + (0x2328, 0x6fcc3fcc), + (0x232c, 0x6fcc3fcc), + (0x234c, 0x1f40), + (0x2354, 0x4), + (0x2368, 0x3f800000), + (0x2370, 0x40ad5555), + (0x237c, 0x47800000), + (0x2380, 0x42080000), + (0x238c, 0x28), + (0x2390, J714_GPU_NUM_PERF_STATES * 100), + (0x2394, J714_GPU_NUM_PERF_STATES * 100), + (0x23a0, 0x2af8), + (0x23d4, J714_GPU_NUM_PERF_STATES * 100), + (0x2cec, 0x20), + (0x2cf0, 0x42000006), + (0x2cfc, 0x3), + (0x2d24, 0x1f40), + (0x2d2c, 0x4), + (0x2d40, 0x3f800000), + (0x2d48, 0x40855555), + (0x2d54, 0x47800000), + (0x2d58, 0x42080000), + (0x2d64, 0x64), + (0x2d68, J714_GPU_NUM_PERF_STATES * 100), + (0x2d6c, J714_GPU_NUM_PERF_STATES * 100), + (0x2d78, 0x2bc0), + (0x2dac, J714_GPU_NUM_PERF_STATES * 100), + (0x36c4, 0x20), + (0x36c8, 0x42000006), + (0x3fdc, 0xb8520000), + (0x3fe0, 0x3f7e), + (0x3fe4, 0xd70a0000), + (0x3fe8, 0x3ba3), + (0x3ff0, 0x43c8), + (0x3ffc, 0x4780), + (0x4000, 0x42c8), + (0x4008, 0x384bac8), + (0x400c, 0x5140000), + (0x4010, 0x3e80000), + (0x4018, 0x80000000), + (0x401c, 0x26ac453b), + (0x4050, 0x5140000), + (0x406c, 0xc80000), + (0x4124, J714_GPU_NUM_PERF_STATES * 100), + (0x4138, 0x1), + (0x41cc, 0x10001), + (0x41d0, 0x40000), + (0x41e8, 0x23f0000), + (0x41ec, 0x10000), + (0x41f0, 0x10000), + (0x4208, 0x2400000), + (0x420c, 0x10000), + (0x4210, 0x10000), + (0x4228, 0x2410000), + (0x42d4, 0x640000), + (0x42f8, 0x1), + (0x438c, 0x10001), + (0x4390, 0x40000), + (0x43a8, 0x25b0000), + (0x43ac, 0x10000), + (0x43b0, 0x10000), + (0x43c8, 0x25c0000), + (0x43cc, 0x10000), + (0x43d0, 0x10000), + (0x43e8, 0x25d0000), + (0x4494, 0x640000), + (0x449c, 0x1), + (0x44a4, 0x18244), + (0x44a8, 0x18244), + (0x44ac, 0x18244), + (0x44bc, 0x3f800000), + (0x44c0, 0x4), + (0x44c4, 0x8), + (0x44c8, 0x2ee00), + (0x4598, 0x7c), + (0x45a4, 0x3f77bdef), + (0x45ac, 0x3d042108), + (0x45b4, 0x3cf5c290), + (0x45c0, 0x47800000), + (0x45c4, 0x3f19999a), + (0x45d0, 0x28), + (0x45d4, J714_GPU_NUM_PERF_STATES * 100), + (0x45d8, J714_GPU_NUM_PERF_STATES * 100), + (0x45e0, 0x47c12200), + (0x45e4, 0x18244), + (0x45f0, 0xf8), + (0x45f8, 0x5ad200), + (0x4618, J714_GPU_NUM_PERF_STATES * 100), + (0x4658, 0x1), + (0x4728, 0x32), + (0x472c, 0x1), + (0x4734, 0x3f638e39), + (0x4738, 0x3f2aaaab), + (0x473c, 0x3de38e39), + (0x4740, 0x3eaaaaab), + (0x4744, 0xbecccccd), + (0x4748, 0xbecccccd), + (0x4750, 0x47800000), + (0x4754, 0xc0a00000), + (0x4758, 0xc0a00000), + (0x4760, 0x64), + (0x4764, J714_GPU_NUM_PERF_STATES * 100), + (0x4768, 0x64), + (0x4770, 0x45fa0000), + (0x4774, 0x578), + (0x477c, 0xe), + (0x4780, 0x48), + (0x4784, 0x18), + (0x4788, 0x1a5e00), + (0x4790, 0x8ca00), + (0x479c, 0x47800000), + (0x47c4, 0x1f40), + (0x47c8, 0x4), + (0x47cc, 0x9c4), + (0x47d0, 0x258), + (0x47d4, 0x2), + (0x47d8, 0x4), + (0x47dc, 0x32), + (0x480c, 0x47800000), + (0x481c, 0x28), + (0x4820, J714_GPU_NUM_PERF_STATES * 100), + (0x48ec, 0x1), + (0x48f0, 0x1), + (0x4910, 0xdac), + (0x4914, 0x3e8), + (0x4918, 0xbb8), + (0x49b8, J714_GPU_NUM_PERF_STATES * 100), + (0x49bc, J714_GPU_NUM_PERF_STATES * 100), + (0x49c0, 0x64), + (0x49c4, 0x20), + (0x49cc, 0x1), + (0x49d0, 0x1), + (0x49fc, 0x1f4), + (0x4a98, J714_GPU_NUM_PERF_STATES * 100), + (0x4a9c, J714_GPU_NUM_PERF_STATES * 100), + (0x4aa0, 0x64), + (0x4aa4, 0x20), + (0x4ac8, 0x4), + (0x4ad4, 0x3f780000), + (0x4adc, 0x3d000000), + (0x4ae4, 0x40a00000), + (0x4af0, 0x47800000), + (0x4af4, 0x41700000), + (0x4b00, 0x4b0), + (0x4b04, J714_GPU_NUM_PERF_STATES * 100), + (0x4b08, J714_GPU_NUM_PERF_STATES * 100), + (0x4b14, J714_GPU_NUM_PERF_STATES * 100), + (0x4b48, J714_GPU_NUM_PERF_STATES * 100), + (0x4b94, 0x42c80000), + (0x4ba4, 0x3), + (0x4ba8, 0x6fcc3fcc), + (0x4bac, 0x6fcc3fcc), + (0x4bcc, 0x7d), + (0x4bd0, 0x1), + (0x11a24, 0x67305452), + (0x11a28, 0x67315452), + (0x11a6c, 0x72305452), + (0x121a4, 0x1), + (0x122e0, 0x186a), + (0x121b0, 0x44d5c000), + (0x121b4, 0x44d20000), + (0x12230, 0x43140000), + (0x12234, 0x43140000), + (0x12280, 0x43d60000), + ]; + for &(offset, value) in INITIAL { put_u32(policy, offset, value); } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs new file mode 100644 index 00000000000000..beccacf8e061d3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware-specific work encodings, using the shared G17 queue primitives. + +pub(crate) mod initdata; +pub(crate) mod power; + +/// M5 adds a statistics record at tag 32, beyond the legacy 32-bit mask. +pub(crate) const STATISTICS_TYPES: u64 = 0x1_5fff_0000; + +/// M5 stores an entry count in the trailer, with its upper half clear. +/// The preceding register storage and self pointer use the shared layout. +pub(crate) fn set_register_array( + array: &mut super::kick::RegisterArray, + address: u64, + writes: &[super::kick::RegisterWrite], +) -> kernel::prelude::Result { + array.set(address, writes)?; + array.size = writes.len() as u32; + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/power.rs b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs new file mode 100644 index 00000000000000..9d0ab94f392e8f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/power.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Live M5 Region C inputs. These offsets do not follow the Neo layout. + +pub(crate) const EFFORT: usize = 0xe8; +pub(crate) const EFFORT_OVERRIDE: usize = 0xec; +pub(crate) const POWER_ASSERT_TALLY: usize = 0x1c99; + +/// Control records which assert power among the opcodes this driver emits. +pub(crate) const fn asserts_gpu_power(opcode: u32) -> bool { + matches!(opcode, 0x08 | 0x0a | 0x29) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 1cb46fce2d6028..df69458699b391 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -19,6 +19,9 @@ pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; pub(crate) mod kick; +// Most M5 encodings are reached only once an M5 board has a page-table boot path. +#[allow(dead_code)] +pub(crate) mod m5; pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 7e77feb2d3ee7e..8faa3ec06c8f7b 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -12,11 +12,14 @@ use super::{ }; use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; use core::ops::Range; -use core::sync::atomic::{fence, AtomicU32, Ordering}; -use kernel::{c_str, prelude::*}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::{c_str, prelude::*, sync::Arc}; use t8140::{kernel_window as high, low}; mod v25g83; +// The M5 graph is built only once an M5 board has a page-table boot path. +#[allow(dead_code)] +mod m5; const EVENT_RING_RECHECK: u32 = 256; @@ -157,6 +160,7 @@ impl WorkPublication<'_> { /// Session-long storage and mappings. Aliases are released before their backing objects. pub(super) struct InitData { + m5: bool, // Live publication uses interior writes under the device mutex. This owner is Send but not // Sync, so safe host code cannot issue those writes concurrently through shared references. _single_publisher: core::marker::PhantomData>, @@ -186,7 +190,7 @@ pub(super) struct InitData { compute_scheduler: Option, compute_active: core::cell::Cell, event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], - completion_rings: [usize; 2], + completion_rings: [usize; 4], stamps: [usize; 2], render_pages: usize, render_runs: usize, @@ -262,6 +266,7 @@ impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { Self { + m5: false, _single_publisher: core::marker::PhantomData, aliases: KVec::new(), objects: KVec::new(), @@ -285,7 +290,7 @@ impl InitData { compute_scheduler: None, compute_active: core::cell::Cell::new(false), event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], - completion_rings: [0; 2], + completion_rings: [0; 4], stamps: [0; 2], render_pages: 0, render_runs: 0, @@ -304,6 +309,18 @@ impl InitData { Ok(self.metrics_fw_va) } + /// Names the M5 renders' metrics object to the firmware through a kernel-window alias. + pub(super) fn bind_m5_metrics(&mut self, uat: &mmu::Uat, metrics: &KernelObject) -> Result { + if !self.m5 || self.metrics_fw_va != 0 { + return Err(EINVAL); + } + let alias = metrics.alias_in(uat.kernel_vm(), uat.kernel_dynamic_range(), + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + self.metrics_fw_va = alias.iova(); + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { self.object(self.pb) } @@ -479,7 +496,7 @@ impl InitData { low::PAGE_POOL_DESCRIPTORS, self.object(self.page_pool)?.gpu_va(), ]; - let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + let base = self.main_config_offset(Role::Primary) + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); let bundle = self.object(self.bundle)?; let mut changed = false; @@ -559,7 +576,7 @@ impl InitData { return Err(EINVAL); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( @@ -662,7 +679,7 @@ impl InitData { return Err(EBUSY); } let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + self.work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), core::mem::size_of_val(record), )?; Ok(WorkPublication { @@ -716,7 +733,7 @@ impl InitData { |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel), + self.work_ring_offset(channel), channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ @@ -1340,12 +1357,25 @@ impl InitData { Ok(mappings) } + pub(super) fn is_m5(&self) -> bool { + self.m5 + } + + /// The effort value the firmware is given for a requested `effort`: M5 runs with 0. + pub(super) fn submission_effort(&self, effort: u32) -> u32 { + if self.m5 { 0 } + else { effort } + } + /// Writes the effort input in firmware order, then cleans its cache line to coherency. pub(super) fn publish_effort(&self, effort: u32) -> Result { let region = self.object(self.region_c)?; - let override_word = region.word(abi::RegionC::UNK_E8)?; - let effort_word = region.word(abi::RegionC::EFFORT)?; - let line = region.pointer(abi::RegionC::EFFORT, 4)?; + let (effort_offset, override_offset) = if self.m5 { + (super::fw::m5::power::EFFORT, super::fw::m5::power::EFFORT_OVERRIDE) + } else { (abi::RegionC::EFFORT, abi::RegionC::UNK_E8) }; + let override_word = region.word(override_offset)?; + let effort_word = region.word(effort_offset)?; + let line = region.pointer(effort_offset, 4)?; override_word.store(0, Ordering::Relaxed); effort_word.store(effort, Ordering::Relaxed); // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache @@ -1360,7 +1390,7 @@ impl InitData { pub(super) fn effort(&self) -> Result { Ok(self .object(self.region_c)? - .word(abi::RegionC::EFFORT)? + .word(if self.m5 { super::fw::m5::power::EFFORT } else { abi::RegionC::EFFORT })? .load(Ordering::Relaxed)) } @@ -1417,7 +1447,18 @@ impl InitData { Ok(opps) } + fn main_config_offset(&self, role: Role) -> usize { + if self.m5 { super::fw::m5::initdata::M5_MAIN_BUNDLE_OFFSETS[role as usize] } + else { abi::bundle::MAIN_CONFIG[role as usize] } + } + + fn work_ring_offset(&self, channel: usize) -> usize { + if self.m5 { super::fw::m5::initdata::M5_WORK_RING_BUNDLE_OFFSETS[channel] } + else { abi::work_ring_offset(channel) } + } + fn state_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATE_GRIDS[role as usize]; } match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, @@ -1425,6 +1466,7 @@ impl InitData { } fn status_offset(&self, role: Role) -> usize { + if self.m5 { return super::fw::m5::initdata::M5_PRIVATE_STATUS_A[role as usize]; } match role { Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, @@ -1445,7 +1487,7 @@ impl InitData { cache.set((checked.0, checked.1, checked.2 - 1)); return Ok(()); } - let offset = abi::bundle::MAIN_CONFIG[role as usize] + let offset = self.main_config_offset(role) + core::mem::offset_of!(abi::MainConfig, channels) + abi::EVENT_CHANNEL * size_of::(); let bundle = self.object(self.bundle)?; @@ -1495,7 +1537,7 @@ impl InitData { abi::ChannelState::PRODUCER, ]) { *counter = state - .word(abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::Relaxed); } Ok(counters) @@ -1503,12 +1545,12 @@ impl InitData { pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) } pub(super) fn control_producer(&self) -> Result<&AtomicU32> { self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) } /// Publishes a control record under the device lock. All checked views are acquired before @@ -1520,13 +1562,15 @@ impl InitData { asserts_power: bool, next: u32, ) -> Result { - let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + abi::MainConfig::CONTROL_RING - + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring_base = if self.m5 { + super::fw::m5::initdata::M5_CONTROL_RING_BUNDLE_OFFSETS[0] + } else { self.main_config_offset(Role::Primary) + abi::MainConfig::CONTROL_RING }; + let offset = ring_base + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; let ring = self.object(self.bundle)?.pointer(offset, record.len())?; let power = self .object(self.region_c)? - .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + .pointer(if self.m5 { super::fw::m5::power::POWER_ASSERT_TALLY } + else { abi::RegionC::POWER_ASSERT_TALLY }, 4)?; let producer = self.control_producer()?; // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. // Firmware only reads records after the producer store and the device mutex serializes @@ -1566,13 +1610,28 @@ impl InitData { } impl InitData { + fn recovery_state_offset(&self) -> usize { + if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_STATE + } else { self.status_b + abi::PrimaryStatusB::RECOVERY_STATE } + } + /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let extent = if self.m5 { + *super::fw::m5::initdata::M5_BUNDLE_VIEW_EXTENTS.get(view).ok_or(EINVAL)? + } else { + *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)? + }; // Separately allocated views live at offset 0 of their own object. let (object, base) = match self.views { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + bundle, + *super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS.get(view).ok_or(EINVAL)?, + ), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; let end = offset @@ -1581,9 +1640,8 @@ impl InitData { if end > extent { return Err(EINVAL); } - let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) - + view * size_of::(); + let pointer_at = self.main_config_offset(Role::Primary) + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1596,10 +1654,22 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { + let mask_offset = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_SLOT_MASK + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK + }; let mask = self .object(self.cluster)? - .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(mask_offset)? .load(Ordering::Relaxed); + // M5's slot-identity storage is not available through this mapping. + // Keep firmware's explicit blame when attribution evidence is missing, + // while still allowing the recovery handshake to complete. + if self.m5 { + return Ok(super::recovery::Sources::sample(mask as u32, |_| None)); + } let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { // Progress is independently available even when this firmware's key // address is unknown. Zero keeps ownership unknown and cannot override @@ -1625,21 +1695,26 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.recovery_state_offset())? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_HOST_RECOVERY + } else { + self.status_b + abi::PrimaryStatusB::HOST_RECOVERY + })? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.recovery_state_offset())?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1655,6 +1730,10 @@ impl super::recovery::Memory for InitData { // their own object. let (table, offset) = match self.views { Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None if self.m5 => ( + self.object(self.bundle)?, + super::fw::m5::initdata::M5_BUNDLE_VIEW_OFFSETS[4], + ), None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), }; // Validate the complete table before publishing its first live word. @@ -1678,9 +1757,13 @@ impl super::recovery::Memory for InitData { return Err(EINVAL); } let cluster = self.object(self.cluster)?; - let offset = self.status_b - + abi::PrimaryStatusB::RECOVERY_INFO - + index * core::mem::size_of::(); + let base = if self.m5 { + super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b + + super::fw::m5::initdata::M5_RECOVERY_INFO + } else { + self.status_b + abi::PrimaryStatusB::RECOVERY_INFO + }; + let offset = base + index * core::mem::size_of::(); fence(Ordering::Acquire); let flags = cluster .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs new file mode 100644 index 00000000000000..6103ea35f8e65a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 firmware graph backed by the shared G17 object and GPUVM owners. + +use super::*; +use crate::g17::fw::m5::initdata as m; + +const BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE: u64 = BASE + 0x2_0000; +const REGION_C: u64 = BASE + 0x1a_0000; +const ROOTS: u64 = BASE + 0x1a_8000; +const REGION_A: u64 = BASE + 0x1b_8000; +const FWCTL: u64 = BASE + 0x1c_0000; +const QOS: u64 = BASE + 0x1c_8000; +const AON: u64 = BASE + 0x1d_0000; +const KICK_SHARED: u64 = BASE + 0x1d_8000; +const BUNDLE: u64 = BASE + 0xc078_8000; +const COMPLETION_HIGH: u64 = BASE + 0xc080_0000; +const COMPLETION_LOW: u64 = 0x70_0184_8000; +const COMPLETION_STRIDE: u64 = 0x2_8000; +const PB: u64 = BASE + 0x15e_0000; +const UMA: u64 = BASE + 0x15e_8000; + +impl InitData { + /// Diagnostic-only read of an aligned word within the existing policy allocation. + /// Callers hold the device state mutex, retaining this mapping for the entire sample. + pub(crate) fn m5_policy_telemetry_word(&self, offset: usize) -> Result { + if !self.m5 || offset & 3 != 0 || offset.checked_add(4).is_none_or(|end| end > m::M5_POLICY_SIZE) { + return Err(EINVAL); + } + let at = m::M5_SHARED_LAYOUT.policy.checked_add(offset).ok_or(EINVAL)?; + Ok(self.object(self.bundle)?.word(at)?.load(Ordering::Relaxed)) + } + + pub(crate) fn note_m5_submission_history(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let private = self.object(self.cluster)?; + let base = m::M5_PRIVATE_LAYOUT.status_config; + for _ in 0..64 { + let generation = private.word(base + 0x74c)?.load(Ordering::Acquire); + if generation & 1 != 0 { core::hint::spin_loop(); continue; } + let config = private.read_words::<4>(base + 0x73c)?; + fence(Ordering::Acquire); + if generation != private.word(base + 0x74c)?.load(Ordering::Acquire) { + core::hint::spin_loop(); continue; + } + let period = u64::from(config[0]) | u64::from(config[1]) << 32; + let count = u64::from(config[2]) | u64::from(config[3]) << 32; + if period == 0 || count == 0 || count > 256 { return Err(EINVAL); } + let (counter, frequency) = crate::g17::qos::clock(); + let ticks = crate::g17::qos::time_from_host(counter, frequency)?; + // One slot per 25000 ticks of the period. + let index = (ticks % period) / 25_000; + if index >= count { return Err(ERANGE); } + let byte = private.pointer(base + 0x750 + index as usize, 1)?; + // SAFETY: retained, bounds-checked WC byte, written as a single byte + // after reading the even, stable generation, as the firmware expects. + unsafe { + let value = core::ptr::read_volatile(byte).wrapping_add(1); + core::ptr::write_volatile(byte, value); + }; + fence(Ordering::SeqCst); + return Ok(()); + } + Err(EBUSY) + } + + pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + id: &crate::regs::G17Id) -> Result { + use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, + PROT_GPU_SHARED_RW as GPU}; + self.m5 = true; + let fixed = |address, size, protection| KernelObject::new(dev, + uat.kernel_vm(), Placement::At(address), size, protection, CpuMap::WriteCombined); + let fixed_controller = |address, size| { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(address), size, + mmu::PROT_FW_PRIV_RW, CpuMap::WriteBack) + }; + let private = m::M5_PRIVATE_LAYOUT; + let shared = m::M5_SHARED_LAYOUT; + self.cluster = self.add(fixed(PRIVATE, private.total, RW)?)?; + self.region_c = self.add(fixed(REGION_C, mmu::UAT_PGSZ, RO)?)?; + self.roots = self.add(fixed(ROOTS, 0xc000, RO)?)?; + let region_a = self.add(fixed(REGION_A, mmu::UAT_PGSZ, RO)?)?; + self.add(fixed(FWCTL, mmu::UAT_PGSZ, RW)?)?; + self.qos = self.add(fixed(QOS, m::G17P_QOS_RESOURCE_SIZE, RW)?)?; + self.add(fixed(KICK_SHARED, m::M5_KICK_SHARED_SIZE, mmu::PROT_GPU_FW_SHARED_RW)?)?; + let aon = self.add(fixed_controller(AON, m::M5_AON_PTD_SIZE)?)?; + self.bundle = self.add(fixed_controller(BUNDLE, shared.total)?)?; + self.pb = self.add(fixed(PB, mmu::UAT_PGSZ, RW)?)?; + self.page_pool = self.add(fixed(UMA, mmu::UAT_PGSZ, RW)?)?; + for stage in 0..2 { + self.stamps[stage] = self.add(KernelObject::new(dev, uat.kernel_vm(), + Placement::In(uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?)?; + } + self.alias(self.pb, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[0], GPU)?; + self.alias(self.page_pool, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[1], GPU)?; + let mut completions = [m::G17PKsmCompletionAliases { low: 0, high: 0 }; m::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (index, pair) in completions.iter_mut().enumerate() { + pair.low = COMPLETION_LOW + index as u64 * COMPLETION_STRIDE; + pair.high = COMPLETION_HIGH + index as u64 * COMPLETION_STRIDE; + // After initialization the host only reads completion records. + // Match the standard G17 completion ring's CPU write-back mapping; + // keep firmware/GPU aliases, size and lifetime unchanged. + let backing = if index == 0 + { + KernelObject::new(dev, uat.kernel_vm(), Placement::At(pair.high), + m::M5_25G76_COMPLETION_BACKING_SIZE, RW, CpuMap::WriteBack)? + } else { + fixed(pair.high, m::M5_25G76_COMPLETION_BACKING_SIZE, RW)? + }; + let object = self.add(backing)?; + self.completion_rings[index] = object; + self.alias(object, uat.kernel_lower_vm(), pair.low, GPU)?; + } + m::encode_common26_aon_ptd(&mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) + .map_err(|_| EINVAL)?; + m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; + m::encode_m5_private_cluster(PRIVATE, FWCTL, + &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; + let windows = m::j714_register_windows(); + for (index, window) in windows.iter().enumerate() { + let spec = m::j714_register_window_spec(index); + let stride = (u64::from(window.size) + mmu::UAT_PGMSK as u64) & !(mmu::UAT_PGMSK as u64); + for (instance, physical) in spec.instances.iter().copied().enumerate() { + let offset = physical & mmu::UAT_PGMSK as u64; + let address = window.device_va.checked_sub(offset).ok_or(EINVAL)? + + instance as u64 * stride; + let size = (offset + u64::from(window.size) + mmu::UAT_PGMSK as u64) + & !(mmu::UAT_PGMSK as u64); + self.aliases.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), address, + (physical & !(mmu::UAT_PGMSK as u64)).try_into()?, size.try_into()?, + mmu::PROT_FW_MMIO_RW)?, GFP_KERNEL)?; + } + } + let addresses = m::SharedAddresses { + region_a: self.object(region_a)?.gpu_va(), region_c: REGION_C, + hw_data_bundle: BUNDLE, hw_data_bundle_alloc: shared.total, + }; + let views = m::PrimaryRegionViews { + sentinel_high: BASE + 0xc07e_0000, alias_high: [PB, UMA], + }; + m::encode_m5_hw_data(completions, QOS, &windows, + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + m::set_m5_kick_shared( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) + .map_err(|_| EINVAL)?; + let masked_cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let enabled_cores = if masked_cores != 0 { masked_cores } + else { id.num_clusters.checked_mul(id.num_cores).ok_or(EOVERFLOW)? }; + m::set_m5_enabled_cores( + &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], enabled_cores) + .map_err(|_| EINVAL)?; + m::encode_m5_bundle_policy(m::M5PolicyBindings { + state: PRIVATE + private.policy_state as u64, aon_ptd: AON, boot_timestamp: 0, + }, self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let calibration: Result> = node.get_property(c_str!("apple,leakage-calibration-words")); + if calibration.as_ref().is_ok_and(|words| words.as_slice() == [0xd2035701, 0xac3a1280, 0x0a281d81]) { + m::encode_m5_policy_controller_defaults(self.object_mut(self.bundle)?.bytes_mut()) + .map_err(|_| EINVAL)?; + let maximum = self.object(self.bundle)?.word(shared.policy + m::M5_POLICY_MAXIMUM_POWER)? + .load(Ordering::Relaxed); + m::set_m5_region_maximum_power(self.object_mut(self.region_c)?.bytes_mut(), maximum) + .map_err(|_| EINVAL)?; + } + for (index, role) in [m::InstanceRole::Primary, m::InstanceRole::Secondary].into_iter().enumerate() { + let instance = m::InstanceAddresses { + root: ROOTS + index as u64 * 0x8000, + main_config: BUNDLE + shared.main_config[index] as u64, + status_a: PRIVATE + private.roles[index].status_a as u64, + status_b: if index == 0 { PRIVATE + private.status_b as u64 } else { 0 }, + secondary_extras: if index == 0 { [0; 2] } else { + [PRIVATE + private.status_config as u64, PRIVATE + private.secondary_extra_1 as u64] + }, + }; + let profile = m::RootProfile::J714G17S26A428; + let channels = m::derive_channel_table_for_profile(profile, role, + PRIVATE + m::M5_PRIVATE_STATE_GRIDS[index] as u64, + instance.status_a, instance.main_config, BUNDLE); + let offset = shared.main_config[index]; + m::encode_main_config_for_profile(profile, role, &addresses, &instance, + &channels, if index == 0 { Some(&views) } else { None }, + &mut self.object_mut(self.bundle)?.bytes_mut()[offset..offset + m::MAIN_CONFIG_SIZE]) + .map_err(|_| EINVAL)?; + let offset = index * 0x8000; + m::encode_root_for_profile(profile, role, &addresses, &instance, + &mut self.object_mut(self.roots)?.bytes_mut()[offset..offset + m::M5_ROOT_SIZE]) + .map_err(|_| EINVAL)?; + } + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "M5 firmware graph ready: private {:#x}+{:#x}, shared {:#x}+{:#x}\n", + PRIVATE, private.total, BUNDLE, shared.total); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index dda1f1564fdf23..4762ee813f2f0f 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -764,11 +764,14 @@ impl Firmware { port: &port::Port, effort: u32, ) -> Result { - if state.applied() == effort && init.effort()? == effort { + // The state machine and the performance-controller demand follow the requested effort; + // the firmware is given the M5 value (InitData::submission_effort). + let published = init.submission_effort(effort); + if state.applied() == effort && init.effort()? == published { return Ok(()); } - init.publish_effort(effort)?; - trace::message(trace::EFFORT, u64::from(effort)); + init.publish_effort(published)?; + trace::message(trace::EFFORT, u64::from(published)); state.did_apply(effort); port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) @@ -909,6 +912,8 @@ struct Shared { deflake: fw::render::Deflake, compute_blocks: usize, descriptor_flags: [u32; 2], + /// Firmware events and work run: off for an M5 GPU until the M5 queues take submission. + submission_enabled: bool, crashed: AtomicBool, feed: Arc, #[pin] @@ -1078,6 +1083,9 @@ impl Shared { /// during a pass. The pass skips the housekeeping that nothing needs (see /// `runtime_event_turn`). Callable from any context. fn queue_event_records(self: &Arc) { + if !self.submission_enabled { + return; + } // Already queued if this fails, which is just as good. let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); } @@ -1233,7 +1241,12 @@ impl Gpu { ias: cfg.uat_ias, oas: cfg.uat_oas, }; - let max_frequency_khz = Self::max_frequency_khz(dev)?; + let m5 = cfg.profile.is_m5(); + let max_frequency_khz = if m5 { + fw::m5::initdata::J714_GPU_PERF_STATES[13].0 * 1000 + } else { + Self::max_frequency_khz(dev)? + }; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; let idle_delay = power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; @@ -1291,6 +1304,7 @@ impl Gpu { deflake: cfg.core_layout.deflake, compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], + submission_enabled: !m5, crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), From 63f20eb366733d0d0255684932ebd914570d9c2c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:11 -0800 Subject: [PATCH 064/171] drm/asahi: g17: M5: back M5 render and compute pools with the M5 free-list layout Add the M5 pool descriptor and free-list encodings, the render storage demand they are sized from, and M5 render and compute pools beside the existing ones. Render pool identities become leases that can move between logical contexts, so idle storage is reused instead of rebuilt. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..77dc6eb34210 (4 commits); SPTM/APIF removed; adapted to integration's freelist (capacity refusals, lower-window placement, per-context render pools); comments and names reworded; original 6613f5ab7b7e, e25ba8a6fb2b, cda233bc91f7, 77dc6eb34210 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit 694aee53b75df47ec0988a7f1d1e8f0361e9d7ac in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/freelist.rs | 681 +++++++++++++++--- .../gpu/drm/asahi/g17/freelist/render_ids.rs | 246 +++++++ drivers/gpu/drm/asahi/g17/fw/freelist.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 5 + drivers/gpu/drm/asahi/g17/fw/m5/pool.rs | 265 +++++++ .../gpu/drm/asahi/g17/fw/m5/pool_budget.rs | 151 ++++ .../gpu/drm/asahi/g17/fw/m5/render_demand.rs | 290 ++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +- 9 files changed, 1538 insertions(+), 108 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/freelist/render_ids.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/pool.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 9015dc82e40765..49d00180363ac5 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -11,31 +11,206 @@ use super::fw::freelist::*; use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; -use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU16, AtomicU64, Ordering}; use kernel::{ new_mutex, prelude::*, sync::{Arc, Mutex}, }; +mod render_ids; + +const NO_RENDER_ID: u16 = u16::MAX; +static NEXT_RENDER_OWNER: AtomicU64 = AtomicU64::new(1); + +struct RenderIdCell { + owner: u64, + slot: AtomicU16, + generation: AtomicU64, + closed: AtomicBool, + storage: Arc, +} +impl RenderIdCell { + fn has_storage(&self) -> bool { + self.storage.backing.lock().is_some() || self.storage.spare.lock().is_some() + } +} + +/// Detached after the device's final free-list release. Drop outside the +/// device and identity locks because mappings may take memory-management locks. +#[must_use] +pub(crate) struct ReclaimedRenderId { + _backing: Option, + spare: Option, +} +impl ReclaimedRenderId { + /// Caller is outside the device lock. Remove the previous client's alias + /// before handing physical storage to another logical pool. + fn into_pages(self) -> Option { + self._backing.map(RenderBacking::into_pages).or(self.spare) + } +} + +struct RenderIdState { + leases: render_ids::State, + cells: [Option>; render_ids::COUNT], +} + /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. +#[pin_data] pub(crate) struct RenderIds { used: [AtomicU64; 2], + available: Arc, + #[pin] + state: Mutex, } impl RenderIds { - pub(crate) const fn new() -> Self { - Self { + #[inline(never)] + pub(crate) fn new() -> Result> { + Arc::pin_init(try_pin_init!(Self { used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + available: super::availability::Availability::new()?, + state <- new_mutex!(RenderIdState { + leases: render_ids::State::new(), + cells: [const { None }; render_ids::COUNT], + }, "G17 render identities"), + }), GFP_KERNEL) + } + + pub(crate) fn availability(&self) -> &Arc { + &self.available + } + + fn pin(self: &Arc, cell: &Arc, assigned: impl FnOnce()) -> Result { + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot != NO_RENDER_ID { + let lease = render_ids::Lease { + slot, owner: cell.owner, generation: cell.generation.load(Ordering::Acquire), + }; + match state.leases.pin(lease) { + Ok(_) => return Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease }), + Err(render_ids::Error::Overflow) => return Err(EOVERFLOW), + Err(_) => {} + } + } + let claim = state.leases.claim(cell.owner, |_| false).map_err(|error| match error { + render_ids::Error::Busy | render_ids::Error::PendingRelease => ENOSPC, + render_ids::Error::Overflow => EOVERFLOW, + _ => EIO, + })?; + let index = usize::from(claim.lease.slot - render_ids::FIRST); + state.cells[index] = Some(cell.clone()); + assigned(); + cell.generation.store(claim.lease.generation, Ordering::Relaxed); + cell.slot.store(claim.lease.slot, Ordering::Release); + Ok(RenderIdPin { allocator: self.clone(), cell: cell.clone(), lease: claim.lease }) + } + + fn close(&self, cell: &Arc) { + cell.closed.store(true, Ordering::Release); + let mut state = self.state.lock(); + let slot = cell.slot.load(Ordering::Acquire); + if slot == NO_RENDER_ID { return; } + let index = usize::from(slot - render_ids::FIRST); + if state.cells[index].as_ref().is_some_and(|owner| Arc::ptr_eq(owner, cell)) + && state.leases.remove(cell.owner, |_| true).is_ok() { + let old = state.cells[index].take(); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + drop(state); + drop(old); + self.available.notify(); } } + + /// Caller holds the device mutex and supplies the exact final release proof. + pub(crate) fn reclaim_one(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(false, released) + } + /// Reuse backing for a pool that already has an ID, without evicting an + /// empty metadata-only owner. Caller supplies the same release proof. + pub(crate) fn reclaim_backing(&self, released: impl FnMut(u16) -> bool) -> Option { + self.reclaim(true, released) + } + fn reclaim(&self, only_storage: bool, mut released: impl FnMut(u16) -> bool) -> Option { + let mut state = self.state.lock(); + let RenderIdState { leases, cells } = &mut *state; + let mut select = |require_storage: bool| leases.reclaim_one(|lease| { + cells[usize::from(lease.slot - render_ids::FIRST)].as_ref().is_some_and(|cell| + cell.owner == lease.owner + && cell.slot.load(Ordering::Acquire) == lease.slot + && cell.generation.load(Ordering::Acquire) == lease.generation + && (!require_storage || cell.has_storage())) + && released(lease.slot) + }); + let lease = select(true).or_else(|| if only_storage { None } else { select(false) })?; + let old = state.cells[usize::from(lease.slot - render_ids::FIRST)].take(); + let (backing, spare) = if let Some(cell) = old.as_ref() { + let backing = cell.storage.backing.lock().take(); + let spare = cell.storage.spare.lock().take(); + cell.storage.page_list.store(0, Ordering::Release); + cell.slot.store(NO_RENDER_ID, Ordering::Release); + (backing, spare) + } else { (None, None) }; + drop(state); + drop(old); + self.available.notify(); + Some(ReclaimedRenderId { _backing: backing, spare }) + } } -struct RenderId { - slot: u16, + +pub(super) struct RenderIdPin { + allocator: Arc, + cell: Arc, + lease: render_ids::Lease, +} +impl Drop for RenderIdPin { + fn drop(&mut self) { + let available = self.allocator.state.lock().leases.unpin(self.lease).unwrap_or(false); + if self.cell.closed.load(Ordering::Acquire) { self.allocator.close(&self.cell); } + if available { self.allocator.available.notify(); } + } +} + +enum RenderIdKind { + Fixed(u16), + Rebindable(Arc), +} +pub(super) struct RenderId { + kind: RenderIdKind, allocator: Arc, } impl RenderId { - fn new(allocator: &Arc) -> Result { + pub(super) fn slot(&self) -> u16 { + match &self.kind { + RenderIdKind::Fixed(slot) => *slot, + RenderIdKind::Rebindable(cell) => cell.slot.load(Ordering::Acquire), + } + } + pub(super) fn logical(allocator: &Arc) -> Result { + let owner = NEXT_RENDER_OWNER.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |next| next.checked_add(1)).map_err(|_| EOVERFLOW)?; + let storage = RenderStorage::new(None)?; + Ok(Self { kind: RenderIdKind::Rebindable(Arc::new(RenderIdCell { + owner, slot: AtomicU16::new(NO_RENDER_ID), generation: AtomicU64::new(0), + closed: AtomicBool::new(false), + storage, + }, GFP_KERNEL)?), allocator: allocator.clone() }) + } + fn storage(&self) -> Option> { + match &self.kind { + RenderIdKind::Rebindable(cell) => Some(cell.storage.clone()), + RenderIdKind::Fixed(_) => None, + } + } + fn pin(&self, assigned: impl FnOnce()) -> Result> { + match &self.kind { + RenderIdKind::Fixed(_) => Ok(None), + RenderIdKind::Rebindable(cell) => self.allocator.pin(cell, assigned).map(Some), + } + } + pub(super) fn new(allocator: &Arc) -> Result { for (bank, used) in allocator.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { @@ -51,7 +226,7 @@ impl RenderId { ) { Ok(_) => { return Ok(Self { - slot: 129 + bank as u16 * 64 + bit as u16, + kind: RenderIdKind::Fixed(129 + bank as u16 * 64 + bit as u16), allocator: allocator.clone(), }) } @@ -64,11 +239,29 @@ impl RenderId { } impl Drop for RenderId { fn drop(&mut self) { - let bit = self.slot as usize - 129; - self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + match &self.kind { + RenderIdKind::Fixed(slot) => { + let bit = *slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } + RenderIdKind::Rebindable(cell) => self.allocator.close(cell), + } } } +/// Profile-specific descriptor data for a fresh physical pool. The runtime +/// installs this only before publication, or after an acknowledged release. +pub(crate) enum PoolDescriptor { + Neo { slot: u16, page_list: u64 }, + M5 { slot: u16, words: [u64; 8] }, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PageLayout { + Neo { blocks: usize, runs: RunLayout }, + M5(super::fw::m5::pool::PoolGeometry), +} + /// Free-list backing is by far the largest user of the shared lower GPU VA /// window, and placing it populates every page before the window is searched. /// After a refused placement, builders waiting for capacity would allocate and @@ -125,8 +318,7 @@ fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { pub(crate) struct Pages { backing: KernelObject, - blocks: usize, - layout: RunLayout, + layout: PageLayout, } impl Drop for Pages { fn drop(&mut self) { @@ -134,7 +326,7 @@ impl Drop for Pages { } } impl Pages { - fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + fn new(alloc: &Allocator<'_>, blocks: usize, runs: RunLayout) -> Result { let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; let backing = place_lower(size, || { alloc.lower( @@ -144,15 +336,14 @@ impl Pages { CpuMap::WriteCombined, ) })?; - Self::from_backing(backing, blocks, layout) + Self::from_backing(backing, blocks, runs) } - fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + fn from_backing(mut backing: KernelObject, blocks: usize, runs: RunLayout) -> Result { let base = backing.gpu_va(); - backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + backing.initialize::(0, |tables| tables.init(base, blocks, runs))?; Ok(Self { backing, - blocks, - layout, + layout: PageLayout::Neo { blocks, runs }, }) } /// Fresh compute backing at the admitted capacity, built off-lock. @@ -162,21 +353,91 @@ impl Pages { } Self::new(alloc, blocks, RunLayout::Compute) } + /// Fresh zeroed compute backing, built without the device mutex. M5 keeps + /// its directory, block list and initial 32 MiB in one retained allocation. + pub(crate) fn compute(alloc: &Allocator<'_>, m5: bool, required: Option) -> Result { + if !m5 { return Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute); } + use super::fw::m5::pool; + let geometry = match required { Some(geometry) => geometry, + None => pool::PoolGeometry::g17s(8).map_err(|_| EINVAL)?, }; + Self::m5(alloc, geometry) + } + fn m5(alloc: &Allocator<'_>, geometry: super::fw::m5::pool::PoolGeometry) -> Result { + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let size = geometry.blocks.checked_mul(geometry.block_bytes()) + .and_then(|n| n.checked_add(tables)).ok_or(EOVERFLOW)?; + let backing = alloc.lower(size, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined)?; + let mut pages = Self { backing, layout: PageLayout::M5(geometry) }; + pages.populate_m5()?; + Ok(pages) + } + fn populate_m5(&mut self) -> Result { + use super::fw::m5::pool; + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; + let base = self.backing.gpu_va(); + let mut blocks = KVec::with_capacity(geometry.blocks, GFP_KERNEL)?; + for index in 0..geometry.blocks { + blocks.push(base + (tables + index * geometry.block_bytes()) as u64, GFP_KERNEL)?; + } + pool::encode_directory(geometry, &blocks, + &mut self.backing.bytes_mut()[..geometry.directory_bytes]).map_err(|_| EINVAL)?; + pool::encode_freelist(geometry, &blocks, + &mut self.backing.bytes_mut()[geometry.directory_bytes..tables]).map_err(|_| EINVAL)?; + Ok(()) + } + fn reset_for_client(&mut self) -> Result { + // Released storage may contain the previous client's private data. + // Clear it before exposing an alias in a different client VM. + self.backing.bytes_mut().fill(0); + self.populate_m5()?; + fence(Ordering::SeqCst); + Ok(()) + } pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - pub(crate) fn page_list_va(&self) -> u64 { - self.backing.gpu_va() + fn fits(&self, required: Option) -> bool { + match (self.layout, required) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (_, None) => true, + _ => false, + } } + /// Block capacity of block-sized (non-M5) backing; M5 backing is sized by + /// its geometry and never matches. + fn fits_blocks(&self, required: usize) -> bool { + matches!(self.layout, PageLayout::Neo { blocks, .. } if blocks >= required) + } + fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 + } + fn m5_layout(&self, slot: u16) -> Result { + let PageLayout::M5(geometry) = self.layout else { return Err(EINVAL); }; + Ok(super::fw::m5::pool::PoolLayout { geometry, directory: self.page_list_va(), + freelist: self.run_list_va(), hardware_id: u32::from(slot) }) + } + fn descriptor(&self, slot: u16) -> Result { + if self.is_m5() { + let words = super::fw::m5::pool::initial_uma_descriptor(self.m5_layout(slot)?) + .map_err(|_| EINVAL)?; + Ok(PoolDescriptor::M5 { slot, words }) + } else { Ok(PoolDescriptor::Neo { slot, page_list: self.page_list_va() }) } } fn repopulate(&self) -> Result { + let PageLayout::Neo { blocks, runs } = self.layout else { return Err(EINVAL); }; let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); PoolTables::for_each_word( self.page_list_va(), - self.blocks, - self.layout, + blocks, + runs, |index, value| { // SAFETY: The page-aligned backing was validated for the complete // tables. The enumerator bounds every index inside those tables; @@ -194,11 +455,17 @@ static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); /// both aliases before the backing. pub(crate) struct RenderBacking { _client: mmu::KernelMapping, - _global: mmu::KernelMapping, + _global: Option, pages: Pages, } impl RenderBacking { + fn into_pages(self) -> Pages { + let Self { _client, _global, pages } = self; + drop(_client); + drop(_global); + pages + } /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, @@ -222,10 +489,37 @@ impl RenderBacking { let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, - _global: global, + _global: Some(global), pages, }) } + /// Fresh M5 backing, aliased into the client VM only: M5 firmware uses + /// the canonical and client mappings of these pages. + fn m5(alloc: &Allocator<'_>, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + let pages = Pages::m5(alloc, geometry)?; + let client = pages.map_client(client)?; + Ok(Self { _client: client, _global: None, pages }) + } +} + +#[pin_data] +struct RenderStorage { + #[pin] + backing: Mutex>, + #[pin] + spare: Mutex>, + page_list: AtomicU64, +} +impl RenderStorage { + fn new(backing: Option) -> Result> { + let address = backing.as_ref().map_or(0, |backing| backing.pages.page_list_va()); + Arc::pin_init(try_pin_init!(Self { + backing <- new_mutex!(backing, "G17 render pool backing"), + spare <- new_mutex!(None, "G17 render pool spare"), + page_list: AtomicU64::new(address), + }), GFP_KERNEL) + } } /// A render pool retained by a logical context and every graph referencing it. @@ -233,10 +527,7 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { - blocks: usize, - backing: Arc>>, - /// Page-list VA of the installed backing; zero while vacant. - page_list: AtomicU64, + storage: Arc, /// Counts installed backings, so a re-armed backing is a new free-list /// owner even if its addresses repeat. epoch: AtomicU64, @@ -247,9 +538,26 @@ pub(crate) struct RenderPool { served: AtomicBool, serial: u64, id: RenderId, + layout: PageLayout, +} + +/// Keeps both a physical identity and its logical pool alive during use. +/// Drop the identity pin before releasing the retained pool reference. +pub(crate) struct PoolPin { + _identity: RenderIdPin, + _pool: Arc, } impl RenderPool { + /// The caller retains this guard across construction and all firmware use. + pub(crate) fn pin_id(self: &Arc) -> Result> { + let pin = self.id.pin(|| { + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + })?; + Ok(pin.map(|pin| PoolPin { _identity: pin, _pool: self.clone() })) + } + pub(crate) fn new( alloc: &Allocator<'_>, ids: &Arc, @@ -259,6 +567,31 @@ impl RenderPool { ) -> Result { let id = RenderId::new(ids)?; let backing = RenderBacking::new(alloc, render_global, client, blocks)?; + Self::with_backing(alloc, id, client, backing) + } + + /// Explicit initial geometry; allocation policy remains with the caller. + /// M5 firmware uses the canonical and client mappings of these pages. + pub(super) fn m5(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + geometry: super::fw::m5::pool::PoolGeometry) -> Result { + if !alloc.uat.is_m5() { return Err(EINVAL); } + // Logical metadata is cheap; physical backing is installed only once + // preparation owns both an identity pin and admission capacity. + let storage = id.storage().ok_or(EINVAL)?; + let state = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let control = alloc.kernel(CONTROL_SIZE, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let serial = RENDER_POOL_SERIAL.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |value| value.checked_add(1)).map_err(|_| EOVERFLOW)? + 1; + Ok(Self { storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + control, state, served: AtomicBool::new(true), serial, id, + layout: PageLayout::M5(geometry) }) + } + + fn with_backing(alloc: &Allocator<'_>, id: RenderId, client: &mmu::Vm, + backing: RenderBacking) -> Result { + let layout = backing.pages.layout; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -273,32 +606,33 @@ impl RenderPool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::render( - &FreeListArgs { - buffer_slot: u32::from(id.slot), - class: FreeListClass::Two, - page_list_va: address, - run_list_va, - blocks: blocks as u32, - state_va: state.gpu_va(), - }, + match layout { + PageLayout::M5(_) => { + super::fw::m5::pool::encode_g17s_render_state(backing.pages.m5_layout(id.slot())?, + state.gpu_va(), control.bytes_mut()).map_err(|_| EINVAL)?; + } + PageLayout::Neo { blocks, .. } => control.write( 0, - ), - )?; + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot()), + class: FreeListClass::Two, + page_list_va: address, + run_list_va, + blocks: blocks as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?, + } fence(Ordering::SeqCst); let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; - let backing = Arc::pin_init( - new_mutex!(Some(backing), "G17 render pool backing"), - GFP_KERNEL, - )?; + let storage = RenderStorage::new(Some(backing))?; Ok(Self { - blocks, - backing, - page_list: AtomicU64::new(address), + storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), control, @@ -306,12 +640,16 @@ impl RenderPool { served: AtomicBool::new(false), serial, id, + layout, }) } /// No backing is installed: the pool must be re-armed before staging. pub(crate) fn vacant(&self) -> bool { - self.page_list.load(Ordering::Acquire) == 0 + self.storage.page_list.load(Ordering::Acquire) == 0 + } + pub(crate) fn needs_storage(&self) -> bool { + self.vacant() && self.storage.spare.lock().is_none() } /// Pressure reclaim evicted the client VM, which is idle. pub(crate) fn client_evicted(&self) -> bool { @@ -322,9 +660,9 @@ impl RenderPool { /// next staging requires a re-armed backing. The caller drops the result /// after releasing the device mutex. pub(crate) fn vacate(&self) -> Option { - let backing = self.backing.lock().take(); + let backing = self.storage.backing.lock().take(); if backing.is_some() { - self.page_list.store(0, Ordering::Release); + self.storage.page_list.store(0, Ordering::Release); } backing } @@ -332,7 +670,7 @@ impl RenderPool { /// The next staging rewrites the control header and descriptor row for it /// (its run-list address changes the free-list owner identity). pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { - let mut backing = self.backing.lock(); + let mut backing = self.storage.backing.lock(); if backing.is_some() { return Ok(()); } @@ -340,9 +678,24 @@ impl RenderPool { *backing = Some(fresh); self.epoch.fetch_add(1, Ordering::AcqRel); self.served.store(true, Ordering::Release); - self.page_list.store(address, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); Ok(()) } + /// Keep released physical storage for the incoming owner, without keeping + /// the old VM alias or assigning backing to every logical context. + pub(crate) fn cache_reclaimed(&self, reclaimed: ReclaimedRenderId) { + let pages = reclaimed.into_pages(); + let backing = self.storage.backing.lock(); + let mut spare = self.storage.spare.lock(); + let mut pages = pages; + if backing.is_none() && spare.is_none() + && pages.as_ref().is_some_and(|pages| pages.layout == self.layout) { + *spare = pages.take(); + } + drop(spare); + drop(backing); + drop(pages); + } /// Re-arms a vacant pool with fresh backing built here, without the device /// mutex. `client` is the VM of the context that owns this pool. pub(crate) fn ensure_backing( @@ -351,10 +704,31 @@ impl RenderPool { render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - if !self.vacant() { - return Ok(()); - } - self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) + let geometry = match self.layout { + PageLayout::M5(geometry) => geometry, + PageLayout::Neo { blocks, .. } => { + if !self.vacant() { + return Ok(()); + } + return self.rearm(RenderBacking::new(alloc, render_global, client, blocks)?); + } + }; + let mut backing = self.storage.backing.lock(); + if backing.is_some() { return Ok(()); } + let spare = self.storage.spare.lock().take(); + let fresh = if let Some(mut pages) = spare { + pages.reset_for_client()?; + let alias = pages.map_client(client)?; + RenderBacking { _client: alias, _global: None, pages } + } else { + RenderBacking::m5(alloc, client, geometry)? + }; + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.storage.page_list.store(address, Ordering::Release); + Ok(()) } pub(crate) fn serial(&self) -> u64 { @@ -364,21 +738,26 @@ impl RenderPool { self.epoch.load(Ordering::Acquire) } pub(crate) fn id(&self) -> u16 { - self.id.slot + self.id.slot() } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.page_list.load(Ordering::Acquire) + self.storage.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 + self.page_list_va() + match self.layout { + PageLayout::Neo { .. } => PAGE_LIST_SIZE, + PageLayout::M5(g) => g.directory_bytes, + } as u64 } pub(crate) fn end_va(&self) -> u64 { - self.run_list_va() - + (self.blocks * size_of::()) as u64 + self.run_list_va() + match self.layout { + PageLayout::Neo { blocks, .. } => blocks * size_of::(), + PageLayout::M5(g) => g.blocks * 8, + } as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -388,13 +767,34 @@ impl RenderPool { return Ok(()); } let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let blocks = match self.layout { + PageLayout::Neo { blocks, .. } => blocks, + PageLayout::M5(_) => { + let backing = self.storage.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { return Err(EIO); } + let mut bytes = [0; super::fw::m5::pool::STATE_BYTES]; + super::fw::m5::pool::encode_g17s_render_state(pages.m5_layout(self.id())?, + self.state.gpu_va(), &mut bytes).map_err(|_| EINVAL)?; + bytes[super::fw::m5::pool::COMPLETED..super::fw::m5::pool::COMPLETED + 8] + .copy_from_slice(&submitted.to_le_bytes()); + let raw = self.control.pointer(0, bytes.len())?; + for (index, byte) in bytes.into_iter().enumerate() { + // SAFETY: The whole retained range was checked and the caller + // observed final release before installing fresh pool backing. + unsafe { raw.add(index).write_volatile(byte) }; + } + fence(Ordering::SeqCst); + return Ok(()); + } + }; let header = FreeListControl::render( &FreeListArgs { buffer_slot: u32::from(self.id()), class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: self.blocks as u32, + blocks: blocks as u32, state_va: self.state.gpu_va(), }, submitted, @@ -409,7 +809,7 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - let backing = self.backing.lock(); + let backing = self.storage.backing.lock(); let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; if pages.page_list_va() != self.page_list_va() { return Err(EIO); @@ -428,7 +828,9 @@ impl RenderPool { /// counter, so retain the byte accesses required by that interface. pub(crate) fn counters(&self) -> Result<[u64; 2]> { let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let offset = if self.is_m5() { super::fw::m5::pool::COMPLETED } + else { FreeListControl::COMPLETED }; + let completed = self.control.pointer(offset, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { // SAFETY: The checked eight-byte range stays mapped; byte accesses @@ -438,6 +840,47 @@ impl RenderPool { Ok([submitted, u64::from_le_bytes(bytes)]) } + pub(crate) fn fits(&self, geometry: Option) -> bool { + match (self.layout, geometry) { + (PageLayout::M5(current), Some(required)) => + current.block_bytes() == required.block_bytes() && current.blocks >= required.blocks, + (PageLayout::Neo { .. }, None) => true, + _ => false, + } + } + + pub(crate) fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } + + fn m5_state(&self) -> Result { + if !self.is_m5() { return Err(EINVAL); } + let mut state = super::fw::m5::pool::PoolState([0; super::fw::m5::pool::STATE_BYTES]); + let raw = self.control.pointer(0, state.0.len())?; + fence(Ordering::Acquire); + for (index, byte) in state.0.iter_mut().enumerate() { + // SAFETY: The entire retained header range was checked above. + *byte = unsafe { raw.add(index).read_volatile() }; + } + Ok(state) + } + + pub(crate) fn m5_descriptor(&self) -> Result<[u64; 8]> { Ok(self.m5_state()?.descriptor()) } + + pub(crate) fn store_m5_descriptor(&self, words: [u64; 8]) -> Result { + let mut state = self.m5_state()?; + if !state.matches(&words) { return Err(EIO); } + state.restore(words); + for offset in super::fw::m5::pool::SAVED_FIELDS { + let raw = self.control.pointer(offset, 4)?; + for index in 0..4 { + // SAFETY: Each mutable field was checked; the caller owns the + // retired descriptor, and immutable header bytes stay intact. + unsafe { raw.add(index).write_volatile(state.0[offset + index]) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + /// Snapshot the unaligned shared fields before loading their descriptor. pub(crate) fn page_pool_state(&self) -> Result { let raw = self.control.pointer(0x14, 0x1c)?; @@ -516,18 +959,31 @@ pub(crate) struct ComputePool { control: KernelObject, state: KernelObject, id: u16, + m5: bool, } impl ComputePool { - pub(crate) fn fits(&self, blocks: usize) -> bool { - self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + /// Block-sized (non-M5) capacity check for admitted growth. + pub(crate) fn fits_blocks(&self, blocks: usize) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits_blocks(blocks)) } - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, m5: bool, required: Option) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let pages = Pages::compute(alloc, m5, required)?; + Self::with_pages(alloc, qid, m5, pages) + } + /// A block-sized (non-M5) pool at the admitted capacity. + pub(crate) fn sized(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } - let id = u16::from(qid) + 1; let pages = Pages::compute_sized(alloc, blocks)?; + Self::with_pages(alloc, qid, false, pages) + } + fn with_pages(alloc: &Allocator<'_>, qid: u8, m5: bool, pages: Pages) -> Result { + let id = u16::from(qid) + 1; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -540,24 +996,47 @@ impl ComputePool { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - control.write( - 0, - FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: blocks as u32, - state_va: state.gpu_va(), - }), - )?; + Self::initialize_control(&mut control, &pages, id, state.gpu_va(), [0; 8])?; fence(Ordering::SeqCst); - Ok(Self { - pages: Some(pages), - control, - state, - id, - }) + Ok(Self { pages: Some(pages), control, state, id, m5 }) + } + fn initialize_control(control: &mut KernelObject, pages: &Pages, id: u16, + submitted: u64, completed: [u8; 8]) -> Result { + match pages.layout { + PageLayout::M5(_) => { + use super::fw::m5::pool; + pool::encode_g17s_compute_state(pages.m5_layout(id)?, submitted, + &mut control.bytes_mut()[..pool::STATE_BYTES]).map_err(|_| EINVAL)?; + control.bytes_mut()[pool::COMPLETED..pool::COMPLETED + 8].copy_from_slice(&completed); + Ok(()) + } + PageLayout::Neo { blocks, .. } => { + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), class: FreeListClass::One, + page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), + blocks: blocks as u32, state_va: submitted, + }); + header.unk_54[..8].copy_from_slice(&completed); + control.write(0, header) + } + } + } + pub(crate) fn descriptor(&self) -> Result { + self.pages.as_ref().ok_or(EAGAIN)?.descriptor(self.id) + } + pub(crate) fn completed_submissions(&self) -> Result { + Ok(u64::from_le_bytes(self.completed_bytes()?)) + } + fn completed_bytes(&self) -> Result<[u8; 8]> { + let offset = if self.m5 { super::fw::m5::pool::COMPLETED } else { FreeListControl::COMPLETED }; + let pointer = self.control.pointer(offset, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked packed counter remains mapped. Firmware may + // update it; callers use it only with the queue's completion proof. + *byte = unsafe { pointer.add(index).read_volatile() }; + } + Ok(bytes) } pub(crate) fn id(&self) -> u16 { self.id @@ -569,6 +1048,9 @@ impl ComputePool { pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn fits(&self, required: Option) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits(required)) + } pub(crate) fn populated(&self) -> bool { self.pages.is_some() } @@ -602,23 +1084,9 @@ impl ComputePool { Ok(()) } fn write_pages(&mut self, pages: &Pages) -> Result { - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().enumerate() { - // SAFETY: The checked eight-byte range stays mapped; byte accesses - // match the unaligned packed counter the firmware writes. - *byte = unsafe { completed.add(index).read_volatile() }; - } - let mut header = FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(self.id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: pages.blocks as u32, - state_va: self.state.gpu_va(), - }); - header.unk_54[..8].copy_from_slice(&bytes); - self.control.write(0, header)?; + if pages.is_m5() != self.m5 { return Err(EINVAL); } + let completed = self.completed_bytes()?; + Self::initialize_control(&mut self.control, pages, self.id, self.state.gpu_va(), completed)?; fence(Ordering::SeqCst); Ok(()) } @@ -826,6 +1294,11 @@ impl Lease { self.phase = LeasePhase::ReleaseSent(cursor); Ok(()) } + /// Only sent releases and inherited release witnesses consume cursors. + pub(crate) fn needs_release_observation(&self) -> bool { + self.previous_release.is_some() || matches!(self.phase, LeasePhase::ReleaseSent(_)) + } + /// Observes all older release records passed by one consumer snapshot. pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { if let Some(cursor) = self.previous_release { @@ -867,7 +1340,7 @@ mod capacity_tests { #[test] fn rejected_pool_construction_returns_all_render_ids() { - let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + let ids = RenderIds::new().unwrap(); for _ in 0..8 { let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); for _ in 0..127 { diff --git a/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs new file mode 100644 index 00000000000000..e750b01bd91c7a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist/render_ids.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure physical-ID lease rules. The caller serializes this state and supplies +//! a firmware release proof before a slot is reassigned to another pool. + +pub(super) const FIRST: u16 = 129; +pub(super) const COUNT: usize = 127; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Lease { + pub(super) slot: u16, + pub(super) owner: u64, + pub(super) generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Claim { + pub(super) lease: Lease, + pub(super) displaced: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Error { + BadOwner, + Busy, + PendingRelease, + StaleLease, + Overflow, +} + +#[derive(Clone, Copy)] +struct Slot { + owner: u64, + generation: u64, + pins: u32, + last_use: u64, +} + +impl Slot { + const EMPTY: Self = Self { owner: 0, generation: 0, pins: 0, last_use: 0 }; + + fn lease(self, index: usize) -> Lease { + Lease { slot: FIRST + index as u16, owner: self.owner, generation: self.generation } + } +} + +pub(super) struct State { + slots: [Slot; COUNT], + clock: u64, +} + +impl State { + pub(super) const fn new() -> Self { Self { slots: [Slot::EMPTY; COUNT], clock: 0 } } + + /// Constant-time cached-owner path; a recycled generation cannot be pinned. + pub(super) fn pin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.owner == 0 { + return Err(Error::StaleLease); + } + let clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = clock; + self.clock = clock; + Ok(lease) + } + + /// A successful claim is a preparation pin. It must remain pinned through + /// any out-of-lock construction and every firmware reference, not just + /// through publication. Existing-owner reacquisition does not reassign it. + pub(super) fn claim(&mut self, owner: u64, mut released: impl FnMut(Lease) -> bool) + -> Result + { + if owner == 0 { return Err(Error::BadOwner); } + let next_clock = self.clock.checked_add(1).ok_or(Error::Overflow)?; + if let Some(index) = self.slots.iter().position(|s| s.owner == owner) { + let slot = &mut self.slots[index]; + let pins = slot.pins.checked_add(1).ok_or(Error::Overflow)?; + slot.pins = pins; + slot.last_use = next_clock; + self.clock = next_clock; + return Ok(Claim { lease: slot.lease(index), displaced: None }); + } + let mut candidate = self.slots.iter().position(|s| s.owner == 0); + let mut displaced = None; + if candidate.is_none() { + // One bounded scan, with no allocation and at most one release + // query per idle slot. Normal cached-owner pinning is O(1). + let mut idle = false; + for (index, slot) in self.slots.iter().enumerate() { + if slot.pins != 0 { continue; } + idle = true; + if candidate.is_some_and(|best| self.slots[best].last_use <= slot.last_use) { continue; } + let old = slot.lease(index); + if released(old) { + candidate = Some(index); + displaced = Some(old); + } + } + if candidate.is_none() { + return Err(if idle { Error::PendingRelease } else { Error::Busy }); + } + } + let index = candidate.ok_or(Error::Busy)?; + let generation = self.slots[index].generation.checked_add(1).ok_or(Error::Overflow)?; + self.slots[index] = Slot { owner, generation, pins: 1, last_use: next_clock }; + self.clock = next_clock; + Ok(Claim { lease: self.slots[index].lease(index), displaced }) + } + + /// An old owner or old generation must never release the successor's pin. + pub(super) fn unpin(&mut self, lease: Lease) -> Result { + let index = lease.slot.checked_sub(FIRST).ok_or(Error::StaleLease)? as usize; + let slot = self.slots.get_mut(index).ok_or(Error::StaleLease)?; + if slot.owner != lease.owner || slot.generation != lease.generation || slot.pins == 0 { + return Err(Error::StaleLease); + } + slot.pins -= 1; + Ok(slot.pins == 0) + } + + /// Pool destruction may also need a pending release to finish. The caller + /// retains its memory when this refuses; dropping it is not a release proof. + pub(super) fn remove(&mut self, owner: u64, released: impl FnOnce(Lease) -> bool) + -> Result<(), Error> + { + if owner == 0 { return Err(Error::BadOwner); } + let Some(index) = self.slots.iter().position(|s| s.owner == owner) else { return Ok(()); }; + let slot = self.slots[index]; + if slot.pins != 0 { return Err(Error::Busy); } + if !released(slot.lease(index)) { return Err(Error::PendingRelease); } + self.slots[index].owner = 0; + Ok(()) + } + + /// Return one cached, unpinned registration only after its release proof. + pub(super) fn reclaim_one(&mut self, mut released: impl FnMut(Lease) -> bool) -> Option { + let index = (0..COUNT) + .filter(|&i| self.slots[i].owner != 0 && self.slots[i].pins == 0) + .filter(|&i| released(self.slots[i].lease(i))) + .min_by_key(|&i| self.slots[i].last_use)?; + let old = self.slots[index].lease(index); + self.slots[index].owner = 0; + Some(old) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn full() -> (State, Vec) { + let mut state = State::new(); + let leases = (1..=COUNT as u64).map(|owner| + state.claim(owner, |_| panic!("free slot must not need release proof")).unwrap().lease + ).collect(); + (state, leases) + } + + #[test] fn bounded_physical_namespace() { + let (state, leases) = full(); + assert_eq!(leases.iter().map(|l| l.slot).collect::>(), (129..=255).collect::>()); + assert_eq!(state.slots.len(), COUNT); + } + + #[test] fn all_preparation_pins_prevent_reassignment() { + let (mut state, _) = full(); + assert_eq!(state.claim(1000, |_| panic!("pinned slot must not be offered")), Err(Error::Busy)); + } + + #[test] fn unpinned_is_not_firmware_released() { + let (mut state, leases) = full(); + for lease in &leases { state.unpin(*lease).unwrap(); } + assert_eq!(state.claim(1000, |_| false), Err(Error::PendingRelease)); + for (i, lease) in leases.iter().enumerate() { assert_eq!(state.slots[i].lease(i), *lease); } + } + + #[test] fn released_slot_can_serve_another_logical_owner() { + let (mut state, leases) = full(); + state.unpin(leases[13]).unwrap(); + let claim = state.claim(1000, |old| old == leases[13]).unwrap(); + assert_eq!(claim.displaced, Some(leases[13])); + assert_eq!(claim.lease.slot, leases[13].slot); + assert!(claim.lease.generation > leases[13].generation); + assert_eq!(state.unpin(leases[13]), Err(Error::StaleLease)); + assert!(state.unpin(claim.lease).unwrap()); + } + + #[test] fn same_owner_reuses_without_release_proof_or_generation_change() { + let mut state = State::new(); + let a = state.claim(1, |_| false).unwrap().lease; + let b = state.claim(1, |_| panic!("same owner is not reassigned")).unwrap().lease; + assert_eq!(a, b); + assert!(!state.unpin(a).unwrap()); + assert!(state.unpin(b).unwrap()); + assert_eq!(state.unpin(a), Err(Error::StaleLease)); + } + + #[test] fn stale_generation_cannot_unpin_returned_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let other = state.claim(1000, |_| true).unwrap().lease; + state.unpin(other).unwrap(); + let returned = state.claim(leases[0].owner, |_| true).unwrap().lease; + assert_eq!(returned.slot, leases[0].slot); + assert_eq!(state.unpin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(returned).unwrap()); + } + + #[test] fn thousands_of_logical_owners_do_not_expand_the_namespace() { + let mut state = State::new(); + for owner in 1..=4096 { + let lease = state.claim(owner, |_| true).unwrap().lease; + assert!((129..=255).contains(&lease.slot)); + assert!(state.unpin(lease).unwrap()); + } + assert_eq!(state.slots.iter().filter(|s| s.owner != 0).count(), COUNT); + } + + #[test] fn removal_waits_for_both_pins_and_firmware() { + let mut state = State::new(); + let lease = state.claim(1, |_| false).unwrap().lease; + assert_eq!(state.remove(1, |_| true), Err(Error::Busy)); + state.unpin(lease).unwrap(); + assert_eq!(state.remove(1, |_| false), Err(Error::PendingRelease)); + state.remove(1, |old| old == lease).unwrap(); + assert_eq!(state.unpin(lease), Err(Error::StaleLease)); + } + + #[test] fn cached_pin_preserves_lease_and_blocks_reclaim() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + assert_eq!(state.pin(leases[0]), Ok(leases[0])); + assert_eq!(state.claim(1000, |_| true), Err(Error::Busy)); + } + + #[test] fn stale_cached_pin_cannot_name_new_owner() { + let (mut state, leases) = full(); + state.unpin(leases[0]).unwrap(); + let replacement = state.claim(1000, |_| true).unwrap().lease; + assert_eq!(state.pin(leases[0]), Err(Error::StaleLease)); + assert!(state.unpin(replacement).unwrap()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs index 86fdc5db75a8f7..56c1f2324e2a17 100644 --- a/drivers/gpu/drm/asahi/g17/fw/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -13,7 +13,7 @@ pub(crate) const RENDER_BLOCKS: usize = 44; pub(crate) const COMPUTE_BLOCKS: usize = 21; /// Render reserves eight run words per block; compute packs runs consecutively. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum RunLayout { Render, Compute, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index beccacf8e061d3..1026f21f5c50ac 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -19,3 +19,8 @@ pub(crate) fn set_register_array( array.size = writes.len() as u32; Ok(()) } + +pub(crate) mod pool; +pub(crate) mod pool_budget; + +pub(crate) mod render_demand; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs new file mode 100644 index 00000000000000..abbde813cb5990 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 private-memory allocation encodings. +#![allow(dead_code)] + +pub(crate) const BLOCK_BYTES: usize = 0x20_0000; +pub(crate) const G17S_RENDER_BLOCK_BYTES: usize = 0x40_0000; +/// G17S private-memory address inventory. This sizes the directory, not the +/// physical allocation; each pool allocates only its requested initial blocks. +const G17S_POOL_BYTES: u64 = 1 << 32; +const RING_MAX_PAGES: u64 = 0x3ff800; +pub(crate) const OPERAND_BYTES: usize = 0x2_0000; +pub(crate) const STATE_BYTES: usize = 0x100; +/// Unaligned firmware-completed submission counter. +pub(crate) const COMPLETED: usize = 0x74; +/// Mutable words saved from the descriptor after the pool's final release. +pub(crate) const SAVED_FIELDS: [usize; 8] = [0x28, 0x20, 0x24, 0x2c, 0x3c, 0x48, 0x38, 0x4c]; +const GPU_PAGE: usize = 0x4000; +const PRIVATE_TOP: u64 = 1 << 42; +pub(crate) const SCENE_TABLE_ENTRIES: usize = 0x50; +pub(crate) const SCENE_STRIDE: usize = 0x80; +pub(crate) const SCENE_TRAILER_SIZE: usize = 0x40; + +pub(crate) fn scene_table_end(address: u64) -> Option { + address.checked_add((SCENE_TABLE_ENTRIES * SCENE_STRIDE) as u64) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PoolError { Geometry, Address, Buffer } + +/// Snapshot of the retained pool header. Address and allocation geometry are +/// host-owned; only SAVED_FIELDS may be restored from a retired descriptor. +pub(crate) struct PoolState(pub(crate) [u8; STATE_BYTES]); + +impl PoolState { + fn u32(&self, offset: usize) -> u64 { + u64::from(u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap())) + } + fn u64(&self, offset: usize) -> u64 { + u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) + } + pub(crate) fn descriptor(&self) -> [u64; 8] { + [self.u64(0x14) >> 7 | self.u32(0x28) << 61, + self.u64(0x20), self.u32(0x2c) | self.u32(0x1c) << 32, 0, + self.u64(0x30) >> 6 | self.u32(0x40) << 44, + self.u32(0x3c) | self.u32(0x44) << 31, + self.u32(0x48), self.u32(0x38) | self.u32(0x4c) << 32] + } + pub(crate) fn restore(&mut self, words: [u64; 8]) { + let values = [words[0] >> 61 & 1, words[1] & 0x0fff_ffff, + words[1] >> 32 & 0x0fff_ffff, words[2] & 0x0fff_ffff, + words[5] & 0x0fff_ffff, words[6] & 0xffff, + words[7] & 0x0fff_ffff, words[7] >> 32 & 0xffff]; + for (offset, value) in SAVED_FIELDS.into_iter().zip(values) { + self.0[offset..offset + 4].copy_from_slice(&(value as u32).to_le_bytes()); + } + } + /// Reject a descriptor belonging to different backing before restoring + /// its mutable counters. Reserved bits are not host address bits. + pub(crate) fn matches(&self, words: &[u64; 8]) -> bool { + let own = self.descriptor(); + words[0] & ((1 << 35) - 1) == own[0] & ((1 << 35) - 1) + && words[2] >> 32 == own[2] >> 32 + && words[4] == own[4] + } +} + +/// Parameter-buffer capacity follows the memory budget and initial demand. +/// Capacity is rounded to groups of 33 half-MiB blocks; it reserves list space, +/// not physical backing for every possible block. +pub(crate) fn parameter_capacity(memory_bytes: u64, initial_bytes: u64) -> Result { + const GROUP: u64 = 33 * 0x80000; + let scaled = memory_bytes.checked_mul(5).ok_or(PoolError::Geometry)? >> 2; + let budget = (((u128::from(scaled) * 0x28f5_c28f_5c28_f5c3) >> 64) as u64) >> 2; + let groups = |bytes: u64| bytes.checked_add(GROUP - 1) + .map(|n| n / GROUP).ok_or(PoolError::Geometry); + let groups = groups(budget)?.max(groups(initial_bytes)?); + usize::try_from(groups.checked_mul(33).ok_or(PoolError::Geometry)?) + .map_err(|_| PoolError::Geometry) +} + +/// Per-scene scratch covers the complete parameter-buffer inventory. Optional +/// cluster entries are supplied by the dispatch policy, not the active draw's +/// physical block count. Each entry and the header occupy eight bytes. +pub(crate) fn parameter_scratch_stride(capacity: usize, cluster_entries: usize) + -> Result +{ + if capacity == 0 { return Err(PoolError::Geometry); } + let pages = capacity.checked_mul(16).ok_or(PoolError::Geometry)?; + let entries = pages.checked_add(0x17ff).ok_or(PoolError::Geometry)? / 0x1800; + let entries = entries.checked_add(cluster_entries).ok_or(PoolError::Geometry)?; + entries.checked_mul(8).and_then(|n| n.checked_add(8 + 63)) + .map(|n| n & !63).ok_or(PoolError::Geometry) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolGeometry { + pub(crate) blocks: usize, + pub(crate) directory_entries: u32, + pub(crate) pages: u32, + pub(crate) directory_bytes: usize, + pub(crate) freelist_bytes: usize, + g17s_render_blocks: bool, +} + +fn round_page(bytes: usize) -> Result { + bytes.checked_add(GPU_PAGE - 1).map(|n| n & !(GPU_PAGE - 1)).ok_or(PoolError::Geometry) +} + +impl PoolGeometry { + pub(crate) fn g17s(blocks: usize) -> Result { + let bytes = G17S_POOL_BYTES.min(RING_MAX_PAGES << 12) + & !(G17S_RENDER_BLOCK_BYTES as u64 - 1); + Self::g17s_render(blocks, (bytes >> 12) as u32) + } + pub(crate) fn new(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, false) + } + + pub(crate) fn g17s_render(blocks: usize, directory_entries: u32) -> Result { + Self::with_kind(blocks, directory_entries, true) + } + + pub(crate) fn block_bytes(self) -> usize { + if self.g17s_render_blocks { G17S_RENDER_BLOCK_BYTES } else { BLOCK_BYTES } + } + + fn with_kind(blocks: usize, directory_entries: u32, g17s_render_blocks: bool) -> Result { + let block_bytes = if g17s_render_blocks { G17S_RENDER_BLOCK_BYTES } else { BLOCK_BYTES }; + let pages = blocks.checked_mul(block_bytes / 0x1000) + .and_then(|n| u32::try_from(n).ok()).ok_or(PoolError::Geometry)?; + if blocks == 0 || pages > directory_entries { return Err(PoolError::Geometry); } + let directory_bytes = round_page((directory_entries as usize).checked_mul(8) + .ok_or(PoolError::Geometry)?)?; + let stride = if g17s_render_blocks { 8 } else { 0x40 }; + let freelist_bytes = round_page(blocks.checked_mul(stride).ok_or(PoolError::Geometry)?)?; + + if freelist_bytes >> 12 >= 1 << 20 { return Err(PoolError::Geometry); } + Ok(Self { blocks, directory_entries, pages, directory_bytes, freelist_bytes, g17s_render_blocks }) + } +} + +fn private_range(address: u64, bytes: usize) -> Result<(), PoolError> { + if address == 0 || address & (GPU_PAGE as u64 - 1) != 0 + || address.checked_add(bytes as u64).is_none_or(|end| end > PRIVATE_TOP) { + Err(PoolError::Address) + } else { Ok(()) } +} + +fn validate_blocks(g: PoolGeometry, blocks: &[u64]) -> Result<(), PoolError> { + if PoolGeometry::with_kind(g.blocks, g.directory_entries, g.g17s_render_blocks)? != g { return Err(PoolError::Geometry); } + if blocks.len() != g.blocks { return Err(PoolError::Geometry); } + for address in blocks { private_range(*address, g.block_bytes())?; } + Ok(()) +} + +pub(crate) fn encode_directory(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.directory_bytes { return Err(PoolError::Buffer); } + out[..g.directory_bytes].fill(0); + + let pages_per_block = g.block_bytes() / 0x1000; + for (block, address) in blocks.iter().enumerate() { + for page in 0..pages_per_block { + let offset = (block * pages_per_block + page) * 8; + out[offset..offset + 8].copy_from_slice(&(address + page as u64 * 0x1000).to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_freelist(g: PoolGeometry, blocks: &[u64], out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_blocks(g, blocks)?; + if out.len() < g.freelist_bytes { return Err(PoolError::Buffer); } + out[..g.freelist_bytes].fill(0); + for (index, address) in blocks.iter().enumerate() { + let (offset, value) = if g.g17s_render_blocks { (index * 8, *address) } + else { (index * 0x40, address | (1 << 61)) }; + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PoolLayout { + pub(crate) geometry: PoolGeometry, + pub(crate) directory: u64, + pub(crate) freelist: u64, + + pub(crate) hardware_id: u32, +} + +fn validate_layout(p: PoolLayout) -> Result<(), PoolError> { + if PoolGeometry::with_kind(p.geometry.blocks, p.geometry.directory_entries, p.geometry.g17s_render_blocks)? != p.geometry { + return Err(PoolError::Geometry); + } + private_range(p.directory, p.geometry.directory_bytes)?; + private_range(p.freelist, p.geometry.freelist_bytes) +} + +pub(crate) fn initial_uma_descriptor(p: PoolLayout) -> Result<[u64; 8], PoolError> { + validate_layout(p)?; + if p.geometry.g17s_render_blocks { + let blocks = p.geometry.blocks as u64; + return Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.block_bytes() as u64 >> 12) << 44), + blocks, 0, blocks]); + } + Ok([p.directory >> 7, u64::from(p.geometry.pages) << 32, + u64::from(p.geometry.pages) | (u64::from(p.geometry.directory_entries) << 32), + 0, (p.freelist >> 6) | ((p.geometry.freelist_bytes as u64 >> 12) << 44), + 0, 0, 0]) +} + +pub(crate) fn encode_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + validate_layout(p)?; + if submitted == 0 || submitted & 3 != 0 { return Err(PoolError::Address); } + if out.len() < STATE_BYTES { return Err(PoolError::Buffer); } + out[..STATE_BYTES].fill(0); + for (offset, value) in [(0, 1u32), (8, p.hardware_id), (0x10, 2), + (0x1c, p.geometry.directory_entries), (0x24, p.geometry.pages), + (0x2c, p.geometry.pages), (0x40, (p.geometry.freelist_bytes >> 12) as u32), + (0x60, 2), (0x80, 1)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(0x14, p.directory), (0x30, p.freelist), + (0x50, p.freelist), (0x6c, submitted)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + if p.geometry.g17s_render_blocks { + + for (offset, value) in [(0x0c, 2), (0x38, p.geometry.blocks as u32), + (0x3c, p.geometry.blocks as u32), + (0x40, (p.geometry.block_bytes() >> 12) as u32), (0x44, 0), + (0x60, 0), + (0x68, p.geometry.blocks as u32), (0x80, 4)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } + Ok(()) +} + +pub(crate) fn encode_g17s_render_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + encode_state(p, submitted, out)?; + out[0x80..0x84].copy_from_slice(&4u32.to_le_bytes()); + Ok(()) +} + +pub(crate) fn encode_g17s_compute_state(p: PoolLayout, submitted: u64, out: &mut [u8]) + -> Result<(), PoolError> +{ + if !p.geometry.g17s_render_blocks { return Err(PoolError::Geometry); } + encode_state(p, submitted, out)?; + out[0x7c..0x80].copy_from_slice(&1u32.to_le_bytes()); + out[0x80..0x84].copy_from_slice(&1u32.to_le_bytes()); + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs new file mode 100644 index 00000000000000..80339f98a9026d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17S shared render private-memory demand. Counts describe hardware and +//! simultaneous work; byte requirements come from the submitted programs. + +use super::pool::{PoolError, PoolGeometry, G17S_RENDER_BLOCK_BYTES}; + +#[derive(Clone, Copy)] +pub(crate) struct RenderBudget { + active_cores: u32, + constant_pages: u32, + active_kicks: u32, +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct Requirements { + pub(crate) minimum_bytes: u64, + pub(crate) ideal_bytes: u64, + pub(crate) self_reserve_pages: u32, + pub(crate) render_target_pages: u32, + pub(crate) has_spill: bool, +} + +/// Retained high-water marks, excluding per-kick reservations from the two +/// base byte counts. They may increase demand but never replace live demand. +#[derive(Clone, Copy, Default)] +pub(crate) struct History { + pub(crate) minimum_base: u64, + pub(crate) ideal_base: u64, + pub(crate) self_reserve: u64, + pub(crate) render_target: u64, + pub(crate) minimum_pool: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Sizes { + pub(crate) minimum: u64, + pub(crate) ideal: u64, +} + +fn add(a: u64, b: u64) -> Result { + a.checked_add(b).ok_or(PoolError::Geometry) +} +fn mul(a: u64, b: u64) -> Result { + a.checked_mul(b).ok_or(PoolError::Geometry) +} +fn aligned(bytes: u64) -> Result { + Ok(add(bytes, G17S_RENDER_BLOCK_BYTES as u64 - 1)? + & !(G17S_RENDER_BLOCK_BYTES as u64 - 1)) +} + +impl RenderBudget { + /// `core_masks` contains the enabled cores of each physical cluster. + /// `ptc_floor` is the configured per-core context count; `active_kicks` + /// includes simultaneously resident tiling and fragment work. + pub(crate) fn new(core_masks: &[u32], ptc_floor: u32, active_kicks: u32) + -> Result + { + let mut active_cores = 0u32; + let mut cluster_cores = 0u32; + for mask in core_masks { + let count = mask.count_ones(); + active_cores = active_cores.checked_add(count).ok_or(PoolError::Geometry)?; + cluster_cores = cluster_cores.max(count); + } + if active_cores == 0 || active_kicks == 0 || ptc_floor == 0 { + return Err(PoolError::Geometry); + } + let constant_pages = cluster_cores.checked_mul(ptc_floor) + .and_then(|n| n.checked_add(32)) + .and_then(|n| n.checked_mul(u32::try_from(core_masks.len()).ok()?)) + .ok_or(PoolError::Geometry)?; + Ok(Self { active_cores, constant_pages, active_kicks }) + } + + /// Compute SCS storage is private to each enabled USC. The eight-page + /// tail belongs to the compute allocator, in addition to its small/large + /// SCS page inventories and the execution-state spill reservation. + pub(crate) fn compute_scs_requirements(&self, mut request: Requirements, + pages: u16) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, 8)?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + /// Render's allocator publishes the already rounded auxiliary page count + /// in the descriptor, including its eight-page tail. This is separate + /// from the layout-derived small/large SCS inventories. The USC + /// spill-buffer allocation multiplies their sum by cores. + pub(crate) fn render_scs_requirements(&self, mut request: Requirements, + pages: u16, auxiliary_pages: u32) -> Result + { + if pages != 0 { + let small = u64::from(pages); + let per_core = add(add(small, small.div_ceil(256))?, + u64::from(auxiliary_pages))?; + let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; + request.minimum_bytes = add(request.minimum_bytes, bytes)?; + request.ideal_bytes = add(request.ideal_bytes, bytes)?; + request.has_spill = true; + } + Ok(request) + } + + pub(crate) fn sizes(&self, request: Requirements, history: History) + -> Result + { + let reserve = u64::from(request.self_reserve_pages) << 12; + let rt = u64::from(request.render_target_pages) << 12; + let reserved = add(reserve, rt)?; + if request.minimum_bytes < reserved || request.ideal_bytes < reserved { + return Err(PoolError::Geometry); + } + // Every enabled USC has execution-state storage. Programs with spill + // state add twelve pages; shared queues reserve another twenty-four. + let execution = mul(u64::from(self.active_cores), + if request.has_spill { 76 << 12 } else { 64 << 12 })?; + let constant = mul(u64::from(self.constant_pages), + u64::from(self.active_kicks - 1) << 12)?; + let per_kick = mul(add(reserve.max(history.self_reserve), + rt.max(history.render_target))?, u64::from(self.active_kicks))?; + let size = |base: u64, high_water: u64| { + let base = add(base, execution)?.checked_sub(reserved) + .ok_or(PoolError::Geometry)?.max(high_water); + add(add(base, constant)?, per_kick) + }; + let ideal = aligned(size(request.ideal_bytes, history.ideal_base)?)?; + let minimum = aligned(size(request.minimum_bytes, history.minimum_base)? + .min(ideal.max(history.minimum_pool)))?; + if minimum > ideal { return Err(PoolError::Geometry); } + Ok(Sizes { minimum, ideal }) + } +} + +impl Sizes { + /// Reserve the full address inventory but allocate only the desired + /// physical blocks. The hardware directory limit is checked by geometry. + pub(crate) fn geometry(self) -> Result { + if self.minimum == 0 || self.minimum > self.ideal { return Err(PoolError::Geometry); } + let blocks = usize::try_from(self.ideal / G17S_RENDER_BLOCK_BYTES as u64) + .map_err(|_| PoolError::Geometry)?; + if self.ideal % G17S_RENDER_BLOCK_BYTES as u64 != 0 { return Err(PoolError::Geometry); } + PoolGeometry::g17s(blocks) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs new file mode 100644 index 00000000000000..d99970e2f126ed --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_demand.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Program and tile storage pressure for shared render pools. Program sizes +//! are byte counts after linking; they are independent of register allocation. + +use super::pool::PoolError; +use super::pool_budget::Requirements; + +type Result = core::result::Result; + +fn add(a: u64, b: u64) -> Result { a.checked_add(b).ok_or(PoolError::Geometry) } +fn mul(a: u64, b: u64) -> Result { a.checked_mul(b).ok_or(PoolError::Geometry) } +fn count(value: u64) -> Result { value.try_into().map_err(|_| PoolError::Geometry) } +fn pages(value: u64, size: u64) -> Result { Ok(add(value, size - 1)? / size) } + +#[derive(Clone, Copy, Default)] +pub(crate) struct FragmentStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) auxiliary_count: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct FragmentParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) auxiliary_units: u32, + pub(crate) primary_large: u32, + pub(crate) secondary_large: u32, +} + +impl FragmentStorage { + pub(crate) fn parameters(self) -> Result { + let primary = mul(add(u64::from(self.primary_bytes), u64::from(self.linked_bytes))? + .max(u64::from(self.minimum_primary_bytes)), 32)?; + let primary = add(primary, 0x4380)?; + let secondary = mul(u64::from(self.secondary_bytes), 32)?; + let primary_large = pages(primary, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(primary, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let auxiliary = mul(u64::from(self.auxiliary_count), 128)?; + let (auxiliary, reserved) = if auxiliary == 0 { (0, 0) } + else { (auxiliary | 64, auxiliary.max(0x4000) | 64) }; + let pressure = add(add(mul(small, 96)?, mul(large, 10)?)?, + add(mul(pages(auxiliary, 0x1000)?, 64)?, mul(pages(auxiliary, 0x10000)?, 5)?)?)?; + Ok(FragmentParams { + pressure: count(pressure)?, page_units: count(add(small, large)?)?, + large_units: count(large)?, + auxiliary_units: count(add(pages(reserved, 0x1000)?, pages(reserved, 0x10000)?)?)?, + primary_large: count(primary_large)?, secondary_large: count(secondary_large)?, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct TileStorage { + pub(crate) threadgroup_bytes: u32, + pub(crate) imageblock_bytes: u32, + pub(crate) multisample_bytes: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TileParams { + pub(crate) pressure: u32, + pub(crate) page_units: u32, + pub(crate) large_units: u32, + pub(crate) threadgroup_large: u32, + pub(crate) imageblock_large: u32, + pub(crate) multisample_large: u32, +} + +impl TileStorage { + pub(crate) fn parameters(self, preempt_formula: bool) -> Result { + let imageblock = u64::from(self.imageblock_bytes); + let sizes = [u64::from(self.threadgroup_bytes), + if imageblock == 0 { 0 } else { add(imageblock, 64)? }, + u64::from(self.multisample_bytes)]; + let large = [pages(sizes[0], 0x10000)?, pages(sizes[1], 0x10000)?, + pages(sizes[2], 0x10000)?]; + let small = add(add(pages(sizes[0], 0x1000)?, pages(sizes[1], 0x1000)?)?, + pages(sizes[2], 0x1000)?)?; + let total_large = add(add(large[0], large[1])?, large[2])?; + let factor = if preempt_formula { 11 } else { 6 }; + Ok(TileParams { + pressure: count(add(mul(small, factor)?, mul(total_large, factor.min(8))?)?)?, + page_units: count(add(small, total_large)?)?, large_units: count(total_large)?, + threadgroup_large: count(large[0])?, imageblock_large: count(large[1])?, + multisample_large: count(large[2])?, + }) + } +} + +/// One submission's maximum pressure across all of its linked render +/// programs. Different programs may maximize different storage dimensions. +#[derive(Clone, Copy)] +pub(crate) struct RenderDemand { + counts: [u32; 5], + minimum_pages: u32, + ideal_pages: u32, + pressure: u32, + clusters: u64, + maximum: u64, + total: u64, + floor: u64, + limited_maximum: u64, + limited_total: u64, + included: bool, +} + +impl RenderDemand { + pub(crate) fn new(core_masks: &[u32], nominal_cores: u32, ptc_floor: u32) -> Result { + let clusters = core_masks.len() as u64; + let maximum = u64::from(core_masks.iter().map(|m| m.count_ones()).max().unwrap_or(0)); + let total = core_masks.iter().try_fold(0u64, |n, m| add(n, u64::from(m.count_ones())))?; + if total == 0 || ptc_floor == 0 || u64::from(nominal_cores) < total { + return Err(PoolError::Geometry); + } + let limit = mul(u64::from(nominal_cores), 2)? / clusters; + Ok(Self { + counts: [0; 5], minimum_pages: 0, ideal_pages: 0, pressure: 0, + clusters, maximum, total, floor: u64::from(ptc_floor), + limited_maximum: maximum.min(limit), limited_total: total.min(mul(limit, clusters)?), + included: false, + }) + } + + pub(crate) fn include_fragment(&mut self, fragment: FragmentParams, tile: TileParams) -> Result<()> { + let Self { clusters, maximum, total, floor, .. } = *self; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let fragment_small = u64::from(fragment.page_units); + let tile_small = u64::from(tile.page_units); + let pressure = add(u64::from(fragment.pressure), u64::from(tile.pressure))?; + let large = add(add(u64::from(fragment.large_units), u64::from(fragment.auxiliary_units))?, + u64::from(tile.large_units))?; + let parts = [fragment.primary_large, fragment.secondary_large, + tile.threadgroup_large, tile.imageblock_large, tile.multisample_large]; + let tails = parts[..4].iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let ideal = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(mul(pressure.saturating_sub(floor), maximum)?, 64)?)?; + let minimum = add(add(add(base, u64::from(tile.multisample_large))?, tails)?, + add(add(mul(add(add(fragment_small, tile_small)?.saturating_sub(floor), tile_small)?, + maximum)?, mul(large, total)?)?, 64)?)?; + self.merge(parts, minimum, ideal, pressure, clusters) + } + + /// Vertex work has no imageblock and uses the tiling execution limits. + pub(crate) fn include_vertex(&mut self, storage: VertexStorage) -> Result<()> { + let params = FragmentStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + auxiliary_count: 0, + }.parameters()?; + let Self { clusters, floor, limited_maximum: maximum, limited_total: total, .. } = *self; + let large = u64::from(params.large_units); + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let minimum = add(add(add(base, large)?, 64)?, + add(mul(u64::from(params.page_units).saturating_sub(floor), maximum)?, + mul(large, total)?)?)?; + let ideal = add(add(add(base, large)?, 64)?, + mul(u64::from(params.pressure).saturating_sub(floor), maximum)?)?; + self.merge([params.primary_large, params.secondary_large, 0, 0, 0], minimum, + ideal, u64::from(params.pressure), clusters) + } + + /// Compute storage is evaluated per dispatch before merging component maxima. + /// `resident_warps` and `shared_budget` are device policy, not API limits. + pub(crate) fn include_compute(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + if storage.threads == 0 || resident_warps == 0 { + return Err(PoolError::Geometry); + } + let frame = mul(add(u64::from(storage.primary_bytes), + u64::from(storage.linked_bytes))? + .max(u64::from(storage.minimum_primary_bytes)), 32)?; + let frame = add(frame, 0x4380)?; + let secondary = mul(u64::from(storage.secondary_bytes), 32)?; + let primary_large = pages(frame, 0x10000)?; + let secondary_large = pages(secondary, 0x10000)?; + let small = add(pages(frame, 0x1000)?, pages(secondary, 0x1000)?)?; + let large = add(primary_large, secondary_large)?; + let shared = u64::from(storage.threadgroup_bytes); + let shared_small = pages(shared, 0x1000)?; + let shared_large = pages(shared, 0x10000)?; + let warps = pages(u64::from(storage.threads), 32)?; + let mut groups = u64::from(resident_warps) / warps; + if shared != 0 { + let aligned_shared = mul(pages(shared, 64)?, 64)?; + groups = groups.min(u64::from(shared_budget) / aligned_shared); + } + if groups == 0 { return Err(PoolError::Geometry); } + let pressure = add(add(mul(mul(groups, warps)?.min(96), small)?, + mul(mul(groups, warps)?.min(10), large)?)?, + add(mul(groups, shared_small)?, mul(groups.min(8), shared_large)?)?)?; + let Self { clusters, maximum, total, floor, .. } = *self; + let group_large = mul(warps.min(10), large)?; + let tile = add(shared_small, shared_large)?; + let transient = add(add(mul(warps, small)?, group_large)?, tile)?; + let transient = add(transient.saturating_sub(floor), tile)?; + let base = add(mul(maximum, add(floor, 11)?)?, 58)?; + let tail = add(add(large, shared_large)?, 64)?; + let minimum = add(add(add(base, tail)?, mul(transient, maximum)?)?, + mul(add(shared_large, group_large)?, total)?)?; + let ideal = add(add(base, tail)?, mul(pressure.saturating_sub(floor), maximum)?)?; + self.merge([count(primary_large)?, count(secondary_large)?, + count(shared_large)?, 0, 0], minimum, ideal, pressure, clusters) + } + + /// Mesh/object execute on the tiling engine. Reuse compute's occupancy + /// arithmetic with the tiling-limited core topology, never compute's full + /// enabled core count. Resident/shared limits are explicit device policy. + pub(crate) fn include_mesh(&mut self, storage: ComputeStorage, + resident_warps: u32, shared_budget: u32) -> Result<()> { + let maximum = self.maximum; + let total = self.total; + self.maximum = self.limited_maximum; + self.total = self.limited_total; + let result = self.include_compute(storage, resident_warps, shared_budget); + self.maximum = maximum; + self.total = total; + result + } + + fn merge(&mut self, parts: [u32; 5], minimum: u64, ideal: u64, pressure: u64, + clusters: u64) -> Result<()> { + // Stored page counts are signed before conversion to byte counts. + // Reject an unrepresentable request instead of wrapping into a small pool. + let round = |value| -> Result { + let value = add(mul(value, clusters)?, 7)? & !7; + if value > i32::MAX as u64 { return Err(PoolError::Geometry); } + count(value) + }; + let mut next = *self; + next.minimum_pages = next.minimum_pages.max(round(minimum)?); + next.ideal_pages = next.ideal_pages.max(round(ideal)?); + next.pressure = next.pressure.max(count(pressure)?); + for (out, value) in next.counts.iter_mut().zip(parts) { *out = (*out).max(value); } + next.resource_flags()?; + next.included = true; + *self = next; + Ok(()) + } + + pub(crate) fn pressure(&self) -> u32 { self.pressure } + + pub(crate) fn resource_flags(&self) -> Result { + let [primary, secondary, shared, imageblock, multisample] = self.counts; + if primary > 0xff || secondary > 0xf || shared > 3 || imageblock > 0xff + || multisample > 0xff { return Err(PoolError::Geometry); } + Ok(0x14b | (u64::from(primary) << 10) | (u64::from(secondary) << 18) + | (u64::from(shared) << 22) | (u64::from(imageblock) << 24) + | (u64::from(multisample) << 32)) + } + + pub(crate) fn requirements(&self) -> Result { + if !self.included { + return Err(PoolError::Geometry); + } + let Self { maximum, clusters, .. } = *self; + let counts = self.counts.iter().try_fold(0u64, |n, &v| add(n, u64::from(v)))?; + let render_target = mul(add(add(mul(maximum, 11)?, counts)?, 10)?, clusters)?; + Ok(Requirements { + minimum_bytes: u64::from(self.minimum_pages) << 12, + ideal_bytes: u64::from(self.ideal_pages) << 12, + self_reserve_pages: 0, render_target_pages: count(render_target)?, has_spill: false, + }) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct VertexStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, +} + +/// Storage and execution width of a linked compute dispatch. +#[derive(Clone, Copy, Default)] +pub(crate) struct ComputeStorage { + pub(crate) primary_bytes: u32, + pub(crate) secondary_bytes: u32, + pub(crate) linked_bytes: u32, + pub(crate) minimum_primary_bytes: u32, + pub(crate) threadgroup_bytes: u32, + pub(crate) threads: u32, +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4762ee813f2f0f..e94fc7f11aaff3 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1547,7 +1547,7 @@ impl Gpu { ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, - render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + render_ids: freelist::RenderIds::new()?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9501ec86785d22..f060c9426d885d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -497,7 +497,7 @@ pub(crate) struct PoolBacking { impl PoolBacking { pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { - let pool = ComputePool::new(alloc, id.qid(), blocks)?; + let pool = ComputePool::sized(alloc, id.qid(), blocks)?; Ok(Self { pool, id }) } @@ -1836,7 +1836,7 @@ impl Queue { Ok((self.pool.id(), self.pool.page_list_va()?)) } pub(crate) fn pool_fits(&self, required: usize) -> bool { - !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits_blocks(required) } /// Only a capacity increase waits for this queue's previously published work. From 9893ab045519e6b590ce90cfdd68041af44c6770 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:17 -0800 Subject: [PATCH 065/171] drm/asahi: g17: M5: install M5 queues and submit through the host command port Add the M5 queue records, configurations and notifications, and the host command port that installs a queue pair and submits work. On M5, tiling, fragment and compute share the compute ring of each priority, whose outer credits are claimed before either host port is written, all under the device mutex. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..f49e91b3f364 (4 commits); SPTM/APIF removed; adapted to integration's InitData work publication and resource wait domains; comments and names reworded; M5 paths restored where the first rewrite lost them: the UAT's range for the compute kick rings and timestamps, the M5 control records, reports and statistics mask, the M5 firmware preparation, the fault window read, and the scheduler page from the retention pool; original 6613f5ab7b7e, e25ba8a6fb2b, 1a021061f226, f49e91b3f364 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit 28976ea2a285a4c188711f2fcb1188307f9036df in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/channel.rs | 15 +- drivers/gpu/drm/asahi/g17/context.rs | 54 ++++- drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fw/m5/queue.rs | 202 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/queue.rs | 8 +- drivers/gpu/drm/asahi/g17/initdata.rs | 175 ++++++++++++++- .../drm/asahi/g17/initdata/outer_credit.rs | 122 +++++++++++ .../gpu/drm/asahi/g17/initdata/work_pair.rs | 75 +++++++ drivers/gpu/drm/asahi/g17/kick.rs | 6 +- drivers/gpu/drm/asahi/g17/mod.rs | 55 ++++- drivers/gpu/drm/asahi/g17/object.rs | 4 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 4 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 2 +- drivers/gpu/drm/asahi/g17/reports.rs | 20 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 9 +- drivers/gpu/drm/asahi/g17/teardown.rs | 12 +- drivers/gpu/drm/asahi/regs.rs | 65 ++++++ 17 files changed, 777 insertions(+), 52 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/queue.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/work_pair.rs diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 7b4c9de0e0fd96..079bc954d6f24f 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -23,19 +23,20 @@ enum ReportRing { } impl ReportRing { - fn geometry(self) -> (usize, usize, u32, u32) { + /// State and entry offsets, capacity and the record tags the family allows. + fn geometry(self, init: &InitData) -> (usize, usize, u32, u64) { match self { Self::Trace => ( status::TRACE_STATE as usize, status::TRACE_RING as usize, TRACE_SLOTS, - TRACE_TYPES, + u64::from(TRACE_TYPES), ), Self::Statistics => ( status::STATS_STATE as usize, status::STATS_RING as usize, STATISTICS_SLOTS, - STATISTICS_TYPES, + init.statistics_types(), ), } } @@ -154,7 +155,7 @@ impl Rings { max_age_ns: u64, accept_statistics: bool, ) -> Result { - let (state, entries, capacity, allowed_types) = ring.geometry(); + let (state, entries, capacity, allowed_types) = ring.geometry(init); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; fence(Ordering::Acquire); @@ -172,12 +173,12 @@ impl Rings { let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; - if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + if tag >= u64::BITS || allowed_types & (1u64 << tag) == 0 { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { let host_ns = super::now_ns(); - reports.record(role, words, host_ns); + reports.record(role, words, host_ns, allowed_types); if accept_statistics { telemetry.record(&words, host_ns, max_age_ns); } @@ -215,7 +216,7 @@ impl Rings { init.publish_control( producer, record.bytes(), - record.opcode().asserts_gpu_power(), + init.control_asserts_gpu_power(record.opcode()), next, )?; super::trace::message(super::trace::CONTROL, record.opcode() as u64); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 8a6d9b572dec09..b232eee184e5ce 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -115,17 +115,42 @@ impl WorkStorage { } } +/// A context's scheduler page. The M5 firmware caches its mapping, so on M5 it +/// comes from the retention pool and stays mapped until firmware stop; elsewhere +/// it is an ordinary allocation. +pub(crate) enum SchedulerStorage { + Ordinary(KernelObject), + Retained(PooledObject), +} +impl core::ops::Deref for SchedulerStorage { + type Target = KernelObject; + fn deref(&self) -> &KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} +impl core::ops::DerefMut for SchedulerStorage { + fn deref_mut(&mut self) -> &mut KernelObject { + match self { + Self::Ordinary(object) => object, + Self::Retained(object) => object, + } + } +} + /// Firmware-visible pages of a context, retained by an installed queue whose /// last owner has exited (see [`Context::retain_firmware_pages`]). pub(crate) struct FirmwarePages { - _scheduler: Arc, + _scheduler: Arc, _work: Arc>, } pub(crate) struct Context { execution: Arc, status: Arc, - scheduler: Arc, + scheduler: Arc, scheduler_generation: u32, published: AtomicBool, /// A kill of this context was published, through a compute queue or on its render pairs' @@ -173,12 +198,23 @@ impl Context { ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; - let mut scheduler = alloc.kernel( - UAT_PGSZ, - UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; + let mut scheduler = if alloc.uat.is_m5() { + SchedulerStorage::Retained(pool.allocate( + alloc, + Family::Scheduler, + UAT_PGSZ, + UAT_PGSZ, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?) + } else { + SchedulerStorage::Ordinary(alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?) + }; scheduler.initialize::(0, |value| { value.init(); Ok(()) @@ -292,7 +328,7 @@ impl Context { } /// Installed physical channels retain this page after the logical execution /// context and its VM lease have completed their scheduler release. - pub(crate) fn scheduler_owner(&self) -> Arc { + pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } /// Scheduler release and command retirement have both been witnessed. diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index 1026f21f5c50ac..e33ada8dbdb881 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -22,5 +22,6 @@ pub(crate) fn set_register_array( pub(crate) mod pool; pub(crate) mod pool_budget; +pub(crate) mod queue; pub(crate) mod render_demand; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs new file mode 100644 index 00000000000000..0a0b64b1de05d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/queue.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 meanings of fields in the shared channel configuration record. +use kernel::prelude::*; +use super::super::queue::{Policy,QueueConfig,QueueConfigArgs,QueueConfigTarget}; + +/// The firmware's channel priority profiles, including the independent QoS +/// weight selected at queue registration. A single owner has a full share. +pub(crate) fn scheduling_policy(priority: u32) -> Result { + match priority { + 0 => Ok(Policy::new(0, 1, 15, 8192)), + 1 => Ok(Policy::new(1, 0, 20, 8192)), + 2 => Ok(Policy::new(2, 2, 4, 8192)), + 3 => Ok(Policy::new(3, 0, 20, 8192)), + _ => Err(EINVAL), + } +} + +/// M5's channel-state record. The scheduler pointer sits earlier than in +/// the Neo record; the policy and owner fields also moved. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct QueueRecord { + pointers_va: u64, + items_va: u64, + job_list_va: u64, + reserved_18: [u8; 0xc], + sentinel_24: u32, + priority: [u32; 2], + reserved_30: [u8; 6], + sentinel_36: u16, + reserved_38: u32, + policy: u32, + sentinel_40: u32, + owner_pid: u32, + reserved_48: [u8; 0x28], + scheduler_va: u64, + reserved_78: [u8; 8], +} +impl QueueRecord { + fn scheduling_words(policy: Policy) -> Result<([u8; 6], u16, u32)> { + match (policy.priority(), policy.policy()) { + (0, 1) => Ok(([0, 0, 0xff, 0xff, 0xff, 0xff], u16::MAX, 1)), + (1, 0) => Ok(([0, 0, 0, 0, 0xff, 0xff], u16::MAX, 0)), + (2, 2) => Ok(([0; 6], u16::MAX, 0)), + (3, 0) => Ok(([0; 6], 0, 0)), + _ => Err(EINVAL), + } + } + /// The scheduling words follow the queue's priority class. + pub(crate) fn new(args: &super::super::queue::QueueRecordArgs) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(args.policy)?; + Ok(Self { + pointers_va: args.pointers_va, items_va: args.items_va, + job_list_va: args.job_list_va, reserved_18: [0; 0xc], + sentinel_24: u32::MAX, priority: [u32::from(args.policy.priority()); 2], reserved_30, + sentinel_36: sentinel, reserved_38, policy: u32::from(args.policy.policy()), + sentinel_40: u32::MAX, owner_pid: args.owner_pid, + reserved_48: [0; 0x28], scheduler_va: args.scheduler_va, + reserved_78: [0; 8], + }) + } + /// Every host-owned identity word already holds what set_compute_owner + /// would store (so that update would not change the record). + pub(crate) fn names_owner(&self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + let (job_list, scheduler_va, pid, priority) = + (self.job_list_va, self.scheduler_va, self.owner_pid, self.priority); + let (r30, s36, r38, current) = (self.reserved_30, self.sentinel_36, self.reserved_38, self.policy); + Ok(job_list == work_head && scheduler_va == scheduler && pid == owner_pid + && priority == [u32::from(policy.priority()); 2] && r30 == reserved_30 + && s36 == sentinel && r38 == reserved_38 && current == u32::from(policy.policy())) + } + /// Replace only host-owned identity fields after the old owner retires. + pub(crate) fn set_compute_owner(&mut self, work_head: u64, scheduler: u64, + owner_pid: u32, policy: super::super::queue::Policy) -> Result { + let (reserved_30, sentinel, reserved_38) = Self::scheduling_words(policy)?; + // Changing a channel's priority invalidates this word when its hardware + // class changes. Retained physical queues can change class on reuse. + if self.priority[0] != u32::from(policy.priority()) { + self.sentinel_40 = u32::MAX; + } + self.job_list_va = work_head; + self.scheduler_va = scheduler; + self.owner_pid = owner_pid; + self.priority = [u32::from(policy.priority()); 2]; + self.reserved_30 = reserved_30; + self.sentinel_36 = sentinel; + self.reserved_38 = reserved_38; + self.policy = u32::from(policy.policy()); + Ok(()) + } +} +const _: () = { + assert!(core::mem::size_of::() == 0x80); + assert!(core::mem::offset_of!(QueueRecord, policy) == 0x3c); + assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x70); +}; + +pub(crate) struct ConfigProfile { + pub(crate) soft_fault_enabled:bool, + pub(crate) extended:bool, + pub(crate) parameter_manager:Option, +} +pub(crate) fn config(args:&QueueConfigArgs,p:ConfigProfile)->Result { + if p.parameter_manager.is_some() && !matches!(args.target,QueueConfigTarget::Tiling(Some(_))) { + return Err(EINVAL); + } + let mut out=QueueConfig::new(args)?; + out.unk_66=u16::from(p.soft_fault_enabled); + out.unk_68[2..6].copy_from_slice(&u32::from(p.extended).to_le_bytes()); + if let Some(manager)=p.parameter_manager {out.fragment_qid=manager;} + Ok(out) +} + +/// M5 publishes the current kick with the common tag-16 wire shape. All +/// reserved bytes are zero; the descriptor is reached through the shared ring's entry. +#[repr(C, packed)] +#[derive(Clone, Copy)] +pub(crate) struct Notification { + record: super::super::queue::KickPredecessor, + reserved: [u8; 6], +} +pub(crate) fn notification(qid:u8, current:super::super::kick::KickTimestamp)->Result { + Ok(Notification{record:super::super::queue::KickPredecessor::new(qid,current)?,reserved:[0;6]}) +} +const _: () = assert!(core::mem::size_of::() == 24); + +/// Host-port transaction for an M5 render pair. The FIFO is a register pair +/// with a 16-byte stride, selecting slot 9. Queue validity +/// is asserted high then low, after its configuration payload. An existing +/// configuration is invalidated in the same order before replacement. +#[derive(Clone, Copy)] +pub(crate) struct HardwareSubmission { + writes: [(usize,u64);16], + count: usize, +} +impl HardwareSubmission { + pub(crate) fn advance_pair(fragment:u8,tiling:u8)->Result { + if fragment==tiling {return Err(EINVAL);} + let mut writes=[(0,0);16]; + for (n,qid) in [fragment,tiling].into_iter().enumerate() { + writes[n*2..n*2+2].copy_from_slice(&[ + (0x4090,Self::configuration_tag(qid)?+0x10),(0x4098,1)]); + } + Ok(Self{writes,count:4}) + } + /// Configure the compute queue at first use, then publish only new kicks. + pub(crate) fn compute(qid:u8,va:u64,priority:u8,configure:bool)->Result { + use super::super::queue::{Policy,DataMaster}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + let tag=Self::configuration_tag(qid)?; + if priority>=Policy::PRIORITIES || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + if configure { + let (low,high)=if qid<64 {(1u64<Result { + use super::super::{kick::QID_MAX,queue::{Policy,DataMaster}}; + const ADDRESS_MASK:u64=0x0000_07ff_ffff_ffe0; + if fragment.0==tiling.0 || priority>=Policy::PRIORITIES {return Err(EINVAL);} + let mut writes=[(0,0);16]; + let mut count=0; + for ((qid,va),engine) in [(fragment,DataMaster::Fragment),(tiling,DataMaster::Tiling)] { + if qid>QID_MAX || va & !ADDRESS_MASK != 0 {return Err(EINVAL);} + let completion=(3*(4-u64::from(priority))+0x3fe).wrapping_shl(54); + let tag=0x0020_0800_0002_1008 | (u64::from(qid)<<44); + let payload=((engine as u64+1)<<60)|va|completion; + let (low,high)=if qid<64 {(1u64<Result { + if qid>super::super::kick::QID_MAX {return Err(EINVAL);} + Ok(0x0020_0800_0002_1008 | (u64::from(qid)<<44)) + } + pub(crate) fn writes(&self)->&[(usize,u64)] {&self.writes[..self.count]} +} + +/// M5's scheduler release uses the shared identity and pointer layout. +pub(crate) const SCHEDULER_RELEASE_OPCODE: u32 = 0x13; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 66e7d07db48a52..3de7282c2c0201 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -386,8 +386,8 @@ pub(crate) struct TilingBinding { /// Queue whose configuration a [`QueueConfig`] carries. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum QueueConfigTarget { - /// A tiling queue and its parameter-buffer binding. - Tiling(TilingBinding), + /// A tiling queue, optionally carrying a parameter-buffer binding. + Tiling(Option), /// A fragment queue. Fragment, /// A compute queue. @@ -513,7 +513,7 @@ impl QueueConfig { return Err(EINVAL); } let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { - QueueConfigTarget::Tiling(binding) => { + QueueConfigTarget::Tiling(Some(binding)) => { if binding.fragment_qid > QID_MAX { return Err(EINVAL); } @@ -525,6 +525,8 @@ impl QueueConfig { u32::from(binding.fragment_qid), ) } + QueueConfigTarget::Tiling(None) => + (DataMaster::Tiling, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), }; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 8faa3ec06c8f7b..4c0cd39d6da156 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -20,6 +20,9 @@ mod v25g83; // The M5 graph is built only once an M5 board has a page-table boot path. #[allow(dead_code)] mod m5; +mod work_pair; +pub(crate) mod outer_credit; +pub(crate) use outer_credit::{OuterClaim, OuterScope}; const EVENT_RING_RECHECK: u32 = 256; @@ -138,6 +141,7 @@ impl ComputeScheduler { /// Checked views used only within one publication call. Every fallible operation completes /// before either member of a paired render publication is copied into shared memory. struct WorkPublication<'a> { + published: Option<&'a AtomicU64>, record: &'a super::fw::channels::WorkSlot, destination: *mut u8, producer: &'a AtomicU32, @@ -156,11 +160,21 @@ impl WorkPublication<'_> { ); } } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(1, Ordering::Release); } + fence(Ordering::SeqCst); + self.next + } } /// Session-long storage and mappings. Aliases are released before their backing objects. pub(super) struct InitData { m5: bool, + outer_credits: Option>, // Live publication uses interior writes under the device mutex. This owner is Send but not // Sync, so safe host code cannot issue those writes concurrently through shared references. _single_publisher: core::marker::PhantomData>, @@ -267,6 +281,7 @@ impl InitData { pub(super) fn new() -> Self { Self { m5: false, + outer_credits: None, _single_publisher: core::marker::PhantomData, aliases: KVec::new(), objects: KVec::new(), @@ -384,6 +399,34 @@ impl InitData { Ok(()) } + /// The statistics record tags the family's firmware emits. + pub(super) fn statistics_types(&self) -> u64 { + if self.m5 { + super::fw::m5::STATISTICS_TYPES + } else { + u64::from(super::fw::channels::STATISTICS_TYPES) + } + } + + /// A scheduler-state release record in the family's opcode. + pub(super) fn scheduler_release(&self, page_va: u64, id: [u8; 4]) -> super::fw::channels::SchedulerStateRelease { + let mut record = super::fw::channels::SchedulerStateRelease::new(page_va, id); + if self.m5 { + record.opcode = super::fw::m5::queue::SCHEDULER_RELEASE_OPCODE; + } + record + } + + /// Whether a control record of `opcode` holds the GPU powered until the firmware drains + /// it, by the family's rule. + pub(super) fn control_asserts_gpu_power(&self, opcode: super::fw::channels::DeviceControlOp) -> bool { + if self.m5 { + super::fw::m5::power::asserts_gpu_power(opcode as u32) + } else { + opcode.asserts_gpu_power() + } + } + pub(super) fn qos(&self) -> Result<&KernelObject> { self.object(self.qos) } @@ -543,15 +586,72 @@ impl InitData { Ok(()) } + /// Resource-specific wait domain; register only after dropping the device + /// mutex. A successful claim allocates no waiter fence. + pub(crate) fn outer_available(&mut self) -> Result> { + if self.outer_credits.is_none() { + self.outer_credits = Some(outer_credit::Credits::new()?); + } + Ok(self.outer_credits.as_ref().ok_or(EIO)?.available()) + } + + /// M5 TA, fragment and compute share the compute outer ring per priority. + /// The physical queue claim prevents reassignment while this credit lives. + pub(crate) fn reserve_outer(&mut self, priority: u8, count: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + self.outer_available()?; + let cursors = self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?; + self.outer_credits.as_ref().ok_or(EIO)?.reserve(priority, count, cursors) + } + + /// Hold the device mutex for this scope, including its Drop. The result + /// borrows only the claim, allowing the caller to publish through Firmware. + pub(crate) fn outer_scope<'a>(&self, claim: &'a mut OuterClaim) -> Result> { + self.outer_credits.as_ref().ok_or(EINVAL)?.scope(claim) + } + + pub(crate) fn outer_waiting(&self) -> bool { + self.outer_credits.as_ref().is_some_and(|credits| credits.waiting()) + } + + /// Return a notification for off-lock dispatch only on consumer progress. + /// Call while actual outer waiters exist; firmware may consume a config + /// without generating a GPU completion IRQ, so those waiters need polling. + pub(crate) fn poll_outer_progress(&self) -> Result>> { + let Some(credits) = self.outer_credits.as_ref().filter(|credits| credits.waiting()) else { + return Ok(None); + }; + let mut progressed = false; + for priority in 0..4 { + progressed |= credits.progressed(priority, + self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?)?; + } + Ok(progressed.then(|| credits.available())) + } + + fn outer_capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + match &self.outer_credits { + Some(credits) => credits.capacity(priority, cursors), + None => outer_credit::free(cursors), + } + } + /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - /// `priority` selects the engine's ring of that priority class. + /// `priority` selects the engine's ring of that priority class. On M5, tiling, + /// fragment and compute share the compute ring, whose unclaimed credits this checks. pub(super) fn work_ready( &self, priority: u8, engine: super::fw::queue::DataMaster, ) -> Result { + use super::fw::queue::DataMaster; + if self.m5 { + let needed = if engine == DataMaster::Compute { 1 } else { 2 }; + return Ok(self.outer_capacity(priority, + self.work_cursors(priority, DataMaster::Compute)?)? >= needed); + } let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) @@ -566,11 +666,17 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - // Compute publications reserve their slots through the port. + // M5 compute is a shared-ring submission on the compute ring, published here + // under the device mutex; the classic compute path reserves through the port. + 2 if self.m5 => DataMaster::Compute, _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; + if self.m5 && engine == DataMaster::Compute + && self.outer_capacity(priority, [consumer, consumer_b, producer])? < 1 { + return Err(EBUSY); + } let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); @@ -584,7 +690,11 @@ impl InitData { + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; + let published = if self.m5 && engine == DataMaster::Compute { + self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()? + } else { None }; Ok(WorkPublication { + published, record, destination, producer, @@ -619,6 +729,53 @@ impl InitData { Ok(values) } + /// Publishes an M5 compute record on the shared compute ring under the device mutex. + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) + } + + /// Reserve both common-ring slots before touching either host port. The + /// returned views retain the graph borrow until the pair is published. + pub(super) fn m5_work_pair( + &self, + priority: u8, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + if !self.m5 + || !matches!(fragment.data_master, 1 | 2) + || !matches!(tiling.data_master, 0 | 2) + { + return Err(EINVAL); + } + let cursors = self.work_cursors(priority, DataMaster::Compute)?; + if self.outer_capacity(priority, cursors)? < 2 { return Err(EBUSY); } + let window = work_pair::Window::new(cursors)?; + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let bundle = self.object(self.bundle)?; + let destinations = [window.start, window.next[0]].map(|index| { + bundle.pointer(self.work_ring_offset(channel) + + index as usize * size_of::(), size_of::()) + }); + let [fragment_destination, tiling_destination] = destinations; + let destinations = [fragment_destination?, tiling_destination?]; + let producer = self.object(self.cluster)?.word(self.state_offset(Role::Primary) + + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER)?; + let copy = |slot: &channels::WorkSlot| channels::WorkSlot::new( + DataMaster::Compute, slot.queue_va, slot.qid, slot.items_end, slot.first != 0, + ); + let records = [copy(fragment)?, copy(tiling)?]; + // SAFETY: Complete, distinct slots were checked in the retained bundle; + // the device's single publisher holds its mutex across this borrow. + let published = self.outer_credits.as_ref().map(|credits| credits.counter(priority)).transpose()?; + Ok(unsafe { work_pair::Publication::new(records, destinations, producer, window, published) }) + } + /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -637,6 +794,13 @@ impl InitData { { return Err(EINVAL); } + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + commit_accounting()?; + publish_late_tiling(); + return Ok(if deferred { pair.next() } else { pair.publish() }); + } let fragment = self.work_publication(priority, fragment)?; let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); @@ -683,6 +847,7 @@ impl InitData { core::mem::size_of_val(record), )?; Ok(WorkPublication { + published: None, record, destination, producer, @@ -700,6 +865,12 @@ impl InitData { next: [u32; 2], ) -> Result { use super::fw::queue::DataMaster; + if self.m5 { + let pair = self.m5_work_pair(priority, fragment, tiling)?; + if pair.next() != next { return Err(EBUSY); } + pair.publish(); + return Ok(()); + } if fragment.data_master != DataMaster::Fragment as u32 || tiling.data_master != DataMaster::Tiling as u32 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs new file mode 100644 index 00000000000000..988cbe28fb5f6e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pending capacity claims for M5's per-priority shared outer ring. Firmware +//! consumption frees posted slots; RAII cancellation frees unposted claims. + +use crate::g17::{availability::Availability, fw::channels::WORK_RING_SLOTS}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const PRIORITIES: usize = 4; + +pub(super) struct Credits { + reserved: [AtomicU32; PRIORITIES], + published: [AtomicU64; PRIORITIES], + consumers: [AtomicU64; PRIORITIES], + scoped: AtomicBool, + available: Arc, +} + +pub(crate) struct OuterClaim { + domain: Arc, + priority: usize, + remaining: u32, +} + +/// Borrows the claim only. The caller must hold the device mutex from entry +/// through Drop: its temporarily lent credits are available to that publisher. +pub(crate) struct OuterScope<'a> { + claim: &'a mut OuterClaim, + before: u64, +} + +pub(super) fn free([a, b, producer]: [u32; 3]) -> Result { + if [a, b, producer].iter().any(|&v| v >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(((a + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS) + .min((b + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS)) +} + +impl Credits { + pub(super) fn new() -> Result> { + Ok(Arc::new(Self { + reserved: [const { AtomicU32::new(0) }; PRIORITIES], + published: [const { AtomicU64::new(0) }; PRIORITIES], + consumers: [const { AtomicU64::new(u64::MAX) }; PRIORITIES], + scoped: AtomicBool::new(false), + available: Availability::new()?, + }, GFP_KERNEL)?) + } + + pub(super) fn available(&self) -> Arc { self.available.clone() } + + pub(super) fn waiting(&self) -> bool { self.available.has_waiters() } + + pub(super) fn capacity(&self, priority: u8, cursors: [u32; 3]) -> Result { + let pending = self.reserved.get(usize::from(priority)).ok_or(EINVAL)? + .load(Ordering::Acquire); + Ok(free(cursors)?.saturating_sub(pending)) + } + + /// Reserve under the sole publisher mutex. Off-lock cancellation may only + /// reduce reserved counts, so the sampled capacity is conservative. + pub(super) fn reserve(self: &Arc, priority: u8, count: u32, + cursors: [u32; 3]) -> Result + { + if count == 0 || count >= WORK_RING_SLOTS { return Err(EINVAL); } + if self.scoped.load(Ordering::Acquire) { return Err(EBUSY); } + let index = usize::from(priority); + let counter = self.reserved.get(index).ok_or(EINVAL)?; + let seen = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + self.consumers[index].compare_exchange(u64::MAX, seen, Ordering::AcqRel, Ordering::Acquire).ok(); + if self.capacity(priority, cursors)? < count { return Err(EBUSY); } + counter.fetch_add(count, Ordering::AcqRel); + Ok(OuterClaim { domain: self.clone(), priority: index, remaining: count }) + } + + pub(super) fn scope<'a>(self: &Arc, claim: &'a mut OuterClaim) -> Result> { + if !Arc::ptr_eq(self, &claim.domain) { return Err(EINVAL); } + if self.scoped.swap(true, Ordering::AcqRel) { return Err(EBUSY); } + self.reserved[claim.priority].fetch_sub(claim.remaining, Ordering::AcqRel); + let before = self.published[claim.priority].load(Ordering::Acquire); + Ok(OuterScope { claim, before }) + } + + pub(super) fn counter(&self, priority: u8) -> Result<&AtomicU64> { + self.published.get(usize::from(priority)).ok_or(EINVAL) + } + + pub(super) fn progressed(&self, priority: u8, cursors: [u32; 3]) -> Result { + free(cursors)?; + let counter = self.consumers.get(usize::from(priority)).ok_or(EINVAL)?; + let current = (u64::from(cursors[0]) << 32) | u64::from(cursors[1]); + let previous = counter.swap(current, Ordering::AcqRel); + Ok(previous != u64::MAX && previous != current) + } +} + +impl Drop for OuterScope<'_> { + fn drop(&mut self) { + let domain = &self.claim.domain; + let published = domain.published[self.claim.priority].load(Ordering::Acquire) + .wrapping_sub(self.before); + // Every publication also checks unreserved physical capacity. If an + // enclosing transaction posts extra records, only this claim's own + // credits are consumed here; another claim can never be debited. + self.claim.remaining -= published.min(u64::from(self.claim.remaining)) as u32; + domain.reserved[self.claim.priority].fetch_add(self.claim.remaining, Ordering::AcqRel); + domain.scoped.store(false, Ordering::Release); + } +} + +impl Drop for OuterClaim { + /// As for physical queue claims, drop outside the device mutex because + /// availability notification can run DMA-fence callbacks. + fn drop(&mut self) { + if self.remaining != 0 { + self.domain.reserved[self.priority].fetch_sub(self.remaining, Ordering::AcqRel); + self.domain.available.notify(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs new file mode 100644 index 00000000000000..eafd6daac4a856 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/work_pair.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Two render queues sharing one outer ring. Capacity and both destinations +//! are checked before the host-port transaction can make either queue runnable. + +use crate::g17::fw::channels::{WorkSlot, WORK_RING_SLOTS}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Window { + pub(super) start: u32, + pub(super) next: [u32; 2], +} + +impl Window { + pub(super) fn new([consumer, consumer_b, start]: [u32; 3]) -> Result { + if [consumer, consumer_b, start].iter().any(|&p| p >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + let next = [(start + 1) % WORK_RING_SLOTS, (start + 2) % WORK_RING_SLOTS]; + if next.iter().any(|&p| p == consumer || p == consumer_b) { + return Err(EBUSY); + } + Ok(Self { start, next }) + } +} + +pub(in crate::g17) struct Publication<'a> { + published: Option<&'a AtomicU64>, + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, +} + +impl<'a> Publication<'a> { + /// SAFETY: Both destinations must identify distinct, writable WorkSlot + /// storage in the retained ring at window.start and window.next[0]. The + /// caller holds the sole host publication lock until this value is dropped. + pub(super) unsafe fn new( + records: [WorkSlot; 2], + destinations: [*mut u8; 2], + producer: &'a AtomicU32, + window: Window, + published: Option<&'a AtomicU64>, + ) -> Self { + Self { records, destinations, producer, window, published } + } + + pub(in crate::g17) fn next(&self) -> [u32; 2] { self.window.next } + + pub(in crate::g17) fn prepare(&self) { + for (record, destination) in self.records.iter().zip(self.destinations) { + // SAFETY: Constructor checked both live destinations. Records are + // initialized, padding-free integer structures owned by this value. + unsafe { + core::ptr::copy_nonoverlapping( + (record as *const WorkSlot).cast::(), destination, size_of::(), + ); + } + } + } + + /// Publish both records with one cursor update, after both inner queues and + /// their accounting are visible. No fallible operation follows the port write. + pub(in crate::g17) fn publish(self) -> [u32; 2] { + self.prepare(); + fence(Ordering::SeqCst); + self.producer.store(self.window.next[1], Ordering::Relaxed); + if let Some(published) = self.published { published.fetch_add(2, Ordering::Release); } + fence(Ordering::SeqCst); + self.window.next + } +} diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 6499cae106309b..5e940ab70ad190 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -538,11 +538,11 @@ impl Queue { mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined, )?; - let mut range = alloc.uat.geometry().kernel_range(); - range.start += t8140::dynamic::KERNEL_OFFSET; + // The UAT's dynamic range: on M5 it lies past the boot graph, its unmapped sentinel and the + // firmware register aperture. let high = low.alias_in( alloc.uat.kernel_vm(), - range, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, )?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index e94fc7f11aaff3..e8ba1242467412 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -747,6 +747,51 @@ impl Firmware { power::release(self) } + /// Retry a temporarily unavailable transaction once under a registration + /// power hold. Other access errors propagate to the retained owner. + fn submit_m5_work(&mut self, submission: &fw::m5::queue::HardwareSubmission) -> Result { + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN => { + self.acquire_registration_power()?; + self.bridge.regs.submit_m5_work(submission) + } + result => result, + } + } + + /// Both outer slots and their capacity are checked before any host-port + /// write. An EAGAIN retry drops the old views and revalidates them under a + /// power hold. After a successful transaction, publication is infallible. + fn submit_m5_render_pair( + &mut self, + priority: u8, + records: &[fw::channels::WorkSlot; 2], + submission: &fw::m5::queue::HardwareSubmission, + commit_accounting: impl FnOnce() -> Result, + publish_inner: impl FnOnce(), + deferred: bool, + ) -> Result<[u32; 2]> { + let mut commit_accounting = Some(commit_accounting); + let mut publish_inner = Some(publish_inner); + for attempt in 0..2 { + let pair = self.init.m5_work_pair(priority, &records[0], &records[1])?; + pair.prepare(); + if let Some(commit) = commit_accounting.take() { commit()?; } + match self.bridge.regs.submit_m5_work(submission) { + Err(error) if error == EAGAIN && attempt == 0 => { + drop(pair); + self.acquire_registration_power()?; + } + Err(error) => return Err(error), + Ok(()) => { + if let Some(publish) = publish_inner.take() { publish(); } + return Ok(if deferred { pair.next() } else { pair.publish() }); + } + } + } + Err(EAGAIN) + } + /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { self.port.note_gpu_activity(); @@ -1810,6 +1855,9 @@ impl gpu::Gpu for Gpu { .uat .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; + if self.cfg.profile.is_m5() { + return Ok(vm); + } let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; let ranges = hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics()?.va).ok_or(EINVAL)?; @@ -1838,7 +1886,9 @@ impl gpu::Gpu for Gpu { self.geometry.user_usable_range(), self.shared.variant, )?; - let policy = scheduling_policy(priority)?; + let policy = if self.cfg.profile.is_m5() { + fw::m5::queue::scheduling_policy(priority)? + } else { scheduling_policy(priority)? }; let admission_started = now_ns(); let (backend, slot) = loop { let observed = self.shared.queues.observe(); @@ -1889,8 +1939,7 @@ impl gpu::Gpu for Gpu { let mapping = { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + let addresses = firmware.uat.kernel_dynamic_range(); bo.map_range_into_range( firmware.uat.kernel_vm(), range, diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 716e5452d4bae7..b952c998c35ef1 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -447,9 +447,11 @@ pub(crate) enum Family { WorkState = 1, QueueState = 2, QueueConfig = 3, + /// An M5 context's scheduler page, whose mapping the M5 firmware caches. + Scheduler = 4, } -const POOL_FAMILIES: usize = 4; +const POOL_FAMILIES: usize = 5; const POOL_SLOTS: usize = 64; const POOL_BYTES: usize = 128 * 1024 * 1024; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 76447a297c5ec9..687cf685f81a67 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -629,12 +629,12 @@ impl Pair { for stage in 0..2 { configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { target: if stage == 0 { - QueueConfigTarget::Tiling(TilingBinding { + QueueConfigTarget::Tiling(Some(TilingBinding { pb_va: self.memory.manager.state_va(), pb_token: lease.buffer_token, pb_slot: self.buffer_id().into(), fragment_qid: qids[1], - }) + })) } else { QueueConfigTarget::Fragment }, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c5..ca74d9524c698d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -20,7 +20,7 @@ use kernel::{prelude::*, sync::Arc}; /// context awaiting an executed replacement keeps its separate retirement owner. pub(crate) struct ParkedPair { kicks: Option<[kick::Queue; 2]>, - _scheduler: Arc, + _scheduler: Arc, ids: [kick::Id; 2], slot: u8, ordinal: u64, diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index 246aa5f02edbfa..a6f66bbcd65a89 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -5,7 +5,7 @@ //! Writers and readers hold the device mutex; reads never power the GPU up. use super::{ - fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + fw::channels::EVENT_WORDS, Role, }; use kernel::prelude::*; @@ -17,6 +17,9 @@ struct Sample { words: [u32; EVENT_WORDS], } +/// One sample per statistics tag a family's 64-bit mask can name. +const TAGS: usize = u64::BITS as usize; + pub(super) struct Reports { samples: KVec, } @@ -24,19 +27,20 @@ pub(super) struct Reports { impl Reports { pub(super) fn new() -> Result { // Allocate before firmware starts; receive and debugfs paths allocate nothing. - let mut samples = KVec::with_capacity(64, GFP_KERNEL)?; - for _ in 0..64 { + let mut samples = KVec::with_capacity(2 * TAGS, GFP_KERNEL)?; + for _ in 0..2 * TAGS { samples.push(Sample::default(), GFP_KERNEL)?; } Ok(Self { samples }) } - pub(super) fn record(&mut self, role: Role, words: [u32; EVENT_WORDS], host_ns: u64) { + /// `allowed` is the family's statistics tag mask (`InitData::statistics_types`). + pub(super) fn record(&mut self, role: Role, words: [u32; EVENT_WORDS], host_ns: u64, allowed: u64) { let tag = words[0]; - if tag >= u32::BITS || STATISTICS_TYPES & (1 << tag) == 0 { + if tag >= u64::BITS || allowed & (1u64 << tag) == 0 { return; } - let sample = &mut self.samples[role as usize * 32 + tag as usize]; + let sample = &mut self.samples[role as usize * TAGS + tag as usize]; sample.count = sample.count.saturating_add(1); sample.host_ns = host_ns; sample.words = words; @@ -56,8 +60,8 @@ impl Reports { writeln!( f, "role={} tag={:#x} count={} host_ns={} words={:08x?}", - index / 32, - index % 32, + index / TAGS, + index % TAGS, sample.count, sample.host_ns, sample.words diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 0222ae4c478c87..2022d3883f1c39 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -7,7 +7,7 @@ use crate::g17::{ buffer::BufferIds, channel::Rings, context::Context, - fw::channels::{ControlRecord, SchedulerStateRelease}, + fw::channels::ControlRecord, initdata::InitData, queue::render::RetiredBinding, recovery::Memory as _, @@ -126,13 +126,10 @@ impl Host for Service<'_> { fn control_backpressured(&self) -> bool { self.render.control_backpressured() } - fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result { Rings::publish_control( self.init, - &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( - record.page_va, - record.id, - )), + &ControlRecord::SchedulerStateRelease(self.init.scheduler_release(page_va, id)), ) } fn notify(&mut self) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index fa2a72a77f0609..39208dc7b3f34b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -4,7 +4,7 @@ //! private-memory release and control-ring consumption are separate ownership witnesses. //! The device mutex serializes this state with the Registry's displaced graph ownership. -use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use super::{context::Context, freelist::control_consumed}; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -82,7 +82,9 @@ pub(crate) trait Host { fn compute_idle(&self, context: &Arc) -> bool; fn flists_released(&self, context: &Arc) -> bool; fn control_backpressured(&self) -> bool; - fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + /// Publishes the release of the scheduler page at `page_va`, identity bytes in record + /// order, in the family's record. + fn publish_release(&mut self, page_va: u64, id: [u8; 4]) -> Result; fn notify(&mut self) -> Result; fn detach(&mut self, context: &Arc) -> Result; fn report(&mut self, context: &Arc, failure: Failure); @@ -215,11 +217,7 @@ impl Pending { } let result = (|| { let id = entry.context.release_identity()?; - let record = SchedulerStateRelease::new( - entry.context.scheduler_va(), - [id[3], id[0], id[1], id[2]], - ); - host.publish(&record) + host.publish_release(entry.context.scheduler_va(), [id[3], id[0], id[1], id[2]]) })(); let next = match result { Ok(next) => next, diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index a984b9d2520387..2dacc4c95a16c5 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -610,6 +610,55 @@ impl G17Resources { Ok(true) } + /// Publishes through the M5 host command port, including the kick that + /// makes an idle queue runnable. Core activity is not a prerequisite. + pub(crate) fn submit_m5_work( + &self, + submission: &crate::g17::fw::m5::queue::HardwareSubmission, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + for &(offset, value) in submission.writes() { + sgx.try_write64(value, offset)?; + } + Ok(()) + } + + /// The M5 fault window, read for recovery evidence: the fault status, the MMU fault status + /// and address, and three further fault words. The selector is observed, never changed. + pub(crate) fn m5_fault_registers(&self) -> Result<[u64; 6]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + Ok([ + sgx.try_read64(0xd800)?, + sgx.try_read64(0xd8c0)?, + sgx.try_read64(0xd8c8)?, + sgx.try_read64(0xc078)?, + sgx.try_read64(0xc008)?, + sgx.try_read64(0xc028)?, + ]) + } + + /// Sample only the two owned queue configuration words through the + /// host tag/data port. The caller holds the device mutex and power hold. + pub(crate) fn m5_queue_configurations(&self, qids: [u8; 2]) -> Result<[u64; 2]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Err(EAGAIN); + } + let mut values = [0; 2]; + for (value, qid) in values.iter_mut().zip(qids) { + let tag = crate::g17::fw::m5::queue::HardwareSubmission::configuration_tag(qid)?; + sgx.try_write64(tag, 0x4090)?; + *value = sgx.try_read64(0x4098)?; + } + Ok(values) + } + /// Registers a queue KSM slot through the firmware mailbox. pub(crate) fn register_ksm_queue(&self, qid: u8, descriptor: u64) -> Result { if qid > 127 { @@ -770,6 +819,22 @@ impl G17Resources { /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. + /// Prepares the GPU for the M5 firmware before its initialization: sets bit 0 of the + /// firmware start control, which reads 0x30000 or 0x30001 until then, and enables dynamic + /// core power gating. + pub(crate) fn prepare_m5_firmware(&self) -> Result { + { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let value = sgx.try_read32(0xe90000)?; + if value != 0x30000 && value != 0x30001 { + return Err(EIO); + } + sgx.try_write32(value | 1, 0xe90000)?; + } + self.enable_g17_gating() + } + pub(crate) fn enable_g17_gating(&self) -> Result { let sgx = self.sgx.try_access().ok_or(ENODEV)?; let sgx = (&*sgx).as_view(); From 45fad2a53e46635403bf034def974d3f895d69a1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:23 -0800 Subject: [PATCH 066/171] drm/asahi: g17: M5: accept the M5 render and compute program UAPI Add the render and compute program resources, the shader-write, stage-prefix and whole-attachment render flags, matrix and no-query compute, and their feature bits. The command parser takes the resource commands and the stage-prefix tail; validation applies the M5 command set only where the firmware variant has it. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..a507978ff411 (7 commits); SPTM/APIF removed; adapted to integration's UAPI numbering, ray-query stages and fw::Variant; the M5 UAPI flags and features in the UAPI header with agx3c Mesa's numbers, the stage-prefix fields appended to drm_asahi_cmd_render; interim forms until the render submission commit; comments and names reworded; M5 paths restored where the first rewrite lost them: VM_BIND accepts DRM_ASAHI_BIND_SPARSE_TILE; original 6613f5ab7b7e, e25ba8a6fb2b, bd4a3d7a0c97, 2f58522abec7, 2c5c77b6aaba, e393a52c7151, a507978ff411 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit 0966df119d3e65d48edea5272a184daa386d4a49 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/file.rs | 77 ++++- drivers/gpu/drm/asahi/g15/wire.rs | 2 +- drivers/gpu/drm/asahi/g17/command.rs | 173 +++++++++- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 118 +++++++ drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fw/render.rs | 110 +++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 5 + drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 8 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/render_resources.rs | 321 ++++++++++++++++++ include/uapi/drm/asahi_drm.h | 184 ++++++++++ 12 files changed, 981 insertions(+), 22 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/render_resources.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index dcc5fdc590fce8..237315c4e3d0f2 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -58,6 +58,77 @@ use kernel::{ xarray, // }; +// The M5 runtime's UAPI numbers. The header's values are Mesa's; the M5 queue and +// GET_PARAMS paths use these names. + +/// DRM_ASAHI_BIND_SPARSE_TILE: with SINGLE_PAGE, fill the range +/// from kernel sparse tiles (read-only: global zero tile; writable: per-VM +/// sink) instead of the GEM's first page. The GEM only anchors the GPUVA. +pub(crate) const DRM_ASAHI_BIND_SPARSE_TILE: u32 = uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SPARSE_TILE; +/// DRM_ASAHI_FEATURE_SPARSE_TILES: BIND_SPARSE_TILE accepted. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_SPARSE_TILES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SPARSE_TILES as u64; +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_MATRIX: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_MATRIX as u64; +/// DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: render programs accept +/// DRM_ASAHI_RENDER_PROGRAM_TASK and such task (object-stage) kicks get the +/// TA mesh rendering buffer, which backs the object stage's payload address. +/// Reported only while m5_mesh_storage_chunks != 0. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_TASK_MESH_STORAGE as u64; +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES as u64; +/// D32 2x/4x ZLS admission; no compressed stencil or resolve bits. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION as u64; +/// DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: single-layer +/// 2x/4x ZLS admission in the compressed Z/S form, stencil included. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION as u64; +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_SHADER_WRITES; +/// Renders may carry separate tiling prefixes of both subqueues +/// (DRM_ASAHI_RENDER_STAGE_PREFIXES). +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES as u64; +/// The tiling half waits only on the render and compute prefixes of the command's stage-prefix +/// fields, its fragment half on the whole ones. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_STAGE_PREFIXES: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; +/// Renders may declare that they write whole attachments only +/// (DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS). +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS as u64; +/// The render area covers every attachment's full extent, so its tile stores +/// write no pixel outside it: an overlap with another such render without an +/// ordering dependency is the application's own race. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +/// DRM_ASAHI_RENDER_PROGRAM_TASK, only with DRM_ASAHI_RENDER_PROGRAM_OBJECT: the object +/// program is an API task stage. +pub(crate) const DRM_ASAHI_RENDER_PROGRAM_TASK: u32 = uapi::DRM_ASAHI_RENDER_PROGRAM_TASK; +/// Bit 16, as Mesa defines it. +#[allow(dead_code)] +pub(crate) const DRM_ASAHI_COMPUTE_USES_MATRIX: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_USES_MATRIX as u32; +/// The compute command neither copies nor resets queries, so it cannot observe +/// timestamps the host publishes after earlier commands complete +/// (DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS). Accepted; DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS +/// is not reported: with it, top-of-pipe timestamps are read out of order. +pub(crate) const DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as u32; + const DEBUG_CLASS: DebugFlags = DebugFlags::File; pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; @@ -925,11 +996,13 @@ impl File { if op.flags & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE - | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE + | DRM_ASAHI_BIND_SPARSE_TILE) != 0 { return Err(EINVAL); } + let sparse_tile = op.flags & DRM_ASAHI_BIND_SPARSE_TILE != 0; let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; let prot = match (read, write) { @@ -956,7 +1029,7 @@ impl File { op.offset, prot, single_page, - false, + sparse_tile, )?)?; } vm.reserve_deferred_user_unmaps(unmaps)?; diff --git a/drivers/gpu/drm/asahi/g15/wire.rs b/drivers/gpu/drm/asahi/g15/wire.rs index 6fbb360e11a610..4157be3495845c 100644 --- a/drivers/gpu/drm/asahi/g15/wire.rs +++ b/drivers/gpu/drm/asahi/g15/wire.rs @@ -181,7 +181,7 @@ unsafe impl AnyBitPattern for drm_asahi_cmd_render {} unsafe impl AnyBitPattern for drm_asahi_cmd_compute {} const _: () = { - assert!(size_of::() == 448); + assert!(size_of::() == 480); assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_vdm) == 320); assert!(size_of::() == 200); assert!(offset_of!(uapi::drm_asahi_cmd_compute, scs_layout) == 128); diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index c65f7de6ac9157..e3f42aaaa7b560 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -10,7 +10,8 @@ use super::fw::{ }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; -use kernel::{prelude::*, uapi}; +use kernel::{prelude::*, sync::Arc, uapi}; +use super::render_resources::{ComputePrograms, Programs}; const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); @@ -29,14 +30,22 @@ const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; const MAX_DIMENSION: u16 = 16384; const MAX_LAYERS: u16 = 2048; -const MAX_TILE_BUFFER: u64 = 32768; +pub(crate) const MAX_TILE_BUFFER: u64 = 32768; +// Eight RGBA8 render targets take 64 bytes a pixel across a 32x32 tile. +pub(crate) const M5_MAX_TILE_BUFFER: u64 = 65536; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const STAGE_PREFIXES: u32 = crate::file::DRM_ASAHI_RENDER_STAGE_PREFIXES; +const WHOLE_ATTACHMENTS: u32 = crate::file::DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS; +const SHADER_WRITES: u32 = crate::file::DRM_ASAHI_RENDER_SHADER_WRITES; +/// Render flags only the M5 runtime implements. +const M5_RENDER_FLAGS: u32 = SHADER_WRITES | STAGE_PREFIXES | WHOLE_ATTACHMENTS; const RENDER_FLAGS: u32 = - PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS | SHADER_WRITES + | STAGE_PREFIXES | WHOLE_ATTACHMENTS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -84,11 +93,34 @@ impl Attachments { } } +/// Where a render command's stage-prefix fields start: after the ray-query stages. +const RENDER_STAGE_PREFIXES: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, ta_cdm_barrier); + +/// A render command. With DRM_ASAHI_RENDER_STAGE_PREFIXES it is the full command, with its +/// reserved fields zero; without it the stage-prefix fields are absent or zero. +fn render_payload(bytes: &[u8]) -> Result { + let cmd: uapi::drm_asahi_cmd_render = payload(bytes)?; + let valid = if cmd.flags & STAGE_PREFIXES != 0 { + bytes.len() == size_of::() + && cmd.flags & FRAGMENT_BARRIERS == 0 + && cmd.reserved0 == 0 + && cmd.reserved1 == 0 + && cmd.reserved2 == 0 + && cmd.reserved3 == 0 + } else { + !bytes.get(RENDER_STAGE_PREFIXES..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) + }; + if !valid { + return Err(EINVAL); + } + Ok(cmd) +} + // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), + Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2], Option>), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables, Option>), } pub(crate) struct Command { @@ -103,6 +135,8 @@ pub(crate) struct Parser<'a> { bytes: &'a [u8], counts: [u16; 2], attachments: [Attachments; 3], + render_resources: Option>, + compute_resources: Option>, } impl<'a> Parser<'a> { @@ -111,6 +145,8 @@ impl<'a> Parser<'a> { bytes, counts: [0; 2], attachments: [Attachments::EMPTY; 3], + render_resources: None, + compute_resources: None, } } @@ -138,18 +174,24 @@ impl<'a> Parser<'a> { { self.counts[0] += 1; no_timestamp_copies(bytes, RENDER_TIMESTAMP_COPIES)?; + // The ray-query stages follow prior_cdm, and the stage-prefix fields follow them. + let ray_query = bytes + .get(RENDER_BASE_SIZE..bytes.len().min(RENDER_STAGE_PREFIXES)) + .unwrap_or(&[]); Payload::Render( - payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + render_payload(bytes)?, [self.attachments[0], self.attachments[1]], - RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + RayQueryStage::parse_pair(ray_query)?, + self.render_resources.clone(), ) } else { self.counts[1] += 1; no_timestamp_copies(bytes, COMPUTE_TIMESTAMP_COPIES)?; Payload::Compute( - payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + payload(bytes)?, self.attachments[2], IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + self.compute_resources.clone(), ) }; return Ok(Some(Command { barriers, payload })); @@ -164,6 +206,25 @@ impl<'a> Parser<'a> { - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; self.attachments[slot] = Attachments::parse(bytes)?; } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_RENDER_RESOURCES_V2 => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { return Err(EINVAL); } + self.render_resources = if bytes.is_empty() { None } + else { Some(Arc::new(Programs::parse_v2(bytes)?, GFP_KERNEL)?) }; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_RESOURCES => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + self.compute_resources = if bytes.is_empty() { None } + else { Some(Arc::new(ComputePrograms::parse(bytes)?, GFP_KERNEL)?) }; + } _ => return Err(EINVAL), } } @@ -311,13 +372,27 @@ pub(crate) enum Validated { long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], + resources: Option>, + plan: Option, attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { long_job: bool, intersection: IntersectionTables, + matrix_ops: bool, + /// The command may copy or reset queries, so it must follow the host's + /// timestamp publication for earlier commands. + query_access: bool, + resources: Option>, + plan: Option, scratch: ScratchRequest, + cdm_auxiliary_allocation: u32, + /// The M5 intersection-table binding (dj's `intersection`; that name is the G17P + /// tables' above). + m5_intersection: super::fw::m5::compute::IntersectionTables, + /// The M5 shader-context scratch request (`scratch` is G17P's). + m5_scratch: super::fw::m5::compute::ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -343,8 +418,20 @@ impl Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, ..) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + + /// The tiling half's own render and compute prefixes (barrier, exact prior), if the + /// command carries them. + pub(crate) fn tiling_prefixes(&self) -> Option<([u16; 2], [u64; 2])> { + match self { + Self::Render(cmd, ..) if cmd.flags & STAGE_PREFIXES != 0 => Some(( + [cmd.ta_vdm_barrier, cmd.ta_cdm_barrier], + [cmd.ta_prior_vdm, cmd.ta_prior_cdm], + )), + _ => None, } } @@ -354,9 +441,13 @@ impl Payload { variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { + // Everything M5 accepts beyond the G17P command set is gated on this. + let m5 = variant.m5_commands; + let max_tile_buffer = if m5 { M5_MAX_TILE_BUFFER } else { MAX_TILE_BUFFER }; match self { - Self::Render(cmd, attachments, ray_query) => { - if cmd.flags & !RENDER_FLAGS != 0 + Self::Render(cmd, attachments, ray_query, resources) => { + if cmd.flags & !(if m5 { RENDER_FLAGS } else { RENDER_FLAGS & !M5_RENDER_FLAGS }) != 0 + || (!m5 && resources.is_some()) || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) || !(1..=MAX_LAYERS).contains(&cmd.layers) @@ -371,13 +462,14 @@ impl Payload { 1 => 0, 2 => 1, 4 => 2, + 8 if m5 => 3, _ => return Err(EINVAL), }; if u64::from(cmd.sample_size_B) * u64::from(cmd.utile_width_px) * u64::from(cmd.utile_height_px) * u64::from(cmd.samples) - > MAX_TILE_BUFFER + > max_tile_buffer || cmd.vdm_ctrl_stream_base == 0 || cmd.vdm_ctrl_stream_base & 3 != 0 || cmd.isp_scissor_base == 0 @@ -427,6 +519,9 @@ impl Payload { depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, + stage_prefixes: cmd.flags & STAGE_PREFIXES != 0, + whole_attachments: cmd.flags & WHOLE_ATTACHMENTS != 0, + shader_writes_memory: cmd.flags & SHADER_WRITES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), @@ -469,18 +564,37 @@ impl Payload { if code != 0 { require_range(space, code, 64, Access::Read)?; } } } + if let Some(programs) = resources { + for stage in programs.stage_scs()? { + let intersection = stage.intersection; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } + } + } Ok(Validated::Render { long_job: super::timeout::select(false, ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, + resources: resources.clone(), + plan: None, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments, intersection) => { - let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; - if cmd.cdm_ctrl_stream_base & 3 != 0 + Self::Compute(cmd, attachments, intersection, resources) => { + let as_build = if m5 { false } else { super::timeout::as_build(cmd.flags).ok_or(EINVAL)? }; + if (m5 + && (cmd.flags + & !(crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX + | crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS) + != 0 + || cmd.reserved != 0)) + || (!m5 && resources.is_some()) + || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -495,7 +609,13 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; - let scratch = ScratchRequest::new(cmd.scs_layout)?; + // M5 shader-context scratch has its own (Gen2) layout; G17P's stays disabled. + let scratch = ScratchRequest::new(if m5 { 0 } else { cmd.scs_layout })?; + let m5_scratch = if m5 { + super::fw::m5::compute::ScratchRequest::new(cmd.scs_layout)? + } else { + super::fw::m5::compute::ScratchRequest::NONE + }; if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { return Err(refuse_ray_traversal()); } @@ -504,10 +624,29 @@ impl Payload { let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } + let m5_intersection = if m5 { + super::fw::m5::compute::IntersectionTables::new( + cmd.intersection_pool, cmd.intersection_pool_bytes, + cmd.intersection_first, cmd.intersection_count, + cmd.intersection_default_program, cmd.intersection_default_control)? + } else { + super::fw::m5::compute::IntersectionTables::NONE + }; + if m5_intersection.enabled() { + require_range(space, m5_intersection.pool, m5_intersection.pool_bytes, Access::Read)?; + let code = m5_intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, + matrix_ops: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_USES_MATRIX != 0, + query_access: cmd.flags & crate::file::DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS == 0, + resources: resources.clone(), plan: None, scratch, + m5_scratch, + cdm_auxiliary_allocation: cmd.cdm_auxiliary_allocation, + m5_intersection, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs new file mode 100644 index 00000000000000..73d23fbd563943 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 compute register programs. The operand and preemption layouts differ +//! from the other G17 profile; sharing a register count does not imply a shared +//! register program. M5 scratch has its own layout and two control writes. + +use kernel::prelude::*; +pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; + +/// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent +/// of the host's CPU page size. Layout sections and control registers are +/// specific to M5; the other G17 profile uses a different fixed region. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct ScratchRequest { + layout: u64, + driver_end: u32, + pages: u16, +} + +impl ScratchRequest { + pub(crate) const NONE: Self = Self { layout: 0, driver_end: 0, pages: 0 }; + + pub(crate) fn new(layout: u64) -> Result { + let cliques = (layout >> 24) & 0xff; + if cliques == 0 { return Ok(Self::NONE); } + let depth = layout & 0xffff; + let levels = (layout >> 16) & 0xff; + let user = (layout >> 32) & 0xffff; + let driver = layout >> 48; + let threads = cliques * 32; + // Four fixed sections followed by per-thread stacks, driver/user + // payloads and per-level state. All inputs are bounded bitfields. + let driver_end = 0x68000 + threads * (depth * 8 + driver); + // The register naming the end of the driver section holds a 24-bit + // byte offset. Reject truncation even if the total page count fits. + if driver_end > 0xffffff { return Err(EINVAL); } + let bytes = driver_end + threads * user + + cliques * 256 * (levels + u64::from(levels > 1)); + let pages = (bytes + 0x1fff) >> 12; + Ok(Self { layout, driver_end: driver_end as u32, + pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn layout(self) -> u64 { self.layout } + pub(crate) fn pages(self) -> u16 { self.pages } + pub(crate) fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(256) << 16) + } + pub(crate) fn window_word(self) -> u64 { + if self.enabled() { (1 << 24) | (((self.layout >> 24) & 0xff) << 16) } + else { 0xff0000 } + } + pub(crate) fn stack_word(self) -> u64 { + if self.enabled() { self.layout & 0xfff8 } else { 8 } + } + pub(crate) fn driver_word(self) -> u64 { + if !self.enabled() { return 0; } + let levels = (self.layout >> 16) & 0xff; + u64::from(self.driver_end) | ((levels + u64::from(levels > 1)) << 24) + } + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + if self.enabled() { 2 } else { 0 }) + } +} +/// ownership and readable ranges are checked by command validation. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // Resource sizes are encoded in 16 KiB pages. Table lengths + // have 18 block-count bits; truncating either field changes addressing. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, first, count, default_program, default_control }) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + [self.pool + u64::from(self.first) * 128 | (1u64 << 51), + 0xc0000000 | (u64::from(self.count) << 7), self.pool, + self.pool_bytes >> 14, self.default_program, self.default_control] + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index e33ada8dbdb881..54f5b2eab41f82 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -23,5 +23,6 @@ pub(crate) fn set_register_array( pub(crate) mod pool; pub(crate) mod pool_budget; pub(crate) mod queue; +pub(crate) mod compute; pub(crate) mod render_demand; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 8fcc25500ab2d0..b699c10923afee 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -243,6 +243,15 @@ pub(crate) struct RenderPass { pub(crate) few_primitives: bool, /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. pub(crate) fragment_barriers: bool, + /// Host-only: the tiling kick waits on its own render and compute prefixes + /// (`Order::tiling_wait_through`), the fragment kick on the whole ones. M5 only. + pub(crate) stage_prefixes: bool, + /// Host-only: the render writes whole attachments: its tile stores reach no pixel + /// outside its own render area (hazard ordering). M5 only. + pub(crate) whole_attachments: bool, + /// A graphics program of the render writes external memory (buffer or image stores, + /// atomics); render-target outputs do not count. M5 only. + pub(crate) shader_writes_memory: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -326,6 +335,10 @@ pub(crate) struct RenderResources { pub(crate) deflake_va: u64, /// GPU address of the tiling status word. pub(crate) ta_status_va: u64, + /// The TA mesh rendering buffer word for mesh/object kicks: the context VA + /// of the queue's mesh rendering buffer in the low 48 bits and its + /// 64 KiB chunk count above. Zero leaves the register out of the program. + pub(crate) mesh_storage: u64, /// GPU address of the fragment status word, as seen by the pass's VM. pub(crate) fragment_status_va: u64, /// Firmware alias of the fragment status word. @@ -469,7 +482,11 @@ struct PassWords { impl PassWords { fn new(pass: &RenderPass) -> Result { - if pass.samples_log2 > SAMPLES_LOG2_MAX { + Self::with_sample_limit(pass, SAMPLES_LOG2_MAX) + } + + fn with_sample_limit(pass: &RenderPass, max_samples_log2: u8) -> Result { + if pass.samples_log2 > max_samples_log2 { return Err(EINVAL); } let samples = 1u64 << pass.samples_log2; @@ -998,6 +1015,97 @@ fn partial_load_registers( ] } +/// The M5 render register storage, shared by its phases. Keep this in the caller's heap +/// workspace when constructing descriptors. +pub(crate) struct M5RegisterPrograms { + pub(crate) ta: [RegisterWrite; TA_REGISTERS], + pub(crate) fragment: [RegisterWrite; FRAGMENT_REGISTERS], + pub(crate) store: [RegisterWrite; PARTIAL_STORE_REGISTERS], + pub(crate) resume: [RegisterWrite; PARTIAL_RESUME_REGISTERS], + pub(crate) load: [RegisterWrite; PARTIAL_LOAD_REGISTERS], +} +// SAFETY: All members contain only integer register writes. +unsafe impl Zeroable for M5RegisterPrograms {} +impl M5RegisterPrograms { + /// M5 partial phases add only their own depth/stencil direction while + /// preserving the pass's existing operations and surface layout bits. + pub(crate) fn write_m5(&mut self, args: &RenderArgs<'_>) -> Result { + // The M5 forms of the shared lowering: the A18 Pro arrays without the SCS enable + // (the M5 adaptation brackets its own scratch and intersection words), the + // one-cluster deflake layout, and up to eight samples. + let none = super::render_rt::RayQueryStage { + intersection: super::compute::IntersectionTables::NONE, + scratch: super::compute::ScratchRequest::new(0)?, + }; + let args = &RenderArgs { + programs: RegisterPrograms::G17P, + deflake: Deflake::ONE_CLUSTER, + ray_query: [none; 2], + ..*args + }; + let g = Geometry::new(args.pass)?; + let w = PassWords::with_sample_limit(args.pass, 3)?; + without_scs_enable(&mut self.ta, &ta_registers(args, &g, &w)?, TA_SCS_ENABLE_INDEX)?; + without_scs_enable( + &mut self.fragment, + &fragment_registers(args, &g, &w), + FRAGMENT_SCS_ENABLE_INDEX, + )?; + self.store = partial_store_registers(args.pass, &w); + self.resume = partial_resume_registers(args.pass, &w); + self.load = partial_load_registers(args.pass, &w); + // The partial phases start from the pass's ZLS word with both directions of each + // present surface, as the M5 lowering always has. + let mut partial = args.pass.zls_control; + if args.pass.depth.va != 0 { + partial |= ZLS_LOAD_DEPTH | ZLS_STORE_DEPTH; + } + if args.pass.stencil.va != 0 { + partial |= ZLS_LOAD_STENCIL | ZLS_STORE_STENCIL; + } + for array in [&mut self.store[..], &mut self.resume[..], &mut self.load[..]] { + for register in array { + if register.reg == 0x15319 { + register.value = partial; + } + } + } + const LOAD: u64 = (1 << 14) | (1 << 15); + const STORE: u64 = (1 << 18) | (1 << 19); + for register in &mut self.store { + if register.reg == 0x15319 { + register.value = (register.value & !LOAD) | (args.pass.zls_control & LOAD); + } else if register.reg == 0x10039 + && (args.pass.depth.va != 0 || args.pass.stencil.va != 0) + { + // A partial store forces empty tiles when either stored ZLS + // address (depth or stencil) exists. + register.value |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + } + for register in &mut self.load { + if register.reg == 0x15319 { + register.value = (register.value & !STORE) | (args.pass.zls_control & STORE); + } + } + Ok(()) + } +} + +/// Copies a lowered array without its SCS enable entry. +fn without_scs_enable( + out: &mut [RegisterWrite; N], + lowered: &[RegisterWrite; M], + index: usize, +) -> Result { + if M != N + 1 || lowered.get(index).is_none_or(|write| write.reg != SCS_ENABLE) { + return Err(EINVAL); + } + out[..index].copy_from_slice(&lowered[..index]); + out[index..].copy_from_slice(&lowered[index + 1..]); + Ok(()) +} + /// Tiling (TA) work descriptor (tag 0). #[repr(C, packed)] #[derive(Debug)] diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index c5287e66a641d0..3119fb2a155ba4 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -77,6 +77,9 @@ pub(crate) struct Variant { pub(crate) compute: super::compute::ComputeProgram, pub(crate) opening_record: bool, pub(crate) split_firmware_area: bool, + /// Command validation applies the M5 command set: resource commands, the M5 render and + /// compute flags, 8x MSAA and the M5 tile-buffer limit. + pub(crate) m5_commands: bool, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -121,6 +124,7 @@ pub(crate) const V26A428: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -152,6 +156,7 @@ pub(crate) const V25G83: Variant = Variant { compute: super::compute::ComputeProgram::G17P, opening_record: true, split_firmware_area: false, + m5_commands: false, }; pub(crate) const V25F84: Variant = Variant { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index e8ba1242467412..cdc26aa341f0a2 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -27,6 +27,9 @@ mod calibration; mod channel; mod command; pub(crate) use command::Attachments; +// M5 render encodings, reached only once the M5 render runtime is wired. +#[allow(dead_code)] +mod render_resources; mod completion; pub(crate) mod context; mod data_segment; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index f6637f2858c26d..8717cad85ec62f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -268,8 +268,14 @@ impl Queue { &[] }; let count = frontier.len(); + // Only a command that copies or resets queries can observe timestamps the + // host has not yet published for its predecessors. + let query_access = match &command { + Validated::Compute { query_access, .. } => *query_access, + Validated::Render { .. } => true, + }; let timestamp_prefix = order.wait_through[index]; - let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let timestamps = if engine == Engine::Compute && query_access && timestamp_prefix.is_some() { let frontier = self.frontiers[index].as_slice(); let end = frontier .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 687cf685f81a67..e5494ffccd8e9f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -436,6 +436,7 @@ impl Pair { gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, + mesh_storage: 0, fragment_status_va: fragment_status, fragment_status_fw_va: fragment_fw, aux_fb_va: auxiliary, diff --git a/drivers/gpu/drm/asahi/g17/render_resources.rs b/drivers/gpu/drm/asahi/g17/render_resources.rs new file mode 100644 index 00000000000000..cb4dc51b65a9a9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/render_resources.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable linked-program storage, shared by commands in one submission. + +use super::fw::m5::{ + pool::PoolGeometry, + pool_budget::{History, RenderBudget, Sizes}, + render_demand::{ComputeStorage, FragmentStorage, RenderDemand, TileStorage, VertexStorage}, +}; +use super::fw::m5::compute::{ScratchRequest, IntersectionTables}; +use crate::util::Reader; +use kernel::{prelude::*, uapi}; + +// SAFETY: The UAPI record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program {} +// SAFETY: The versioned record contains only unsigned integer storage. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_render_program_v2 {} + +pub(crate) struct Programs { + entries: KVec, +} +impl Programs { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program = reader.read()?; + Self::validate(&entry)?; + entries.push(Self::wrap(entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + pub(crate) fn from_entries(values: &[uapi::drm_asahi_render_program]) -> Result { + if values.is_empty() { return Err(EINVAL); } + let mut entries = KVec::with_capacity(values.len(), GFP_KERNEL)?; + for entry in values { + Self::validate(entry)?; + entries.push(Self::wrap(*entry), GFP_KERNEL)?; + } + Ok(Self { entries }) + } + + fn wrap(legacy: uapi::drm_asahi_render_program) -> uapi::drm_asahi_render_program_v2 { + uapi::drm_asahi_render_program_v2 { legacy, threads: 0, usage_flags: 0, + scs_layout: 0, auxiliary_allocation: 0, reserved: 0, + intersection_pool: 0, intersection_pool_bytes: 0, + intersection_first: 0, intersection_count: 0, + intersection_default_program: 0, intersection_default_control: 0 } + } + + pub(crate) fn parse_v2(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_render_program_v2 = reader.read()?; + let mesh = entry.usage_flags & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH + | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0; + let task = entry.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0; + if entry.reserved != 0 || entry.usage_flags & !0xf != 0 + || entry.usage_flags & 6 == 6 || entry.legacy.flags != 0 + || (task && entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT == 0) + || entry.legacy.stage > 1 || (mesh && (entry.legacy.stage != 0 + || entry.threads == 0 || entry.legacy.auxiliary_count != 0 + || entry.legacy.imageblock_bytes != 0 || entry.legacy.multisample_bytes != 0)) + || (!mesh && entry.threads != 0) { return Err(EINVAL); } + if !mesh { Self::validate(&entry.legacy)?; } + let scratch = ScratchRequest::new(entry.scs_layout)?; + let intersection = Self::intersection(&entry)?; + if scratch.enabled() != (entry.usage_flags & uapi::DRM_ASAHI_RENDER_PROGRAM_SCS != 0) + || scratch.enabled() != intersection.enabled() + || (!scratch.enabled() && (entry.scs_layout != 0 || entry.auxiliary_allocation != 0)) { + return Err(EINVAL); + } + entries.push(entry, GFP_KERNEL)?; + } + let programs = Self { entries }; + // A render kick has one SCS layout and one intersection-table binding + // per engine, even when several linked programs contribute demand. + programs.stage_scs()?; + Ok(programs) + } + + fn intersection(e: &uapi::drm_asahi_render_program_v2) -> Result { + IntersectionTables::new(e.intersection_pool, e.intersection_pool_bytes, + e.intersection_first, e.intersection_count, + e.intersection_default_program, e.intersection_default_control) + } + + /// Any linked program runs as a hardware mesh or object (task) stage. + pub(crate) fn uses_mesh(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags + & (uapi::DRM_ASAHI_RENDER_PROGRAM_MESH | uapi::DRM_ASAHI_RENDER_PROGRAM_OBJECT) != 0) + } + + /// An API task (object) stage is linked. Only such kicks get the TA mesh + /// rendering buffer: only the object→mesh TA program binds it, never the + /// mesh-single, grid or indirect ones. + pub(crate) fn uses_task(&self) -> bool { + self.entries.iter().any(|e| e.usage_flags & crate::file::DRM_ASAHI_RENDER_PROGRAM_TASK != 0) + } + + /// 0 ordinary, 1 hardware mesh (including the mesh-only passthrough object + /// stage), 2 API task (object) → mesh. + pub(crate) fn mesh_class(&self) -> u8 { + if self.uses_task() { 2 } + else if self.uses_mesh() { 1 } + else { 0 } + } + + pub(crate) fn stage_scs(&self) -> Result<[StageScratch; 2]> { + let mut stages = [StageScratch::NONE; 2]; + for e in &self.entries { + if e.scs_layout == 0 { continue; } + let next = StageScratch { scratch: ScratchRequest::new(e.scs_layout)?, + intersection: Self::intersection(e)?, auxiliary_allocation: e.auxiliary_allocation }; + let out = &mut stages[e.legacy.stage as usize]; + if out.scratch.enabled() && *out != next { return Err(EINVAL); } + *out = next; + } + Ok(stages) + } + + fn validate(entry: &uapi::drm_asahi_render_program) -> Result { + if entry.stage > 1 || entry.flags != 0 + || (entry.stage == 0 && (entry.auxiliary_count != 0 + || entry.threadgroup_bytes != 0 || entry.imageblock_bytes != 0 + || entry.multisample_bytes != 0)) { return Err(EINVAL); } + Ok(()) + } + + fn demands(&self, mut demand: RenderDemand) -> Result<(RenderDemand, [RenderDemand; 2])> { + let mut stages = [demand; 2]; + stages[0].include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + let empty = FragmentStorage::default().parameters().map_err(|_| EINVAL)?; + let no_tile = TileStorage::default().parameters(false).map_err(|_| EINVAL)?; + stages[1].include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + demand.include_vertex(VertexStorage::default()).map_err(|_| EINVAL)?; + demand.include_fragment(empty, no_tile).map_err(|_| EINVAL)?; + for e in &self.entries { + let entry = &e.legacy; + if entry.stage == 0 { + let storage = VertexStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + }; + if e.usage_flags & 6 != 0 { + let storage = ComputeStorage { + primary_bytes: storage.primary_bytes, secondary_bytes: storage.secondary_bytes, + linked_bytes: storage.linked_bytes, minimum_primary_bytes: storage.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, threads: e.threads, + }; + demand.include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + stages[0].include_mesh(storage, 96, 114688).map_err(|_| EINVAL)?; + } else { + demand.include_vertex(storage).map_err(|_| EINVAL)?; + stages[0].include_vertex(storage).map_err(|_| EINVAL)?; + } + } else { + let fragment = FragmentStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + auxiliary_count: entry.auxiliary_count, + }.parameters().map_err(|_| EINVAL)?; + let tile = TileStorage { + threadgroup_bytes: entry.threadgroup_bytes, + imageblock_bytes: entry.imageblock_bytes, + multisample_bytes: entry.multisample_bytes, + }.parameters(false).map_err(|_| EINVAL)?; + demand.include_fragment(fragment, tile).map_err(|_| EINVAL)?; + stages[1].include_fragment(fragment, tile).map_err(|_| EINVAL)?; + } + } + Ok((demand, stages)) + } +} + +/// Frozen device topology and shared-queue policy. No per-command allocation. +#[derive(Clone, Copy)] +pub(crate) struct M5Policy { + demand: RenderDemand, + budgets: [RenderBudget; 2], + compute_budgets: [RenderBudget; 2], +} + +#[derive(Clone, Copy)] +pub(crate) struct Plan { + pub(crate) sizes: Sizes, + pub(crate) geometry: PoolGeometry, + pub(crate) resource_flags: u64, + pub(crate) pressure: u32, + pub(crate) stages: Option<[StagePlan; 2]>, +} + +#[derive(Clone, Copy)] +pub(crate) struct StagePlan { + pub(crate) scs: StageScratch, + pub(crate) resource_flags: u64, + pub(crate) ptc_max: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct StageScratch { + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + pub(crate) auxiliary_allocation: u32, +} +impl StageScratch { + pub(crate) const NONE: Self = Self { scratch: ScratchRequest::NONE, + intersection: IntersectionTables::NONE, auxiliary_allocation: 0 }; +} + +impl M5Policy { + pub(crate) fn new(id: &crate::regs::G17Id) -> Result { + // Configured G17S context storage per enabled core. This is separate + // from the final per-kick PTC cap selected during command preparation. + const CONTEXT_FLOOR: u32 = 96; + let nominal = id.num_cores.checked_mul(id.num_clusters).ok_or(EOVERFLOW)?; + Ok(Self { + demand: RenderDemand::new(&id.core_masks, nominal, CONTEXT_FLOOR) + .map_err(|_| EINVAL)?, + compute_budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 2).map_err(|_| EINVAL)?, + ], + budgets: [ + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 3).map_err(|_| EINVAL)?, + RenderBudget::new(&id.core_masks, CONTEXT_FLOOR, 4).map_err(|_| EINVAL)?, + ], + }) + } + + pub(crate) fn plan_compute(&self, programs: &ComputePrograms, priority: u8) -> Result { + self.plan_compute_scs(programs, priority, 0) + } + + pub(crate) fn plan_compute_scs(&self, programs: &ComputePrograms, priority: u8, + scs_layout: u64) -> Result { + let scratch = super::fw::m5::compute::ScratchRequest::new(scs_layout)?; + // UMA pool class 1 is realtime; class 2 covers all other + // public priorities. Its resident-kick count also depends on engine. + let (budget, active_kicks) = match priority { + 0 => (&self.compute_budgets[0], 3), + 1..=3 => (&self.compute_budgets[1], 2), + _ => return Err(EINVAL), + }; + let mut demand = self.demand; + for entry in &programs.entries { + demand.include_compute(ComputeStorage { + primary_bytes: entry.primary_bytes, + secondary_bytes: entry.secondary_bytes, + linked_bytes: entry.linked_bytes, + minimum_primary_bytes: entry.minimum_primary_bytes, + threadgroup_bytes: entry.threadgroup_bytes, + threads: entry.threads, + }, 96, 114688).map_err(|_| EINVAL)?; + } + let request = budget.compute_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, scratch.pages()).map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()).map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: (demand.pressure() / active_kicks).clamp(96, 512), + stages: None, + }) + } + + pub(crate) fn plan(&self, programs: &Programs, priority: u8) -> Result { + let (budget, active_kicks) = match priority { 0 => (&self.budgets[0], 3), 1..=3 => (&self.budgets[1], 4), + _ => return Err(EINVAL) }; + let (demand, stage_demands) = programs.demands(self.demand)?; + let scs = programs.stage_scs()?; + let stage = |d: RenderDemand, scs| -> Result { Ok(StagePlan { + scs, + resource_flags: d.resource_flags().map_err(|_| EINVAL)?, + ptc_max: (d.pressure() / active_kicks).clamp(96, 512), + }) }; + let request = budget.render_scs_requirements( + demand.requirements().map_err(|_| EINVAL)?, + scs[0].scratch.pages().max(scs[1].scratch.pages()), + scs[0].auxiliary_allocation.max(scs[1].auxiliary_allocation)).map_err(|_| EINVAL)?; + let sizes = budget.sizes(request, History::default()) + .map_err(|_| EINVAL)?; + Ok(Plan { + geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, + resource_flags: demand.resource_flags().map_err(|_| EINVAL)?, + pressure: demand.pressure(), + stages: Some([stage(stage_demands[0], scs[0])?, stage(stage_demands[1], scs[1])?]), + }) + } +} + +// SAFETY: The record contains only fixed-width unsigned integers. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_compute_program {} + +pub(crate) struct ComputePrograms { + entries: KVec, +} +impl ComputePrograms { + pub(crate) fn parse(bytes: &[u8]) -> Result { + let stride = size_of::(); + if bytes.is_empty() || bytes.len() % stride != 0 { return Err(EINVAL); } + let mut entries = KVec::with_capacity(bytes.len() / stride, GFP_KERNEL)?; + let mut reader = Reader::new(bytes); + for _ in 0..bytes.len() / stride { + let entry: uapi::drm_asahi_compute_program = reader.read()?; + if entry.flags != 0 || entry.threads == 0 { return Err(EINVAL); } + entries.push(entry, GFP_KERNEL)?; + } + Ok(Self { entries }) + } +} diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 5641384b51bfee..78a34e3d7ab4db 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -407,12 +407,24 @@ enum drm_asahi_feature { */ DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, + /** + * @DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES. + */ + DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES = (1UL) << 25, + /** * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed * multisample stencil. */ DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, + /** + * @DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: M5: render commands accept + * DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS. + */ + DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS = (1UL) << 27, + /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, @@ -422,6 +434,12 @@ enum drm_asahi_feature { * include the earlier points of that timeline without adding GPU waits. */ DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, + + /** + * @DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS: Compute commands accept + * DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as an ordering hint. + */ + DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS = (1UL) << 30, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -1018,6 +1036,87 @@ struct drm_asahi_sync { */ #define DRM_ASAHI_BARRIER_NONE (0xFFFFu) +/** + * struct drm_asahi_compute_program - Storage of one linked compute dispatch. + * + * SET_COMPUTE_RESOURCES replaces the list for subsequent compute commands. + * Include each distinct dispatch profile in the command stream. The kernel + * combines their requirements before publishing work. All sizes are bytes. + */ +struct drm_asahi_compute_program { + /** @primary_bytes: Per-thread primary storage. */ + __u32 primary_bytes; + /** @secondary_bytes: Per-thread secondary storage. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage after linking. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across variants. */ + __u32 minimum_primary_bytes; + /** @threadgroup_bytes: Storage shared by a workgroup. */ + __u32 threadgroup_bytes; + /** @threads: Number of invocations in a workgroup. */ + __u32 threads; + /** @flags: Reserved; must be zero. */ + __u32 flags; +}; + +/** + * struct drm_asahi_render_program - Storage of one linked render program. + * + * SET_RENDER_RESOURCES replaces the list for subsequent renders in this submit. + * Include every vertex and fragment program, including background and end-of-tile + * programs. Sizes describe the final linked program, in bytes. Each entry is + * evaluated separately before taking the maximum demand across the pass. + */ +struct drm_asahi_render_program { + /** @stage: 0 for vertex, 1 for fragment (including background/end-of-tile). */ + __u32 stage; + /** @primary_bytes: Primary per-thread storage after linking. */ + __u32 primary_bytes; + /** @secondary_bytes: Secondary per-thread storage after linking. */ + __u32 secondary_bytes; + /** @linked_bytes: Additional primary storage from linked programs. */ + __u32 linked_bytes; + /** @minimum_primary_bytes: Minimum primary storage across linked variants. */ + __u32 minimum_primary_bytes; + /** @auxiliary_count: Fragment auxiliary entries; zero for vertex programs. */ + __u32 auxiliary_count; + /** @threadgroup_bytes: Fragment threadgroup storage; zero for vertex. */ + __u32 threadgroup_bytes; + /** @imageblock_bytes: Fragment imageblock storage; zero for vertex. */ + __u32 imageblock_bytes; + /** @multisample_bytes: Fragment multisample storage; zero for vertex. */ + __u32 multisample_bytes; + /** @flags: MBZ. */ + __u32 flags; +}; +/** Version 2 linked render storage. Version 1 remains exactly 40 bytes. */ +#define DRM_ASAHI_RENDER_PROGRAM_SCS (1U << 0) +#define DRM_ASAHI_RENDER_PROGRAM_MESH (1U << 1) +#define DRM_ASAHI_RENDER_PROGRAM_OBJECT (1U << 2) +/* With OBJECT: the object program is an API task stage. The kernel binds the + * TA mesh rendering buffer (the backing of the object payload address) to + * such kicks only. Requires DRM_ASAHI_FEATURE_TASK_MESH_STORAGE. */ +#define DRM_ASAHI_RENDER_PROGRAM_TASK (1U << 3) +struct drm_asahi_render_program_v2 { + struct drm_asahi_render_program legacy; + /** @threads: Mesh/object workgroup invocation count; zero otherwise. */ + __u32 threads; + /** @usage_flags: DRM_ASAHI_RENDER_PROGRAM_* bits. */ + __u32 usage_flags; + /** @scs_layout: Packed depth/levels/cliques/user/driver SCS layout. */ + __u64 scs_layout; + /** @auxiliary_allocation: The stage's auxiliary allocation word. */ + __u32 auxiliary_allocation; + __u32 reserved; + __u64 intersection_pool; + __u64 intersection_pool_bytes; + __u32 intersection_first; + __u32 intersection_count; + __u64 intersection_default_program; + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_header - Top level command structure * @@ -1252,6 +1351,37 @@ enum drm_asahi_render_flags { * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. */ DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), + + /** + * @DRM_ASAHI_RENDER_SHADER_WRITES: A graphics program in this command + * writes external memory, including buffer/image stores or atomic RMW. + * Ordinary raster outputs and compiler-private transport do not count. + * Requires DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES. Omission retains the + * existing encoding, allowing older userspace to submit unchanged. + */ + DRM_ASAHI_RENDER_SHADER_WRITES = (1U << 21), + + /** + * @DRM_ASAHI_RENDER_STAGE_PREFIXES: The tiling half waits only on the + * render and compute prefixes in @drm_asahi_cmd_render.ta_vdm_barrier, + * ta_prior_vdm, ta_cdm_barrier and ta_prior_cdm, each no later than the + * command's own prefix of that subqueue; the fragment half waits on the + * whole ones. Earlier renders in the tiling prefix are waited on to + * completion. The command must be the full struct drm_asahi_cmd_render. + * Not combined with DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. Requires + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES. + */ + DRM_ASAHI_RENDER_STAGE_PREFIXES = (1U << 23), + + /** + * @DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS: The render area covers the full + * extent of every attachment, so the render's tile stores write no pixel + * outside it. The kernel need not order such a render against another + * one that writes the same attachments without a dependency between + * them: any overlap is the application's own race. Requires + * DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS. + */ + DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS = (1U << 24), }; /** @@ -1632,6 +1762,46 @@ struct drm_asahi_cmd_render { * userspace omits it; a shorter command is read with both stages zeroed. */ struct drm_asahi_render_ray_query_stage ray_query[2]; + + /** + * @ta_cdm_barrier: Compute prefix of the tiling half, a command index like + * drm_asahi_cmd_header::cdm_barrier. + * + * This and the fields up to @ta_prior_vdm are read only with + * DRM_ASAHI_RENDER_STAGE_PREFIXES; without it they must be zero, and a + * shorter command reads them as zero. + */ + __u16 ta_cdm_barrier; + + /** @reserved0: MBZ. */ + __u16 reserved0; + + /** @reserved1: MBZ. */ + __u32 reserved1; + + /** + * @ta_prior_cdm: Exact compute prefix of the tiling half, an ordinal like + * @prior_cdm. + */ + __u64 ta_prior_cdm; + + /** + * @ta_vdm_barrier: Render prefix of the tiling half, a command index like + * drm_asahi_cmd_header::vdm_barrier. + */ + __u16 ta_vdm_barrier; + + /** @reserved2: MBZ. */ + __u16 reserved2; + + /** @reserved3: MBZ. */ + __u32 reserved3; + + /** + * @ta_prior_vdm: Exact render prefix of the tiling half, an ordinal like + * @prior_vdm. + */ + __u64 ta_prior_vdm; }; /** @@ -1738,6 +1908,20 @@ enum drm_asahi_compute_flags { * ordered helpers cannot time out while waiting for the build. */ DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), + + /** + * @DRM_ASAHI_COMPUTE_USES_MATRIX: The command's dispatches use the + * hardware's matrix instructions. Requires DRM_ASAHI_FEATURE_COMPUTE_MATRIX. + */ + DRM_ASAHI_COMPUTE_USES_MATRIX = (1L << 16), + + /** + * @DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS: The command neither copies nor + * resets queries, so it cannot observe timestamps the kernel publishes + * for earlier commands and need not wait for that publication. Requires + * DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS. + */ + DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS = (1L << 17), }; /** From d336efb49929db973c79d228060afe9e58e8a9f4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:31 -0800 Subject: [PATCH 067/171] drm/asahi: g17: M5: submit compute on M5 queues Encode the M5 compute program and descriptor, install M5 compute queues cold and refresh their priority on reuse, and publish compute as M5 work on the shared ring. Timestamps come from the queue's own record, waits on other queues stay in firmware, and the classic compute paths are never taken on M5. A closed context's hung M5 compute is killed through its queue's exit, and the kill settles once the full-queue retirement witness covers the killed commands. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..e393a52c7151 (7 commits); SPTM/APIF removed; adapted to integration's compute queue (cached graph, descriptor ring), completions and runtime; interim forms until the render submission commit; comments and names reworded; M5 paths restored where the first rewrite lost them: the firmware's own compute queue record, the shared compute aperture and global state left to G17P, every compute doorbell in the submission history, the M5 compute pool format, the kill of a closed context's hung compute settled by the full-queue witness, the full CPU refill floor on M5 with the feed's guard against unexecuted intervals and only exact compute execution records feeding it, and the item ring's count and cursor checks before its room is computed; M5 compute capacity threaded through integration's admission (rebind, readiness, growth, rearm); original 6613f5ab7b7e, e25ba8a6fb2b, f49e91b3f364, 7c95830ea3a1, 77dc6eb34210, ee68abb10520, e393a52c7151 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/queue/mod.rs] (cherry picked from commit 7c27d3ea195081ec567fa285c0b90c198ad968bd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer.rs | 2 + drivers/gpu/drm/asahi/g17/buffer/geometry.rs | 79 ++++ drivers/gpu/drm/asahi/g17/completion.rs | 98 +++- drivers/gpu/drm/asahi/g17/context.rs | 66 ++- drivers/gpu/drm/asahi/g17/feed.rs | 33 +- drivers/gpu/drm/asahi/g17/freelist.rs | 80 ++-- drivers/gpu/drm/asahi/g17/fw/kick.rs | 41 +- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 302 +++++++++++++ drivers/gpu/drm/asahi/g17/initdata.rs | 79 +++- drivers/gpu/drm/asahi/g17/job.rs | 10 +- drivers/gpu/drm/asahi/g17/mod.rs | 9 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 418 ++++++++++++++---- .../drm/asahi/g17/queue/compute/activation.rs | 77 ++++ .../asahi/g17/queue/compute/configuration.rs | 40 ++ .../gpu/drm/asahi/g17/queue/compute/exit.rs | 35 +- .../drm/asahi/g17/queue/compute/retirement.rs | 12 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 124 ++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 2 + drivers/gpu/drm/asahi/g17/runtime.rs | 136 ++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 34 +- .../drm/asahi/g17/runtime/render/prepare.rs | 6 +- 21 files changed, 1411 insertions(+), 272 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/buffer/geometry.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/activation.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index fd6460db2d1fbc..3c4e4bf73a394c 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -9,6 +9,8 @@ use super::fw::buffer as fw; use crate::hw::t8140::resources as cfg; pub(crate) mod target; +mod geometry; +pub(crate) use geometry::BlockGeometry; use kernel::prelude::*; diff --git a/drivers/gpu/drm/asahi/g17/buffer/geometry.rs b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs new file mode 100644 index 00000000000000..c871f470c1a153 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/geometry.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer backing geometry. Wire records carry each block's address, +//! so allocation stride and the reserved trailing guard are host policy. + +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BlockGeometry { + pub(crate) base: u64, + pub(crate) end: u64, + pub(crate) block_bytes: usize, + pub(crate) backing_stride: usize, + pub(crate) stride: usize, + pub(crate) capacity: usize, +} + +impl BlockGeometry { + pub(crate) const PAGE: usize = 0x8000; + /// M5 parameter resources use 32-bit offsets from this compact base. + pub(crate) const M5_COMPACT: core::ops::Range = 0x10_0000_0000..0x11_0000_0000; + pub(crate) const NEO: Self = Self { + base: cfg::USER_BASE, + end: cfg::USER_END, + block_bytes: cfg::TVB_BLOCK_SIZE, + backing_stride: cfg::TVB_BLOCK_STRIDE as usize, + stride: cfg::TVB_BLOCK_STRIDE as usize, + capacity: cfg::TVB_BLOCK_SLOTS, + }; + + pub(crate) fn m5(memory_bytes: u64, initial_bytes: u64) -> Result { + let capacity = crate::g17::fw::m5::pool::parameter_capacity( + memory_bytes, initial_bytes, + ).map_err(|_| EINVAL)?; + // Retain one unallocated inventory slot, as in the shared ring manager. + if capacity < 2 || capacity.checked_mul(16).is_none_or(|n| n > 0x3f_ffff) { + return Err(EINVAL); + } + Ok(Self { + base: Self::M5_COMPACT.start, + end: Self::M5_COMPACT.end, + block_bytes: 0x80000, + backing_stride: 0x80000, + stride: 0x88000, + capacity, + }) + } + + pub(crate) const fn pages(self) -> usize { self.block_bytes / Self::PAGE } + pub(crate) const fn max_blocks(self) -> usize { self.capacity - 1 } + + pub(crate) fn validate(self) -> Result { + if self.capacity < 2 || self.capacity > u32::MAX as usize + || self.base >= self.end || self.base % Self::PAGE as u64 != 0 + || self.block_bytes == 0 || self.block_bytes % Self::PAGE != 0 + || self.backing_stride < self.block_bytes || self.backing_stride % Self::PAGE != 0 + || self.stride <= self.block_bytes || self.stride % Self::PAGE != 0 { + return Err(EINVAL); + } + Ok(()) + } + + pub(crate) fn backing_bytes(self, blocks: usize) -> Result { + self.validate()?; + if blocks == 0 || blocks > self.max_blocks() { return Err(EINVAL); } + (blocks - 1).checked_mul(self.backing_stride) + .and_then(|n| n.checked_add(self.block_bytes)).ok_or(EOVERFLOW) + } + + pub(crate) fn compact_page(self, address: u64) -> Result { + let relative = address.checked_sub(self.base).ok_or(EINVAL)?; + if relative % Self::PAGE as u64 != 0 + || address.checked_add(self.block_bytes as u64).ok_or(EOVERFLOW)? > self.end { + return Err(EINVAL); + } + (relative / Self::PAGE as u64).try_into().map_err(|_| EOVERFLOW) + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4d3d5e5a7904d2..15678f5c7c92f3 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -246,6 +246,31 @@ impl Drop for PooledTimestamps { /// Republications of one command whose earlier publications firmware dropped unexecuted. pub(crate) const MAX_REPLAYS: u8 = 3; +/// M5 timestamp pairs share a retained page of the physical compute queue. +/// A completion keeps its backing alive through quarantine and replay. +struct M5ComputeTimestamps { + object: Arc, + offset: usize, +} + +impl M5ComputeTimestamps { + fn clear(&self) -> Result { + let start = self.object.dword(self.offset)?; + let end = self.object.dword(self.offset + 8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + fn words(&self) -> Result<[u64; 2]> { + let words = [self.object.dword(self.offset)?.load(Ordering::Relaxed), + self.object.dword(self.offset + 8)?.load(Ordering::Relaxed)]; + barrier(Ordering::Acquire); + Ok(words) + } +} + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -254,6 +279,11 @@ pub(crate) struct Completion { // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, compute: Option, + #[pin] + m5_compute: Mutex>, + /// A compute command. M5 compute binds its timestamps at publication, so + /// `compute` alone does not identify the engine. + is_compute: bool, feed: Option>, usage: Arc, sources: Option>, @@ -312,7 +342,9 @@ impl Completion { } else { HostOutput::Immediate(output) }; - let compute = if compute { + // M5 compute binds its timestamp pair in its physical queue's page at publication. + let is_compute = compute; + let compute = if compute && !context.is_m5() { Some(context.compute_timestamps().take(dev, context.vm())?) } else { None @@ -323,6 +355,8 @@ impl Completion { output, terminal, compute, + m5_compute <- new_mutex!(None, "M5 queue timestamps"), + is_compute, feed, usage, sources, @@ -421,14 +455,33 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) + if let Some(compute) = &self.compute { return Ok(compute.page()?.mapping.iova()); } + let compute = self.m5_compute.lock(); + let compute = compute.as_ref().ok_or(EINVAL)?; + Ok(compute.object.gpu_va() + compute.offset as u64) + } + + /// The queue checked its record-slot retirement before calling this. + /// Clearing and binding precede descriptor publication under the device lock. + pub(crate) fn bind_m5_compute_timestamps(&self, object: &Arc, offset: usize) -> Result { + if self.compute.is_some() || self.signalled.load(Ordering::Acquire) { + return Err(EINVAL); + } + let timestamps = M5ComputeTimestamps { object: object.clone(), offset }; + timestamps.clear()?; + let address = timestamps.object.gpu_va() + offset as u64; + *self.m5_compute.lock() = Some(timestamps); + Ok(address) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.words(); + }; + let object = &compute.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -444,6 +497,10 @@ impl Completion { self.compute_words().ok() } + pub(crate) fn signalled(&self) -> bool { + self.signalled.load(Ordering::Acquire) + } + pub(crate) fn begin_replay(&self) -> bool { let accepted = self .replays @@ -481,7 +538,10 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let Some(compute) = self.compute.as_ref() else { + return self.m5_compute.lock().as_ref().ok_or(EINVAL)?.clear(); + }; + let object = &compute.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -648,7 +708,21 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + // M5 compute feeds the refill detector only from its exact execution record + // (RetiredCompute); M5 compute is the compute without pooled timestamps. + let execution = if self.is_compute && self.compute.is_none() { + None + } else { + result.as_ref().ok().map(|ticks| [ticks[0], ticks[1]]) + }; + self.complete_with_compute(result, execution); + } + + /// Compute feedback is separate from public timestamps, which may contain + /// a synthesized retirement envelope when the execution record is absent. + #[track_caller] + fn complete_with_compute(&self, mut result: Result<[u64; 4]>, compute: Option<[u64; 2]>) { if result.is_ok() { if let Err(error) = self.check_source_inputs() { result = Err(error); @@ -663,7 +737,7 @@ impl Completion { if let Ok(ticks) = &result { use crate::usage::Engine; let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; - if self.compute.is_some() { + if self.is_compute { self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); } else { self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); @@ -673,8 +747,10 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. - if self.compute.is_some() { - feed.note_pass(ticks[0], ticks[1]); + if self.is_compute { + if let Some([start, end]) = compute { + feed.note_pass(start, end); + } } else { feed.note_pass(ticks[2], ticks[3]); } @@ -737,6 +813,8 @@ fn discarded(error: Error) -> bool { /// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. pub(crate) enum Deferred { Retired(Arc, Result<[u64; 4]>), + /// Only the optional interval comes from an exact compute execution record. + RetiredCompute(Arc, [u64; 4], Option<[u64; 2]>), Drained(Arc, Result<[u64; 4]>), FailedOwned(Arc, Error), Unpublished(Arc, Error), @@ -746,6 +824,10 @@ pub(crate) enum Deferred { impl Deferred { pub(crate) fn finish(self) { match self { + Self::RetiredCompute(completion, ticks, execution) => { + completion.complete_with_compute(Ok(ticks), execution); + completion.release_after_retirement(); + } Self::Retired(completion, result) => { completion.complete(result); // An earlier failure may already have signalled this fence while diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index b232eee184e5ce..ef7934585e4983 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -148,6 +148,8 @@ pub(crate) struct FirmwarePages { } pub(crate) struct Context { + /// The GPU runs the M5 firmware's protocol. + m5: bool, execution: Arc, status: Arc, scheduler: Arc, @@ -257,6 +259,7 @@ impl Context { let order = Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { + m5: alloc.uat.is_m5(), execution, status, scheduler, @@ -301,6 +304,18 @@ impl Context { pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } + /// M5: this VM's client view of the shared metrics backing, mapped once + /// per VM in the compact range. + pub(crate) fn metrics_client(&self, backing: &Arc) -> Result { + self.vm().metrics_alias(backing.gpu_va(), || { + backing.alias_in( + self.vm(), + super::buffer::BlockGeometry::M5_COMPACT, + UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + ) + }) + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -314,6 +329,24 @@ impl Context { pub(crate) fn is_current(&self) -> bool { self.execution.is_current() } + /// Claim the context's hardware identity for one published M5 command until it retires. + pub(crate) fn pin(&self) -> Result { + self.execution.pin() + } + pub(crate) fn unpin(&self, pin: mmu::ContextPin) { + self.execution.unpin(pin) + } + /// Claim the queue's QoS slot for one published M5 command. Stage 1: the slot is the + /// context's own, taken at creation. + pub(crate) fn pin_qos(&self) -> Result { + Ok(self.qos_id()) + } + pub(crate) fn unpin_qos(&self, _slot: u8) {} + + pub(crate) fn is_m5(&self) -> bool { + self.m5 + } + pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } @@ -425,21 +458,44 @@ impl Context { alloc: &Allocator<'_>, ids: &Arc, render_global: &Vm, + plan: Option, ) -> Result> { if !self.is_current() { return Err(EFAULT); } + if alloc.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } let mut render = self.render.lock(); if let Some(pool) = &*render { - // A pool vacated under memory pressure is re-armed before reuse. - pool.ensure_backing(alloc, render_global, self.vm())?; - return Ok(pool.clone()); + if pool.fits(plan.as_ref().map(|p| p.geometry)) { + // A pool vacated under memory pressure is re-armed before reuse. + // M5 backing is installed by the pair that pins the pool. + if !alloc.uat.is_m5() { + pool.ensure_backing(alloc, render_global, self.vm())?; + } + return Ok(pool.clone()); + } + // M5: existing pairs retain the old pool through their rebind and + // its release acknowledgement. Allocation failure leaves it installed. } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, + match plan { + Some(plan) => RenderPool::m5( + alloc, + super::freelist::RenderId::logical(ids)?, + self.vm(), + plan.geometry, + )?, + None => { + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())? + } + }, GFP_KERNEL, )?; - *render = Some(pool.clone()); + let previous = render.replace(pool.clone()); + drop(render); + drop(previous); Ok(pool) } /// Drops the pool once its free-list release is consumed and the closed diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index bd1228a82e26f2..a96f67ae6b8f9a 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -9,6 +9,9 @@ //! Policy creation and the initial snapshot take the policy rwsem before the //! requests mutex. Removal callbacks and the worker take only requests; //! notifier unregistration joins callbacks without holding that mutex. +//! M5 requests full CPU capacity only while refill pressure keeps the detector +//! active. Sustained demand can renew its deadline indefinitely; withdrawal +//! occurs on a later worker poll, not at an exact eight-millisecond boundary. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ @@ -27,6 +30,7 @@ use core::ptr; use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const _: () = assert!(CLOCK_HZ == crate::hw::t8140::resources::COMMAND_TIMESTAMP_HZ); const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; @@ -76,7 +80,7 @@ impl Detector { } fn note(&self, start: u64, end: u64, now: u64) { - if end <= start { + if start == 0 || end <= start { return; } let previous = self.last_end.fetch_max(end, Ordering::AcqRel); @@ -141,6 +145,8 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], + /// Full CPU capacity for M5 refill pressure; other profiles retain their floor. + floor_percent: u64, } // SAFETY: Each nonnull policy owns a kobject reference. The device requests @@ -150,12 +156,21 @@ unsafe impl Send for Requests {} #[cfg(CONFIG_CPU_FREQ)] impl Requests { - fn new() -> Self { + fn new(m5: bool) -> Self { Self { requests: Opaque::zeroed(), policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], + floor_percent: if m5 { 100 } else { FLOOR_PERCENT }, + } + } + + fn target_floor(maximum: u32, floor_percent: u64, enabled: bool) -> u32 { + if enabled { + (u64::from(maximum) * floor_percent / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 } } @@ -206,11 +221,7 @@ impl Requests { if this.policies[index].is_null() { continue; } - let value = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE - }; + let value = Self::target_floor(this.maxima[index], *this.floor_percent, enabled); if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; // the policy reference keeps its constraints alive through REMOVE. @@ -296,12 +307,14 @@ pub(crate) struct Feed { } impl Feed { - pub(crate) fn new() -> Result> { + pub(crate) fn new(m5: bool) -> Result> { + #[cfg(not(CONFIG_CPU_FREQ))] + let _ = m5; let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), #[cfg(CONFIG_CPU_FREQ)] - requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + requests <- new_mutex!(Requests::new(m5), "G17 CPU feed"), #[cfg(CONFIG_CPU_FREQ)] notifier <- pin_init!(PolicyNotifier { block: Opaque::new(bindings::notifier_block { @@ -422,6 +435,8 @@ impl Feed { Monotonic::ktime_get() as u64 } + /// Firmware execution intervals in the device's 24 MHz timestamp domain. + /// Missing, unexecuted and wrapped intervals must not affect the detector. pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 49d00180363ac5..32078e0daadbef 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -249,10 +249,20 @@ impl Drop for RenderId { } } +/// What one command needs of its compute queue's USC pool, in the family's +/// terms: G17P pools count blocks, M5 pools have the command plan's geometry +/// (`None`: the default M5 pool). Derived once per command +/// (`Backend::compute_capacity`); every admission check and allocation takes it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ComputeCapacity { + Blocks(usize), + M5(Option), +} + /// Profile-specific descriptor data for a fresh physical pool. The runtime /// installs this only before publication, or after an acknowledged release. pub(crate) enum PoolDescriptor { - Neo { slot: u16, page_list: u64 }, + Neo { slot: u16, page_list: u64, blocks: usize }, M5 { slot: u16, words: [u64; 8] }, } @@ -346,17 +356,23 @@ impl Pages { layout: PageLayout::Neo { blocks, runs }, }) } - /// Fresh compute backing at the admitted capacity, built off-lock. - pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + /// Fresh compute backing at the admitted capacity, built off-lock, in the + /// capacity's family format. + pub(crate) fn for_compute(alloc: &Allocator<'_>, capacity: ComputeCapacity) -> Result { + match capacity { + ComputeCapacity::Blocks(blocks) => Self::compute_sized(alloc, blocks), + ComputeCapacity::M5(geometry) => Self::compute_m5(alloc, geometry), + } + } + fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } Self::new(alloc, blocks, RunLayout::Compute) } - /// Fresh zeroed compute backing, built without the device mutex. M5 keeps + /// Fresh zeroed M5 compute backing, built without the device mutex. M5 keeps /// its directory, block list and initial 32 MiB in one retained allocation. - pub(crate) fn compute(alloc: &Allocator<'_>, m5: bool, required: Option) -> Result { - if !m5 { return Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute); } + fn compute_m5(alloc: &Allocator<'_>, required: Option) -> Result { use super::fw::m5::pool; let geometry = match required { Some(geometry) => geometry, None => pool::PoolGeometry::g17s(8).map_err(|_| EINVAL)?, }; @@ -406,10 +422,13 @@ impl Pages { _ => false, } } - /// Block capacity of block-sized (non-M5) backing; M5 backing is sized by - /// its geometry and never matches. - fn fits_blocks(&self, required: usize) -> bool { - matches!(self.layout, PageLayout::Neo { blocks, .. } if blocks >= required) + /// Whether this backing holds `required`; backing of the other family never does. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + match (self.layout, required) { + (PageLayout::Neo { blocks, .. }, ComputeCapacity::Blocks(required)) => blocks >= required, + (PageLayout::M5(_), ComputeCapacity::M5(geometry)) => self.fits(geometry), + _ => false, + } } fn is_m5(&self) -> bool { matches!(self.layout, PageLayout::M5(_)) } pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } @@ -424,12 +443,17 @@ impl Pages { Ok(super::fw::m5::pool::PoolLayout { geometry, directory: self.page_list_va(), freelist: self.run_list_va(), hardware_id: u32::from(slot) }) } - fn descriptor(&self, slot: u16) -> Result { - if self.is_m5() { - let words = super::fw::m5::pool::initial_uma_descriptor(self.m5_layout(slot)?) - .map_err(|_| EINVAL)?; - Ok(PoolDescriptor::M5 { slot, words }) - } else { Ok(PoolDescriptor::Neo { slot, page_list: self.page_list_va() }) } + /// The descriptor row of this backing installed at `slot`. + pub(crate) fn descriptor(&self, slot: u16) -> Result { + match self.layout { + PageLayout::M5(_) => { + let words = super::fw::m5::pool::initial_uma_descriptor(self.m5_layout(slot)?) + .map_err(|_| EINVAL)?; + Ok(PoolDescriptor::M5 { slot, words }) + } + PageLayout::Neo { blocks, .. } => + Ok(PoolDescriptor::Neo { slot, page_list: self.page_list_va(), blocks }), + } } fn repopulate(&self) -> Result { let PageLayout::Neo { blocks, runs } = self.layout else { return Err(EINVAL); }; @@ -962,26 +986,20 @@ pub(crate) struct ComputePool { m5: bool, } impl ComputePool { - /// Block-sized (non-M5) capacity check for admitted growth. - pub(crate) fn fits_blocks(&self, blocks: usize) -> bool { - self.pages.as_ref().is_some_and(|pages| pages.fits_blocks(blocks)) + /// Capacity check for admission and growth. + pub(crate) fn fits_capacity(&self, required: ComputeCapacity) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.fits_capacity(required)) } - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, m5: bool, required: Option) -> Result { + /// A pool at the admitted capacity, in the capacity's family format. + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, required: ComputeCapacity) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } - let pages = Pages::compute(alloc, m5, required)?; + let pages = Pages::for_compute(alloc, required)?; + let m5 = pages.is_m5(); Self::with_pages(alloc, qid, m5, pages) } - /// A block-sized (non-M5) pool at the admitted capacity. - pub(crate) fn sized(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { - if qid > super::fw::kick::QID_MAX { - return Err(EINVAL); - } - let pages = Pages::compute_sized(alloc, blocks)?; - Self::with_pages(alloc, qid, false, pages) - } fn with_pages(alloc: &Allocator<'_>, qid: u8, m5: bool, pages: Pages) -> Result { let id = u16::from(qid) + 1; let mut control = alloc.kernel( @@ -1041,10 +1059,6 @@ impl ComputePool { pub(crate) fn id(&self) -> u16 { self.id } - /// EAGAIN while the pool is vacant: the owner must re-arm it first. - pub(crate) fn page_list_va(&self) -> Result { - self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) - } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2a035524bdffbd..cd5e53683724da 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -371,6 +371,9 @@ pub(crate) struct KickArgs<'a> { pub(crate) parent: KickTimestamp, /// Kicks of other queues this kick must wait for. pub(crate) barriers: &'a [KickDependency], + /// Two-bit completion classes for each explicit dependency followed by + /// the implicit parent. Interfaces without typed dependencies use `None`. + pub(crate) dependency_types: Option<&'a [u8]>, /// GPU address of the work descriptor (firmware alias). pub(crate) descriptor_va: u64, /// GPU address of the queue record. @@ -383,12 +386,23 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], - /// The compute register program includes a shader-context scratch request. + /// The stage's register program includes a shader-context scratch (SCS) + /// bracket (header bit 61; compute, TA and 3D alike). pub(crate) scs_enabled: bool, + /// The stage's register program includes a matrix bracket (bit 63). + pub(crate) compute_matrix: bool, /// Priority class of the queue. pub(crate) priority: u8, } +/// Per-stage (TA, 3D) kick header flags of a render, derived from the same +/// state that emits the SCS and matrix register brackets. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct StageKickFlags { + pub(crate) scratch: [bool; 2], + pub(crate) matrix: [bool; 2], +} + /// One kick ring entry. /// /// Live state: none. The host writes the whole entry before announcing it and @@ -397,7 +411,8 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. + /// 58, queue priority in bits 60:59, SCS bracket in the stage's register + /// program in bit 61 and matrix bracket in bit 63. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -418,7 +433,10 @@ pub(crate) struct KickEntry { pub(crate) register_arrays: [u64; 4], pub(crate) unk_170: u64, pub(crate) unk_178: u64, - pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], + /// Two bits per valid dependency, in the same order as `dependencies`. + /// Zero on interfaces without typed dependencies. + pub(crate) dependency_types: u64, + pub(crate) pad_188: [u8; KICK_ENTRY_STRIDE - 0x188], } static_assert!(size_of::() == KICK_ENTRY_STRIDE); @@ -426,6 +444,7 @@ static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(core::mem::offset_of!(KickEntry, dependency_types) == 0x180); static_assert!(size_of::() == 0xc); static_assert!(size_of::() == 0x10); static_assert!(size_of::() == 0x720); @@ -438,6 +457,7 @@ impl KickEntry { const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; const SCS_ENABLED: u64 = 1 << 61; + const COMPUTE_MATRIX: u64 = 1 << 63; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -463,6 +483,9 @@ impl KickEntry { if args.scs_enabled { header |= Self::SCS_ENABLED; } + if args.compute_matrix { + header |= Self::COMPUTE_MATRIX; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; @@ -481,6 +504,15 @@ impl KickEntry { } .word(); let count = barriers + 1; + let mut dependency_types = 0; + if let Some(types) = args.dependency_types { + if types.len() != count || types.iter().any(|kind| *kind > 3) { + return Err(EINVAL); + } + for (index, kind) in types.iter().enumerate() { + dependency_types |= u64::from(*kind) << (2 * index); + } + } let mask = if count == KICK_DEPENDENCIES { u32::MAX } else { @@ -502,7 +534,8 @@ impl KickEntry { register_arrays: args.register_arrays.map(RegisterArrayBinding::word), unk_170: 0, unk_178: Self::UNK_178, - pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + dependency_types, + pad_188: [0; KICK_ENTRY_STRIDE - 0x188], }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index 73d23fbd563943..b2e439398d826f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -5,6 +5,12 @@ //! register program. M5 scratch has its own layout and two control writes. use kernel::prelude::*; +use super::super::{kick::RegisterWrite, offset_va, work_key}; + +pub(crate) const MAIN_COUNT: usize = 40; +pub(crate) const ALTERNATE_COUNT: usize = 4; +pub(crate) const PREEMPT_BYTES: usize = 0x4000; +pub(crate) const OPERAND_BYTES: usize = 0x40000; pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; /// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent @@ -64,6 +70,9 @@ impl ScratchRequest { CHECKPOINTS.map(|count| count + if self.enabled() { 2 } else { 0 }) } } + +/// Per-command intersection table bindings. The pool contains 128-byte +/// blocks of eight 16-byte program entries. Addresses are UAT user addresses; /// ownership and readable ranges are checked by command validation. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct IntersectionTables { @@ -116,3 +125,296 @@ impl IntersectionTables { self.pool_bytes >> 14, self.default_program, self.default_control] } } + +/// Addresses and policy owned by a queued compute command. +pub(crate) struct RegisterArgs { + pub(crate) matrix_ops: bool, + pub(crate) scratch: ScratchRequest, + pub(crate) intersection: IntersectionTables, + /// Userspace's opaque auxiliary allocation word, independent of the SCS layout. + pub(crate) auxiliary_allocation: u32, + pub(crate) preempt_va: u64, + pub(crate) cdm_va: u64, + pub(crate) operand_va: u64, + pub(crate) usage_va: u64, + /// Private-memory pool slot, as named by the usage record. + pub(crate) free_list_slot: u32, + pub(crate) context_id: u16, + pub(crate) work_state: u32, + /// Packed predecessor/current object identifiers. + pub(crate) object_ids: u64, + /// Pool target and resource policy for the program's first resource-policy write. + pub(crate) resource_policy: u64, + /// Start of the four eight-byte state words in the preemption buffer. + pub(crate) preempt_state_offset: u32, + /// Size of one operand-state region; the program exchanges two regions. + pub(crate) operand_region_bytes: u32, + pub(crate) operand_aux_offset: u32, + pub(crate) resource_control: u64, + pub(crate) dispatch_control: u64, + pub(crate) auxiliary_control: u64, +} + +impl RegisterArgs { + fn validate(&self) -> Result { + if self.preempt_va & 0x3fff != 0 || self.operand_va & 0x3fff != 0 + || self.cdm_va & 7 != 0 || self.usage_va & 7 != 0 || self.context_id >= 64 { + return Err(EINVAL); + } + offset_va(self.preempt_va, PREEMPT_BYTES as u64)?; + offset_va(self.operand_va, OPERAND_BYTES as u64)?; + if self.preempt_state_offset & 7 != 0 + || u64::from(self.preempt_state_offset) + 32 > PREEMPT_BYTES as u64 + || self.operand_region_bytes == 0 || self.operand_region_bytes & 0xfff != 0 + || u64::from(self.operand_region_bytes) * 2 + 0x808 > OPERAND_BYTES as u64 + || self.operand_aux_offset & 7 != 0 + || u64::from(self.operand_aux_offset) + 8 > OPERAND_BYTES as u64 { + return Err(EINVAL); + } + Ok(()) + } +} + +/// Construct the complete primary program before publishing its descriptor. +pub(crate) fn main(args: &RegisterArgs) -> Result<[RegisterWrite; MAIN_COUNT]> { + args.validate()?; + let preempt = args.preempt_va; + let operand = args.operand_va; + let ids = args.object_ids; + let intersection = args.intersection.words(); + let state = u64::from(args.preempt_state_offset); + let region = u64::from(args.operand_region_bytes); + let key = work_key(args.context_id, args.work_state); + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, state)?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, state + 8)?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, state + 16)?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, state + 24)?), + RegisterWrite::new(0x1a440, 0x154024201), + RegisterWrite::new(0x0d4a1, args.resource_control), + RegisterWrite::new(0x0d411, 0x200000001), + RegisterWrite::new(0x0a599, args.resource_policy), + RegisterWrite::new(0x1a458, args.dispatch_control), + RegisterWrite::new(0x101d9, 0x1c), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), + RegisterWrite::new(0x101d1, args.scratch.size_word()), + RegisterWrite::new(0x0d479, u64::from(args.auxiliary_allocation)), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), + RegisterWrite::new(0x107a1, args.scratch.window_word() | (u64::from(args.matrix_ops) << 25)), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x0a731, args.auxiliary_control), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot.into()), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, region * 2 + 0x800)?), + RegisterWrite::new(0x140a8, offset_va(operand, args.operand_aux_offset.into())?), + RegisterWrite::new(0x10099, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x10091, offset_va(operand, region * 2 + 0x400)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, 5)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region)?), + RegisterWrite::new(0x0a731, 0), + RegisterWrite::new(0x1a440, 0x154024209), + RegisterWrite::new(0x0a599, 0x6000400020), +]) +} + +fn program(args: &RegisterArgs) -> Result<([RegisterWrite; MAIN_COUNT + 4], usize)> { + let main = main(args)?; + let mut writes = [RegisterWrite::new(0, 0); MAIN_COUNT + 4]; + let mut count = 0; + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17e1, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17f1, 1); + count += 1; + } + for (index, register) in main.iter().enumerate() { + if index == 33 { + if args.scratch.enabled() { + writes[count] = RegisterWrite::new(0x17d9, 1); + count += 1; + } + if args.matrix_ops { + writes[count] = RegisterWrite::new(0x17e9, 1); + count += 1; + } + } + writes[count] = *register; + count += 1; + } + Ok((writes, count)) +} + +/// Operand-state exchange used by the alternate program. +pub(crate) fn alternate(args: &RegisterArgs) -> Result<[RegisterWrite; ALTERNATE_COUNT]> { + args.validate()?; + let operand = args.operand_va; + let region = u64::from(args.operand_region_bytes); + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, 5)?), + RegisterWrite::new(0x10091, offset_va(operand, region)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, region + 0x405)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, region * 2 + 0x400)?), + ]) +} + +/// Register-array bindings in a compute kick. The first checkpoint follows +/// the initial operand-region write, before the second auxiliary-control write. +pub(crate) fn bindings(descriptor_va: u64, scratch: ScratchRequest, matrix_ops: bool) + -> Result<[Option; 4]> +{ + use super::super::kick::RegisterArrayBinding; + Ok([ + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x40)?, (scratch.checkpoints()[0] + if matrix_ops { 2 } else { 0 }) as u8)?), + Some(RegisterArrayBinding::new(offset_va(descriptor_va, 0x760)?, ALTERNATE_COUNT as u8)?), + None, + None, + ]) +} + +/// The fixed prefix of a compute descriptor. Storage beyond this prefix is +/// owned and initialized by the runtime allocation, not described here. +pub(crate) const DESCRIPTOR_PREFIX_BYTES: usize = 0x1000; + +/// Per-command fields outside the two register programs. Fields whose bit +/// meanings are not established remain explicit opaque values. +pub(crate) struct DescriptorArgs { + pub(crate) registers: RegisterArgs, + pub(crate) descriptor_va: u64, + pub(crate) submit_sequence: u64, + pub(crate) work_state_va: u64, + pub(crate) cdm_end_va: u64, + pub(crate) stamp_va: [u64; 2], + pub(crate) stamp_value: u32, + pub(crate) qid: u32, + pub(crate) kick_timestamp: u64, + pub(crate) completion_ordinal: u32, + pub(crate) status_va: [u64; 2], + pub(crate) timestamp_va: [u64; 2], + pub(crate) shared_control_va: u64, + pub(crate) event_metadata: [u8; 16], + pub(crate) usage_fw_va: u64, + pub(crate) completion_identity: u64, +} + +/// The M5 descriptor has a shorter reserved region before shared-control +/// state. Register-array offsets are shared with G17, tail offsets are not. +#[repr(C, packed)] +pub(crate) struct Descriptor { + tag: u32, + submit_sequence: u64, + context_id: u32, + work_state_va: u64, + checkpoints: [u16; 4], + reserved_20: [u8; 0x20], + registers: super::super::kick::RegisterArray, + alternate: super::super::kick::RegisterArray, + reserved_e80: [u8; 0x58], + preempt_va: u64, + cdm_end_va: u64, + reserved_ee8: [u8; 0x20], + control: u64, + reserved_f10: [u8; 0x10], + predecessor_id: u32, + reserved_f24: u32, + unk_f28: u32, + firmware_timestamps: [u64; 2], + reserved_f3c: u32, + stamp_va: [u64; 2], + stamp_value: u32, + qid: u32, + reserved_f58: u64, + kick_timestamp: u64, + current_id: u64, + completion_ordinal: u32, + reserved_f74: [u8; 8], + status_va: [u64; 2], + timestamp_va: [u64; 2], + reserved_f9c: [u8; 0x10], + shared_control_va: u64, + shared_control_enabled: u8, + event_metadata: [u8; 16], + usage_fw_va: u64, + completion_identity: u64, + reserved_fd5: [u8; 0x2b], +} + +static_assert!(size_of::() == DESCRIPTOR_PREFIX_BYTES); +static_assert!(core::mem::offset_of!(Descriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(Descriptor, alternate) == 0x760); +static_assert!(core::mem::offset_of!(Descriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(Descriptor, completion_ordinal) == 0xf70); +static_assert!(core::mem::offset_of!(Descriptor, shared_control_va) == 0xfac); +static_assert!(core::mem::offset_of!(Descriptor, usage_fw_va) == 0xfc5); +static_assert!(core::mem::offset_of!(Descriptor, completion_identity) == 0xfcd); + +// SAFETY: Every field consists only of integer values and their arrays. +unsafe impl Zeroable for Descriptor {} + +impl Descriptor { + /// Fill the entire prefix; no address or register becomes visible before + /// validation completes. The owner publishes only after this returns. + pub(crate) fn write(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + /// Populate a prefix already cleared by `KernelObject::initialize`. + /// The caller must provide all-zero storage so reserved bytes stay zero. + pub(crate) fn write_zeroed(&mut self, args: &DescriptorArgs) -> Result { + self.populate::(args) + } + + fn populate(&mut self, args: &DescriptorArgs) -> Result { + if args.cdm_end_va < args.registers.cdm_va || args.qid >= 128 { + return Err(EINVAL); + } + let (main, count) = program(&args.registers)?; + let alternate = alternate(&args.registers)?; + let main_va = offset_va(args.descriptor_va, 0x40)?; + let alternate_va = offset_va(args.descriptor_va, 0x760)?; + // Also validate the kick's encodable addresses before mutating storage. + bindings(args.descriptor_va, args.registers.scratch, args.registers.matrix_ops)?; + if CLEAR { + super::super::clear(self); + } + self.tag = 3; + self.submit_sequence = args.submit_sequence; + self.context_id = args.registers.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = args.registers.scratch.checkpoints().map(|n| n + if args.registers.matrix_ops { 2 } else { 0 }); + super::set_register_array(&mut self.registers, main_va, &main[..count])?; + super::set_register_array(&mut self.alternate, alternate_va, &alternate)?; + self.preempt_va = args.registers.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.control = 0x154024201; + self.predecessor_id = (args.registers.object_ids >> 32) as u32; + self.unk_f28 = u32::MAX; + self.stamp_va = args.stamp_va; + self.stamp_value = args.stamp_value; + self.qid = args.qid; + self.kick_timestamp = args.kick_timestamp; + self.current_id = args.registers.object_ids as u32 as u64; + self.completion_ordinal = args.completion_ordinal; + self.status_va = args.status_va; + self.timestamp_va = args.timestamp_va; + self.shared_control_va = args.shared_control_va; + self.shared_control_enabled = u8::from(args.shared_control_va != 0); + self.event_metadata = args.event_metadata; + self.usage_fw_va = args.usage_fw_va; + self.completion_identity = args.completion_identity; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 4c0cd39d6da156..130e914d845104 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -187,9 +187,12 @@ pub(super) struct InitData { qos: usize, pb: usize, page_pool: usize, - compute_global: usize, - boot_control: usize, - boot_state: usize, + /// The G17P compute objects: the global state page and the boot free + /// list's control and state. M5 has none of them, and its build leaves + /// these `None`, so an unset handle can never name another object. + compute_global: Option, + boot_control: Option, + boot_state: Option, metrics_fw_va: u64, /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). pm_metrics_va: u64, @@ -292,9 +295,9 @@ impl InitData { qos: 0, pb: 0, page_pool: 0, - compute_global: 0, - boot_control: 0, - boot_state: 0, + compute_global: None, + boot_control: None, + boot_state: None, metrics_fw_va: 0, pm_metrics_va: 0, recovery_slot_keys: None, @@ -367,16 +370,25 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks( - &self, - slot: u16, - page_list_va: u64, - blocks: usize, - ) -> Result { + /// Called for a fresh row or after its free-list release was consumed. The row has the + /// family's format: a G17P block row, or the M5 pool's descriptor. + pub(super) fn install_compute_pool(&self, descriptor: super::freelist::PoolDescriptor) -> Result { + use super::freelist::PoolDescriptor; + let slot = match descriptor { + PoolDescriptor::Neo { slot, .. } | PoolDescriptor::M5 { slot, .. } => slot, + }; if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } + match descriptor { + PoolDescriptor::Neo { slot, page_list, blocks } if !self.m5 => + self.initialize_compute_pool_blocks(slot, page_list, blocks), + PoolDescriptor::M5 { slot, words } if self.m5 => self.set_m5_uma(slot, words), + _ => Err(EINVAL), + } + } + + fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; @@ -399,6 +411,19 @@ impl InitData { Ok(()) } + /// One M5 page-pool row: the pool's eight descriptor words at its 64-byte slot. + fn set_m5_uma(&self, slot: u16, words: [u64; 8]) -> Result { + let object = self.object(self.page_pool)?; + let offset = usize::from(slot) * core::mem::size_of::<[u64; 8]>(); + // Check the full row before the first store. + object.pointer(offset, core::mem::size_of::<[u64; 8]>())?; + for (index, word) in words.into_iter().enumerate() { + object.dword(offset + index * 8)?.store(word, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + /// The statistics record tags the family's firmware emits. pub(super) fn statistics_types(&self) -> u64 { if self.m5 { @@ -455,7 +480,9 @@ impl InitData { if self.compute_shared.is_some() { return Ok(()); } - let state_va = self.object(self.boot_state)?.gpu_va(); + let boot_control = self.boot_control.ok_or(ENODEV)?; + let boot_state = self.boot_state.ok_or(ENODEV)?; + let state_va = self.object(boot_state)?.gpu_va(); let mut control = abi::FreeListControl::new(&abi::FreeListArgs { buffer_slot: u32::MAX, class: abi::FreeListClass::Two, @@ -465,9 +492,9 @@ impl InitData { state_va, }); control.run_cursor = COMPUTE_CONTROL_CURSOR; - self.object_mut(self.boot_control)?.bytes_mut().fill(0); - self.object_mut(self.boot_control)?.write(0, control)?; - self.object_mut(self.boot_state)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.bytes_mut().fill(0); + self.object_mut(boot_control)?.write(0, control)?; + self.object_mut(boot_state)?.bytes_mut().fill(0); shared .pages .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { @@ -487,10 +514,14 @@ impl InitData { Ok(()) } + /// Whether `vm` has the shared compute aperture mapped. M5 has no shared + /// aperture: its compute queues need only their own pools, so every VM is ready. pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { - self.compute_shared - .as_ref() - .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + self.m5 + || self + .compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) } pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { @@ -560,7 +591,7 @@ impl InitData { return Ok(()); } let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); - let object = self.object(self.compute_global)?; + let object = self.object(self.compute_global.ok_or(ENODEV)?)?; let state = object .pointer(0, core::mem::size_of::())? .cast::(); @@ -1096,7 +1127,7 @@ impl InitData { }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; - self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + self.compute_global = Some(self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?); let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { @@ -1172,8 +1203,8 @@ impl InitData { let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { - self.boot_control = control; - self.boot_state = state; + self.boot_control = Some(control); + self.boot_state = Some(state); } } self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9e3f7092ea91ec..3e80f72ad8b953 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -275,9 +275,17 @@ pub(crate) trait Backend: Send + Sync { None } + /// The device accepts command submission (an M5 GPU before its queues take submission + /// does not). + fn supports_submission(&self) -> bool { true } + fn supports_engine(&self, _engine: Engine) -> bool { true } + /// Device policy applied to a validated command before it is ordered (the M5 resource + /// plan). + fn prepare_command(&self, _command: &mut super::command::Validated) -> Result { Ok(()) } + /// Validate compute sizing without reserving a physical channel. Queued /// commands may wait on a host signal submitted after this ioctl returns. - fn ensure_compute(&self, scs_pages: u16) -> Result; + fn ensure_compute(&self, command: &super::command::Validated) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } Ok(super::fw::freelist::RENDER_BLOCKS) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index cdc26aa341f0a2..9e26d47a309265 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -962,6 +962,10 @@ struct Shared { descriptor_flags: [u32; 2], /// Firmware events and work run: off for an M5 GPU until the M5 queues take submission. submission_enabled: bool, + /// An M5 GPU before its render queues take submission: compute only. + firmware_test: bool, + /// The M5 shared-queue resource policy (frozen device topology). + render_policy: Option, crashed: AtomicBool, feed: Arc, #[pin] @@ -1310,7 +1314,7 @@ impl Gpu { let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; - let feed = feed::Feed::new()?; + let feed = feed::Feed::new(m5)?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { clusters == id.num_clusters @@ -1341,6 +1345,7 @@ impl Gpu { "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" ); } + let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1353,6 +1358,8 @@ impl Gpu { compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], submission_enabled: !m5, + firmware_test: m5, + render_policy, crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f060c9426d885d..0fb686001cd091 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -4,6 +4,8 @@ //! installed graph; the device retains it until both firmware processors have stopped. pub(crate) mod exit; +mod activation; +mod configuration; pub(crate) mod retirement; use crate::g17::{ @@ -87,7 +89,7 @@ pub(crate) struct Binding { preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, - _queue_context: mmu::KernelMapping, + _queue_context: Option, context: Arc, } @@ -126,14 +128,16 @@ impl DescriptorRing { /// Firmware-shared storage of one compute QID, separate from its common kick ring. /// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { + m5: bool, ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, support: KernelObject, + m5_timestamps: Option>, graph: PooledObject, - context_low: KernelObject, - context_high: KernelObject, + context_low: Option, + context_high: Option, /// Announcements, which stay uncached for the firmware. records: KernelObject, /// Configuration, predecessor and completion-wait records at the same @@ -156,6 +160,7 @@ impl Graph { pointers: PointerLayout, operand_state: &'static OperandState, program: ComputeProgram, + m5: bool, ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; @@ -163,13 +168,16 @@ impl Graph { let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(operand_state.size, page, prot, cpu)?; - let mut support = alloc.kernel(0xc000, page, prot, cpu)?; - let support_va = support.gpu_va(); - support.write(0, support_va + page)?; - support.write(0x100, support_va + page + 4)?; - support.write(0x110, 0x50u32)?; - support.write(mmu::UAT_PGSZ + 4, 1u32)?; + let operand = alloc.lower(if m5 { crate::g17::fw::m5::compute::OPERAND_BYTES } + else { operand_state.size }, page, prot, cpu)?; + let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, prot, cpu)?; + if !m5 { + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + } let mut graph = objects.allocate( alloc, Family::QueueState, @@ -180,14 +188,6 @@ impl Graph { )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { - view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + CURSORS as u64, - items_va: graph_va + ITEMS as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }); view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) @@ -196,9 +196,29 @@ impl Graph { *view = PointerBlock::new(&pointers)?; Ok(()) })?; - let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; - let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let record = QueueRecordArgs { + pointers_va: graph_va + CURSORS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + // One queue record, the firmware's own: the M5 record is shorter than the G17P one the + // layout's slot holds, and the zeroed layout leaves the rest of the slot zero. + if m5 { + graph.write(0, crate::g17::fw::m5::queue::QueueRecord::new(&record)?)?; + } else { + graph.write(0, QueueRecord::new(&record))?; + } + let (context_low, context_high) = if m5 { (None, None) } else { + (Some(alloc.lower(CONTEXT_SIZE, page, prot, cpu)?), + Some(alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?)) + }; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let m5_timestamps = if m5 { + Some(Arc::new(alloc.kernel(RECORD_SLOTS * 16, page, prot, cpu)?, GFP_KERNEL)?) + } else { None }; let cached_records = objects.allocate( alloc, Family::QueueConfig, @@ -208,11 +228,13 @@ impl Graph { cpu, )?; Ok(Self { + m5, ring: Some(ring), preempt, usage, operand, support, + m5_timestamps, graph, context_low, context_high, @@ -237,9 +259,9 @@ impl Graph { return Err(EFAULT); } let vm = context.vm(); - vm.install_job_context_aliases()?; + if !self.m5 { vm.install_job_context_aliases()?; } let descriptors = self.descriptor_alias(vm)?; - let prot = mmu::PROT_GPU_FW_SHARED_RW; + let prot = if self.m5 { mmu::PROT_GPU_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }; let page = mmu::UAT_PGSZ as u64; let preempt = self .preempt @@ -248,9 +270,8 @@ impl Graph { let operand = self .operand .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let queue_context = self - .context_low - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let queue_context = self.context_low.as_ref() + .map(|context| context.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)).transpose()?; Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), @@ -265,6 +286,15 @@ impl Graph { /// Caller has excluded publication and proved the retained graph idle. fn set_owner(&mut self, context: &Context) -> Result { + if self.m5 { + let pointer = self.graph.pointer(0, + core::mem::size_of::())? + .cast::(); + // SAFETY: The checked packed record stays mapped. The device mutex + // and idle witness exclude access by another owner during this update. + return unsafe { (*pointer).set_compute_owner(context.work_head_va(), + context.scheduler_va(), context.owner_pid(), context.policy()) }; + } let pointer = self .graph .pointer(0, core::mem::size_of::())? @@ -373,7 +403,8 @@ impl Graph { } use crate::g17::{ - completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + completion::Deferred, context::WorkStateLease, + freelist::{ComputeCapacity, ComputePool, PoolDescriptor}, job::Packet, kick, qos, recovery, status::VmStatus, }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; @@ -496,14 +527,14 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { - let pool = ComputePool::sized(alloc, id.qid(), blocks)?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, required: ComputeCapacity) -> Result { + let pool = ComputePool::new(alloc, id.qid(), required)?; Ok(Self { pool, id }) } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { - Ok((self.pool.id(), self.pool.page_list_va()?)) + pub(crate) fn pool_descriptor(&self) -> Result { + self.pool.descriptor() } } @@ -549,7 +580,8 @@ pub(crate) struct Queue { pool_phase: PoolPhase, release_sent_ns: Option, growing: bool, - growth_blocks: usize, + /// What the growth in progress must provide. + growth: Option, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, @@ -575,6 +607,8 @@ pub(crate) struct Queue { /// and whether that wait was reported. kill_wait: Option<(u64, bool)>, installed: bool, + configuration: Option, + m5_install: Option, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, @@ -618,9 +652,10 @@ impl Queue { pointers: PointerLayout, operand_state: &'static OperandState, program: ComputeProgram, + m5: bool, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program, m5)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -633,7 +668,7 @@ impl Queue { pool_phase: PoolPhase::Populated, release_sent_ns: None, growing: false, - growth_blocks: 0, + growth: None, needs_fresh_backing: false, released_at: 0, free_list_generation: 0, @@ -651,6 +686,8 @@ impl Queue { exit: None, kill_wait: None, installed: false, + configuration: None, + m5_install: None, outer_started: 0, published_config: None, config_cache: None, @@ -740,10 +777,16 @@ impl Queue { !self .context() .is_some_and(|bound| Arc::ptr_eq(bound, context)) - || (self.active.len == 0 && self.replays.len == 0) + || (self.active.len == 0 && self.replays.len == 0 && self.configuration.is_none()) } pub(crate) fn in_flight(&self) -> bool { - self.active.len != 0 && !self.quarantined + (self.active.len != 0 || self.configuration.is_some()) && !self.quarantined + } + /// Cold M5 installation publishes one configuration before the first kick. + /// A prepared claim prevents reassignment while these credits are owned. + #[allow(dead_code)] // Read by M5 lazy admission, a follow-up. + pub(crate) fn outer_slots(&self) -> u32 { + if self.graph.m5 && self.m5_install.is_none() { 2 } else { 1 } } /// The oldest command is a published long job (ray traversal or AS build) whose /// completion record the driver has not seen yet. @@ -826,7 +869,7 @@ impl Queue { /// A completion hint can require a second visibility/recordless pass even /// when firmware has no more events to send. Do not poll idle or failed queues. pub(crate) fn retirement_polling(&self) -> bool { - !self.quarantined && self.retire_pending + self.configuration.is_some() || (!self.quarantined && self.retire_pending) } /// A release has no completion event: keep observing the control consumer @@ -981,6 +1024,7 @@ impl Queue { mut complete_qos: impl FnMut(qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { + if recovery_state == 0 { self.retire_configuration(epoch)?; } if self.quarantined || !self.selected(masks) { return Ok(()); } @@ -1028,7 +1072,7 @@ impl Queue { recordless, self.last_end, next_end, - record_aged, + record_aged && !self.graph.m5, )?; if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { retirement::note_stampless(self.qid(), ticket.kick); @@ -1062,10 +1106,18 @@ impl Queue { ); } self.retire_pending = true; - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok([timestamps[0], timestamps[1], 0, 0]), - ))?; + if self.graph.m5 { + defer(Deferred::RetiredCompute( + front.packet.completion.clone(), + [timestamps[0], timestamps[1], 0, 0], + front.observed.map(|record| record.timestamps), + ))?; + } else { + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok([timestamps[0], timestamps[1], 0, 0]), + ))?; + } self.retire_pending = false; self.submitted = submitted; let active = self.active.pop().ok_or(EIO)?; @@ -1139,21 +1191,42 @@ impl Queue { )? { return Ok(()); } + let unexecuted = if self.graph.m5 { + // M5 command words stay zero even for executed commands. The + // front needs its exact zero-duration ring record; a later + // command, chained to its predecessor by the kick parent, must + // not show a record with a real duration. As on G17P, work a + // render consumes through a firmware dependency is never spared. + self.active.iter().enumerate().all(|(index, active)| { + !active.packet.completion.has_firmware_consumer() + && !active.packet.completion.signalled() + && match active.observed { + // Only reached for spared work quarantined after a + // recovery: every retained command crossed it. + Some(record) => { + record.timestamps[0] != 0 && retirement::unexecuted(record, true) + } + None => index != 0, + } + }) + } else { + !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }) + }; let candidate = self.spared_deferred && self.spared_quarantine && !self.released && self.owner == Some(newest.owner) - && !self - .active - .iter() - .any(|active| active.packet.completion.has_firmware_consumer()) - && self.active.iter().all(|active| { - active - .packet - .completion - .replay_witness_words() - .is_some_and(|words| words[0] == words[1]) - }); + && unexecuted; if candidate && !replay { return Ok(()); } @@ -1262,6 +1335,7 @@ impl Queue { /// Caller already stopped both processors. No firmware owner can access retained mappings. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { configuration.release(); } while let Some(packet) = self.replays.front() { defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; drop(self.replays.pop()); @@ -1539,6 +1613,7 @@ impl Queue { return false; } self.installed + && self.configuration.is_none() // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // A recycled killed queue is offered only once its backing is vacant (re-arm path). @@ -1625,7 +1700,7 @@ impl Queue { return Ok(true); } self.released = true; - if self.active.len == 0 && !self.quarantined { + if self.active.len == 0 && !self.quarantined && self.configuration.is_none() { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; @@ -1700,7 +1775,7 @@ impl Queue { /// ring is reset to empty at the firmware's consumer cursor; the USC pool and /// descriptor ring are vacated and re-armed fresh before the next owner binds. pub(crate) fn recycle_killed(&mut self) -> Result { - if !self.installed + if self.configuration.is_some() || !self.installed || !self.exit.as_ref().is_some_and(|exit| exit.released()) || self.active.len != 0 || self.replays.len != 0 @@ -1824,7 +1899,7 @@ impl Queue { &mut self, pages: crate::g17::freelist::Pages, ring: DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { if !self.can_rearm() { return Err(EAGAIN); } @@ -1833,10 +1908,10 @@ impl Queue { self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.needs_fresh_backing = false; - Ok((self.pool.id(), self.pool.page_list_va()?)) + self.pool.descriptor() } - pub(crate) fn pool_fits(&self, required: usize) -> bool { - !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits_blocks(required) + pub(crate) fn pool_fits(&self, required: ComputeCapacity) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits_capacity(required) } /// Only a capacity increase waits for this queue's previously published work. @@ -1844,7 +1919,7 @@ impl Queue { pub(crate) fn begin_growth( &mut self, init: &crate::g17::initdata::InitData, - blocks: usize, + required: ComputeCapacity, ) -> Result { if self.growing { return Err(EAGAIN); @@ -1871,7 +1946,7 @@ impl Queue { self.release_sent_ns = None; } self.growing = true; - self.growth_blocks = blocks; + self.growth = Some(required); Ok(self.installed) } @@ -1880,9 +1955,10 @@ impl Queue { init: &crate::g17::initdata::InitData, pages: &mut Option, alias: &mut Option, - blocks: usize, ) -> Result { - if !self.growing || blocks < self.growth_blocks { + // The fresh backing must provide what the growth was begun for. + let target = self.growth.filter(|_| self.growing).ok_or(EAGAIN)?; + if !pages.as_ref().ok_or(EINVAL)?.fits_capacity(target) { return Err(EAGAIN); } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { @@ -1899,13 +1975,13 @@ impl Queue { let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. // The old release is consumed; no queue can use this row until commit. - init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + init.install_compute_pool(fresh.descriptor(self.pool.id())?)?; self.pool.replace_pages(pages)?; let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; self.release_sent_ns = None; self.growing = false; - self.growth_blocks = 0; + self.growth = None; Ok(Vacated { _alias: old_alias, _pages: pages.take(), @@ -1929,7 +2005,7 @@ impl Queue { return Ok(None); } self.growing = false; - self.growth_blocks = 0; + self.growth = None; self.release_owner().map(Some) } @@ -2157,6 +2233,7 @@ pub(crate) trait Host { fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; + fn submit_m5(&mut self, submission: &crate::g17::fw::m5::queue::HardwareSubmission) -> Result; } impl Queue { @@ -2191,6 +2268,11 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + matrix_ops: requested_matrix_ops, + cdm_auxiliary_allocation, + plan, + m5_intersection, + m5_scratch, intersection, scratch, usc_base, @@ -2203,20 +2285,26 @@ impl Queue { else { return Err(EINVAL); }; + let m5_scratch = *m5_scratch; + if !self.graph.m5 && (*requested_matrix_ops || *cdm_auxiliary_allocation != 0) { return Err(EINVAL); } + if self.graph.m5 { + if *sampler_count != 0 || !native.is_empty() { return Err(ENOTSUPP); } + if plan.is_some_and(|plan| !self.pool.fits(Some(plan.geometry))) { return Err(EINVAL); } + self.install_m5(&packet, host)?; + } let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); } let work_node = work.node_va()?; - let timestamp_va = packet.completion.compute_timestamp_va()?; let kick_timestamp = self.kick.timestamp(); let kick_parent = self.kick.parent(); let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; - let first = !self.installed && self.ordinal == 0; + let first = self.ordinal == 0 && (self.graph.m5 || !self.installed); // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. if first && !native.is_empty() { @@ -2229,7 +2317,22 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); - item_capacity(self.graph.cursors()?, entry_signal)?; + if self.graph.m5 { + // The configuration and its notification are the only items an M5 kick appends. + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < 2 { + return Err(ENOSPC); + } + } else { + item_capacity(self.graph.cursors()?, entry_signal)?; + } if !native.is_empty() { let cursors = self.graph.cursors()?; let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; @@ -2237,10 +2340,15 @@ impl Queue { return Err(EAGAIN); } } - let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + let record_slot = (self.graph.m5 || !first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; } + let timestamp_va = if self.graph.m5 { + packet.completion.bind_m5_compute_timestamps( + self.graph.m5_timestamps.as_ref().ok_or(EINVAL)?, + record_slot.ok_or(EINVAL)? * 16)? + } else { packet.completion.compute_timestamp_va()? }; let args = ComputeArgs { program: self.graph.program, intersection: *intersection, @@ -2280,20 +2388,48 @@ impl Queue { }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); - let ring = 1u8 << policy.priority(); - let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let activation = activation::Publication::new( + self.outer_started, policy.priority(), self.graph.m5); + let config = self.queue_config(&identity, first, activation.refresh, + args.context_id, policy, &packet)?; let cached = self .config_cache .and_then(|(config, at)| (at == epoch).then_some(config)); - let send_config = first || super::render::publication::required(cached, config); + let send_config = self.graph.m5 || first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; + let matrix_ops = self.graph.m5 && *requested_matrix_ops; + let m5_args = crate::g17::fw::m5::compute::DescriptorArgs { + registers: crate::g17::fw::m5::compute::RegisterArgs { + matrix_ops, + scratch: m5_scratch, + intersection: *m5_intersection, + auxiliary_allocation: *cdm_auxiliary_allocation, + preempt_va: addresses.preempt, cdm_va: *cdm_va, operand_va: addresses.operand, + usage_va: addresses.usage, free_list_slot: u32::from(self.pool.id()), + context_id: args.context_id, work_state: work_word, + object_ids: 0x0000_0001_0000_0001, + resource_policy: plan.map_or(0xf5_0040_0020, |p| u64::from(p.pressure) << 32 | 0x0040_0020), + preempt_state_offset: 0x1b80, operand_region_bytes: 0x1e000, + operand_aux_offset: 0x3d800, resource_control: plan.map_or(0x54b, |p| p.resource_flags), + dispatch_control: 0x14c08a60, auxiliary_control: 1, + }, + descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, + cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], + stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), + kick_timestamp: kick_timestamp.get(), completion_ordinal: 0, + status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], + shared_control_va: self.pool.control_va(), event_metadata: [0;16], + usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, + }; + let m5_dependency_types = [2u8; crate::g17::fw::kick::QID_MAX as usize + 1]; let kick_args = KickArgs { qid, timestamp: kick_timestamp, parent: kick_parent, barriers: dependencies, + dependency_types: self.graph.m5.then_some(&m5_dependency_types[..dependencies.len() + 1]), descriptor_va: descriptor_high, queue_va: self.graph.graph.gpu_va(), qos: KickQos { @@ -2301,9 +2437,14 @@ impl Queue { class: COMPUTE_QOS_CLASS, }, mcache: None, - event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)?, - scs_enabled: scratch.enabled(), + event_mask: if self.graph.m5 { [0, 0, 8, 0] } else { COMPUTE_KICK_EVENT_MASK }, + register_arrays: if self.graph.m5 { + crate::g17::fw::m5::compute::bindings(descriptor_low, m5_scratch, matrix_ops)? + } else { + ComputeDescriptor::register_bindings(descriptor_low, *scratch, self.graph.program)? + }, + scs_enabled: if self.graph.m5 { m5_scratch.enabled() } else { scratch.enabled() }, + compute_matrix: matrix_ops, priority: policy.priority(), }; let published = host.next_compute_publication()?; @@ -2326,17 +2467,16 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph - .ring - .as_mut() - .ok_or(EAGAIN)? - .object - .initialize::( - kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write_zeroed(&args), - )?; - if first { - self.graph.context_high.write( + let object = &mut self.graph.ring.as_mut().ok_or(EAGAIN)?.object; + let offset = kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + if self.graph.m5 { + object.initialize::(offset, + |descriptor| descriptor.write_zeroed(&m5_args))?; + } else { + object.initialize::(offset, |descriptor| descriptor.write_zeroed(&args))?; + } + if !self.graph.m5 && first { + self.graph.context_high.as_mut().ok_or(EINVAL)?.write( 0x200, crate::g17::fw::compute::QueueContextItem::new( descriptor_high, @@ -2344,12 +2484,13 @@ impl Queue { qid, ), )?; - } else { + } else if !self.graph.m5 { // The retained context item remains at index one; only its descriptor changes. - self.graph.context_high.write(0x210, descriptor_high)?; + self.graph.context_high.as_mut().ok_or(EINVAL)?.write(0x210, descriptor_high)?; let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } + if self.graph.m5 { let cursors = self.graph.cursors()?; self.graph.recycle(cursors)?; } let config_va = if send_config { Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) } else { @@ -2373,7 +2514,26 @@ impl Queue { let kick_ring_va = self.kick.low_va(); let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; - if first { + if self.graph.m5 { + let target = self.pool.reserve_submission()?; + let qos = host.qos_publish(qos::Owner { qid, qos: identity.qos, data_master: 2 }, + identity.scheduler, policy).or_else(|error| { + self.pool.cancel_unpublished(target)?; Err(error) + })?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; + producer = self.graph.append(cursors.producer, config_va.ok_or(EINVAL)?, + || packet.mark_published()) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + visible = true; + self.unpublished_binding = false; + // This Active and its Packet pins now own all visible records. + if let Some(configuration) = self.configuration.take() { configuration.release(); } + } else if first { host.register_queue(&crate::g17::fw::kick::KsmQueue::new( qid, kick_ring_va, @@ -2458,10 +2618,13 @@ impl Queue { } prepared_kick.commit(); self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; - let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + let announce_va = if self.graph.m5 { + self.graph.record(record_slot, ANNOUNCE, 0x40, + crate::g17::fw::m5::queue::notification(qid, kick_timestamp)?)? + } else { self.graph.record(record_slot, ANNOUNCE, 0x40, announce)? }; producer = self.graph.append(producer, announce_va, || {})?; fence(Ordering::SeqCst); - if entry_signal { + if entry_signal && !self.graph.m5 { let predecessor_va = self.graph .record(record_slot, PREDECESSOR, 0x400, predecessor)?; @@ -2478,10 +2641,11 @@ impl Queue { self.graph.graph.gpu_va(), qid, producer.try_into()?, - self.outer_started & ring == 0, + activation.first, )?, )?; - self.outer_started |= ring; + let notify_activation = activation.first; + self.outer_started = activation.next_started; self.installed = true; self.published_config = Some(identity); if send_config { @@ -2517,6 +2681,13 @@ impl Queue { // ring itself and delayed dispatch. fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + if self.graph.m5 { + // The firmware work notification must precede a deferred host + // doorbell: an idle-policy hold alone does not wake gated cores. + host.submit_m5(&crate::g17::fw::m5::queue::HardwareSubmission::compute( + qid, self.kick.low_va(), policy.priority(), + first || notify_activation)?)?; + } crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); @@ -2538,6 +2709,58 @@ impl Queue { Ok(()) } + /// The firmware consumes an empty channel installation before host-port + /// configuration. EAGAIN leaves the packet unpublished and is retried by + /// the common runtime with the device mutex released. + fn retire_configuration(&mut self, epoch: u64) -> Result { + let Some(configuration) = self.configuration.as_ref() else { return Ok(()); }; + if !configuration.consumed(self.graph.cursors()?, epoch) { return Ok(()); } + self.configuration.take().ok_or(EIO)?.release(); + if self.ordinal == 0 && self.active.len == 0 && self.replays.len == 0 { + self.retirement_ready = true; + if self.released { self.owner = None; } + } + Ok(()) + } + + fn install_m5(&mut self, packet: &Arc, host: &mut impl Host) -> Result { + let (epoch, state) = host.epoch()?; + if state == 0 { self.retire_configuration(epoch)?; } + if let Some(end) = self.m5_install { + return if self.graph.cursors()?.consumer == end || self.ordinal != 0 { Ok(()) } + else { Err(EAGAIN) }; + } + let context = &packet.context; + let policy = context.policy(); + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), install: true, + completion_seed: None, context_id: context.id().try_into()?, + free_list: FreeListBinding { va: 0, generation: u64::MAX, slot: u32::MAX }, + scheduler_va: context.scheduler_va(), policy, qos_slot: context.qos_id(), + qos_update: true, owner_pid: context.owner_pid(), context_update: false, + })?; + if self.configuration.is_some() { return Err(EBUSY); } + self.configuration = Some(configuration::Configuration::new(context, 1, epoch)?); + let result = (|| { + let va = self.graph.record(None, CONFIG, 0xc0, config)?; + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let producer = self.graph.append(0, va, || configuration.mark_visible())?; + self.m5_install = Some(producer); + self.installed = true; + host.publish_qid(self.kick.id())?; + host.publish_outer(policy.priority(), &crate::g17::fw::channels::WorkSlot::new(DataMaster::Compute, + self.graph.graph.gpu_va(), self.qid(), producer.try_into()?, true)?)?; + self.outer_started = 1 << policy.priority(); + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + Err(EAGAIN) + })(); + if result.is_err() && self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + result + } + /// Everything that must hold before a publication changes any state: a healthy device /// epoch, room in the active ring (or this packet at the replay front), the bound /// owner's current context and a clean completion status. Returns the epoch and the @@ -2590,6 +2813,7 @@ impl Queue { &self, identity: &ConfigIdentity, first: bool, + priority_refresh: bool, context_id: u16, policy: crate::g17::fw::queue::Policy, packet: &Arc, @@ -2599,8 +2823,12 @@ impl Queue { kick_ring_va: self.kick.low_va(), kick_ring_fw_va: self.kick.firmware_va(), qid: self.qid(), - install: first, - completion_seed: None, + // An existing M5 queue needs the firmware install/update path to + // replace its private scheduling state when its class changes. + // The ordinary binding fields below are still processed afterward; + // no completion seed is supplied for a retained queue. + install: (first && !self.graph.m5) || priority_refresh, + completion_seed: (first && self.graph.m5).then_some(0), context_id, free_list: FreeListBinding { va: self.pool.control_va(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs new file mode 100644 index 00000000000000..b306bbe820cc04 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/activation.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A retained M5 channel belongs to its current priority ring, not every ring it +//! has previously used. A priority change must submit it to the new ring again, +//! including when a later owner returns to an earlier priority. + +pub(super) struct Publication { + pub(super) first: bool, + /// Refresh firmware's per-QID scheduling state on a retained class change. + pub(super) refresh: bool, + pub(super) next_started: u8, +} + +impl Publication { + pub(super) fn new(started: u8, priority: u8, m5: bool) -> Self { + let ring = 1u8 << priority; + Self { + first: started & ring == 0, + refresh: m5 && started != 0 && started & ring == 0, + next_started: if m5 { ring } else { started | ring }, + } + } +} + +#[cfg(test)] +mod tests { + use super::Publication; + + #[test] + fn first_publication_activates() { + for priority in 0..4 { + let publication = Publication::new(0, priority, true); + assert!(publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn same_priority_keeps_fast_path() { + for priority in 0..4 { + let publication = Publication::new(1 << priority, priority, true); + assert!(!publication.first); + assert!(!publication.refresh); + assert_eq!(publication.next_started, 1 << priority); + } + } + + #[test] + fn returning_to_old_priority_reactivates() { + for old in 0..4 { + for new in 0..4 { + if old == new { continue; } + let changed = Publication::new(1 << old, new, true); + assert!(changed.first); + assert!(changed.refresh); + assert_eq!(changed.next_started, 1 << new); + let returned = Publication::new(changed.next_started, old, true); + assert!(returned.first); + assert!(returned.refresh); + assert_eq!(returned.next_started, 1 << old); + } + } + } + + #[test] + fn non_m5_preserves_historical_membership() { + let changed = Publication::new(1 << 2, 3, false); + assert!(changed.first); + assert!(!changed.refresh); + assert_eq!(changed.next_started, (1 << 2) | (1 << 3)); + let returned = Publication::new(changed.next_started, 2, false); + assert!(!returned.first); + assert!(!returned.refresh); + assert_eq!(returned.next_started, changed.next_started); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs new file mode 100644 index 00000000000000..7531ee2b0b33d2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/configuration.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initial M5 queue installation names physical execution/QoS identities +//! before an Active command exists. Retain the exact tokens independently. + +use super::{Context, Cursors, ITEM_RING_ENTRIES}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + target: u32, + epoch: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, target: u32, epoch: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, target, epoch, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: Cursors, epoch: u64) -> bool { + self.visible && self.epoch == epoch && self.target < ITEM_RING_ENTRIES + && cursors.count == ITEM_RING_ENTRIES + && cursors.producer == self.target && cursors.consumer == self.target + } + /// Only before visibility, after exact empty-channel consumption, after + /// transfer to the same pinned Active, or after verified processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 719a74787ca706..b731e8e24c38c9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -65,6 +65,9 @@ enum Phase { Published, Acknowledged, Unproven, + /// M5: the full-queue retirement witness covered the killed commands. Cached across + /// collector failures, since `submitted` still counts entries a retry already drained. + Witnessed, Released, } @@ -196,13 +199,13 @@ impl Queue { let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { return Ok(()); }; - if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { + if !matches!(phase, Phase::Acknowledged | Phase::Unproven | Phase::Witnessed) { return Ok(()); } // An unproven kill (acknowledged before the scheduler page showed the killed // state) is re-read on later passes and settles only once the page proves it, // even after its commands were retired by other means. - if self.active.len != 0 || phase == Phase::Unproven { + if (self.active.len != 0 || phase == Phase::Unproven) && phase != Phase::Witnessed { if !recovery_closed { return Ok(()); } @@ -217,7 +220,33 @@ impl Queue { self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; return Ok(()); } - if !self.active.iter().all(|active| { + if self.graph.m5 { + // A kill acknowledgement and the killed flag identify the control + // transaction, not the end of the firmware's reads, and M5 compute + // timestamp words stay zero even for executed work. The commands stay + // until the full-queue retirement witness covers the newest of them. + if self.active.len != 0 { + for active in self.active.iter() { + let Some(ticket) = active.ticket.as_ref() else { + return Ok(()); + }; + if self.valid_ticket(active, ticket).is_err() { + return Ok(()); + } + } + let newest = self.active.back().and_then(|active| active.ticket).ok_or(EIO)?; + if !super::retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.witness_stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Witnessed; + } else if !self.active.iter().all(|active| { active .packet .completion diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index d7f821bbce68d1..902e7ff03ef04b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -141,6 +141,18 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { } } +/// Firmware retires the compute commands in flight at a restart without executing them and +/// stores a record whose end is at most this many timer ticks after its start (0 or 1 tick +/// observed). The shortest executed command record seen is 18 ticks. +pub(crate) const CANCELLED_TICKS: u64 = 1; + +/// The record shows the command never executed: zero duration, or, for a command that was in +/// flight across a recovery (`crossed`), the restart's cancellation record. +pub(crate) fn unexecuted(record: Observation, crossed: bool) -> bool { + let [start, end] = record.timestamps; + start == end || (crossed && start != 0 && end >= start && end - start <= CANCELLED_TICKS) +} + /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 8717cad85ec62f..c0a916a5a559e3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -70,7 +70,7 @@ impl Frontend { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, @@ -85,6 +85,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -105,7 +106,10 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -114,6 +118,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -126,6 +131,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -143,10 +149,10 @@ impl Queue { }) } - fn ensure_compute(&mut self, scs_pages: u16) -> Result { + fn ensure_compute(&mut self, command: &Validated) -> Result { // Validate every request, including an existing scheduler. Physical // admission happens after its DRM dependencies, in backend publication. - self.backend.ensure_compute(scs_pages)?; + self.backend.ensure_compute(command)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -160,7 +166,11 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; // Without room the entries are released here, as they always could be. - let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { Some(&mut self.finished) } else { None @@ -198,35 +208,51 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; - self.ensure_compute(scratch.pages())?; + self.ensure_compute(&command)?; } // Only convert inputs when no independent host-prefix wait could keep // the retained source bookkeeping queued behind an external signal. if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute && !inputs.is_empty() + let sources = if engine == Engine::Compute + && !inputs.is_empty() && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| - !Order::contains(order.wait_through[0], *sequence) - || fences.status() > 0) - && self.frontiers[1].iter().all(|(sequence, fences)| + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0) - && self.failures.iter_mut().enumerate().all(|(index, failure)| - failure.earliest().is_none_or(|(sequence, _)| - !Order::contains(order.wait_through[index], sequence))) - { self.backend.input_sources(inputs) } else { None }; + || fences.status() > 0 + }) + && self.failures.iter_mut().enumerate().all(|(index, failure)| { + failure.earliest().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); } // Only earlier render ordinals covered by the explicit logical prefix @@ -234,8 +260,7 @@ impl Queue { let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -249,8 +274,10 @@ impl Queue { host_predecessors.clone().next().is_some(), self.backend.feed(), sources.clone(), + self.usage.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -294,8 +321,10 @@ impl Queue { }); } let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -333,7 +362,10 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } - if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { continue; } dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -363,10 +395,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -389,7 +422,7 @@ impl Queue { let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { - let payload = self.proof.validate( + let mut payload = self.proof.validate( self.context.vm(), self.range.clone(), &self.window, @@ -400,7 +433,9 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new( + if !self.backend.supports_engine(engine) { return Err(ENOTSUPP); } + self.backend.prepare_command(&mut payload)?; + let mut order = Order::new( base, preceding, command.barriers, @@ -433,8 +468,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -448,22 +485,25 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { + if !self.backend.supports_submission() { + return Err(ENOTSUPP); + } if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { self.context.note_render(); } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); let firmware_source = pending.len() == 1 diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index e5494ffccd8e9f..2b231de157a71b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -726,6 +726,8 @@ impl Pair { event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, scs_enabled: ray_query[stage].scratch.enabled(), + dependency_types: None, + compute_matrix: false, priority: policy.priority(), }; let ta_args = args( diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index af76e8b62f0403..15ec8897499d19 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -265,7 +265,7 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { @@ -312,8 +312,8 @@ impl Registry { /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks /// only whether some channel is free for a new owner. - fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { - if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + fn compute_readiness(&self, owner: u64, query: Option<(&Arc, super::freelist::ComputeCapacity)>) -> Readiness { + if let (Some((context, required)), Some(queue)) = (query, self.compute(owner).filter(|_| owner != 0)) { return if queue.spared_pending() { Readiness::Progress } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { @@ -344,7 +344,7 @@ impl Registry { /// Binds `owner` to a reusable channel whose last owner released it, or /// names a vacant one to re-arm. None if no channel is reusable. - fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; for mut queue in self @@ -379,7 +379,7 @@ impl Registry { qid: u8, pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, - ) -> Result<(u16, u64)> { + ) -> Result { let mut queue = self .compute .iter_mut() @@ -795,6 +795,11 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. fn shared_compute(&self, admission: &Admission<'_>) -> Result { + // The shared aperture is G17P compute state. M5 compute queues need only + // their own pools, and the objects it would install are not M5's. + if self.context.is_m5() { + return Ok(()); + } loop { admission.check()?; let lease = self.shared.preparations.enter_until(admission)?; @@ -937,7 +942,7 @@ impl Backend { Ok(()) } - fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + fn grow_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; { let state = self.shared.state.lock(); @@ -950,7 +955,7 @@ impl Backend { let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; - let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); + let mut pages = Some(super::freelist::Pages::for_compute(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; let mut growth = None; @@ -977,7 +982,7 @@ impl Backend { firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } } - queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + queue.finish_growth(&firmware.init, &mut pages, &mut alias).map(Some) })(); deferred.finish(); match result { @@ -1018,7 +1023,7 @@ impl Backend { result } - fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + fn build_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; self.admission_open()?; loop { @@ -1098,7 +1103,7 @@ impl Backend { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + let pages = crate::g17::freelist::Pages::for_compute(&alloc, required)?; Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { @@ -1123,13 +1128,13 @@ impl Backend { .filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.qid() == qid && queue.can_rearm()); if !ready { drop(state); continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { + let descriptor = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, }; // The firmware consumed this slot's release before the backing // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; + firmware.init.install_compute_pool(descriptor)?; firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; return Ok(()); } @@ -1145,13 +1150,15 @@ impl Backend { } self.admission_open()?; firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; + firmware.init.install_compute_pool(pool.pool_descriptor()?)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease, admission)?; + // It is G17P compute state; M5 has none. + if !self.context.is_m5() { + self.compute_scheduler(&alloc, &lease, admission)?; + } let queue = compute::Queue::new( &alloc, &objects, @@ -1161,6 +1168,7 @@ impl Backend { self.shared.variant.pointers, self.shared.operand_state, self.shared.variant.compute, + alloc.uat.is_m5(), )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); @@ -1305,8 +1313,9 @@ impl DeferredBatch { Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => { completion.note_input_error(*error); }, - Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => - completion.note_source_retired(), + Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) + | Deferred::Drained(completion, Ok(_)) => completion.note_source_retired(), _ => {}, } self.values.push_within_capacity(value).map_err(|_| EIO)?; @@ -1315,6 +1324,7 @@ impl DeferredBatch { // succeeds; failure fences and rejected events are not retirement. if let Some( Deferred::Retired(completion, _) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, _) | Deferred::Release(completion), ) = self.values.last() @@ -1336,6 +1346,7 @@ impl DeferredBatch { let mut changed = false; for value in &self.values { if let Deferred::Retired(completion, Ok(_)) + | Deferred::RetiredCompute(completion, _, _) | Deferred::Drained(completion, Ok(_)) = value { if let Err(error) = completion.check_source_inputs() { @@ -1367,6 +1378,7 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, + regs: &'a crate::regs::G17Resources, port: &'a super::port::Port, outer: Option>, effort: &'a mut super::power::Effort, @@ -1384,6 +1396,10 @@ impl compute::Host for ComputeHost<'_> { )) } fn prepare_compute_shared(&mut self) -> Result { + // The global compute state is G17P's; M5 has none to activate. + if self.init.is_m5() { + return Ok(()); + } self.init.activate_compute() } fn qos_publish( @@ -1403,16 +1419,29 @@ impl compute::Host for ComputeHost<'_> { fn register_queue(&mut self, queue: &super::fw::kick::KsmQueue) -> Result { self.port.register_queues(core::slice::from_ref(queue)) } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + if self.init.is_m5() { + return self.init.publish_work(priority, slot).map(|_| ()); + } self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { + // Every M5 pipe-2 doorbell (0x83) is noted in the firmware's submission history first, + // as the render doorbell notes its pair. + if self.init.is_m5() && message >> super::MSG_TYPE_SHIFT == 0x83 && message & 3 == 2 { + if let Err(error) = self.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } self.port.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) } + fn submit_m5(&mut self, submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + self.regs.submit_m5_work(submission) + } } /// Publication into a bound, installed queue through the port alone. The caller checked the @@ -1459,6 +1488,10 @@ impl compute::Host for DirectHost<'_> { fn note_submission(&mut self) -> Result { Ok(()) } + fn submit_m5(&mut self, _submission: &super::fw::m5::queue::HardwareSubmission) -> Result { + // M5 never publishes without the device mutex (see compute_direct). + Err(EINVAL) + } } impl super::Firmware { @@ -1487,6 +1520,10 @@ impl super::Firmware { let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; let mut native: KVec = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + // M5 shared-ring queues have no completion-wait records: the consumer's kick entry names + // the source's kick, and the firmware orders it. + let m5 = self.init.is_m5(); + let mut waited: u128 = 0; for source in &sources.sources { if source.submission.source_status()? { continue; } let found = self.queues.compute.iter_mut().flatten() @@ -1499,6 +1536,12 @@ impl super::Firmware { let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { return Err(EAGAIN); }; + waited |= 1u128 << dependency.qid(); + if m5 { + merged.insert(sequence, dependency)?; + holds.push(hold, GFP_KERNEL)?; + continue; + } // The completion-wait record orders the consumer behind its // source; naming the source in the kick entry as well only adds // a second, slower wait and a predecessor record. @@ -1516,7 +1559,7 @@ impl super::Firmware { if packet.is_published() { for queue in self.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + .filter(|queue| waited & (1u128 << queue.qid()) != 0) { for source in &sources.sources { queue.note_input_consumer(source); } } @@ -1539,8 +1582,10 @@ impl super::Firmware { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // reservation; dropping it unused returns the work-ring slot. - let outer = self.port.reserve_compute(packet.context.policy().priority())?; + // reservation; dropping it unused returns the work-ring slot. M5 compute takes its + // credit from the shared ring under this mutex instead (InitData::publish_work). + let outer = if self.init.is_m5() { None } + else { Some(self.port.reserve_compute(packet.context.policy().priority())?) }; let queues = &mut self.queues; let mut queue = queues .compute @@ -1560,8 +1605,9 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, + regs: &self.bridge.regs, port: &self.port, - outer: Some(outer), + outer, effort: &mut self.effort, qids: &mut queues.qids, generation: self.recovery.generation(), @@ -1579,7 +1625,7 @@ impl Registry { /// Everything ensure_compute establishes for `owner`: the shared compute /// objects mapped into its VM, and its queue bound to `context` with a /// pool that fits and no spared witness to release. - fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + fn compute_bound(&self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity) -> Result { let Some(queue) = self.compute(owner) else { return Ok(false); }; @@ -1739,9 +1785,20 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + /// The compute pool capacity `command` needs, in its family's terms: the M5 + /// plan's geometry, or the G17P block count of its SCS request. + pub(super) fn compute_capacity(&self, command: &super::command::Validated) -> Result { + use super::freelist::ComputeCapacity; + let super::command::Validated::Compute { scratch, plan, .. } = command else { return Err(EINVAL); }; + if self.context.is_m5() { + return Ok(ComputeCapacity::M5(plan.map(|plan| plan.geometry))); + } + super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) + .map(ComputeCapacity::Blocks) + } + + fn ensure_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages)?; let mut deferred = DeferredBatch::publication()?; loop { admission.check()?; @@ -1779,10 +1836,13 @@ impl Backend { fn compute_direct( &self, packet: &Arc, - required: usize, + required: super::freelist::ComputeCapacity, deferred: &mut DeferredBatch, ) -> Option { - if packet.completion.sources().is_some() + // M5 compute is a shared-ring submission on the ring M5 render shares: it is published only + // under the device mutex, never through the classic compute ring producer. + if self.context.is_m5() + || packet.completion.sources().is_some() || self.context.render_seen() || packet.cancel_requested() || packet.completion.status().get() != 0 @@ -1839,8 +1899,7 @@ impl Backend { { return None; } - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()).ok()?; + let required = self.compute_capacity(&packet.command).ok()?; loop { let channels = self.shared.channels.observe(); let progress = self.shared.progress.observe(); @@ -1858,7 +1917,7 @@ impl Backend { } } - fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + fn compute_readiness(&self, packet: &Arc, required: super::freelist::ComputeCapacity) -> Readiness { match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { Err(error) if error == EBUSY => return Readiness::Progress, Err(_) => return Readiness::Ready, @@ -1901,7 +1960,7 @@ impl Backend { { return Readiness::Progress; } - firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + firmware.queues.compute_readiness(self.owner, Some((&self.context, required))) } /// Replaces the prepared packet's channel claim; None releases it after publication, @@ -1980,10 +2039,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Option { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { - return Some(Err(EINVAL)); - }; - let required = match super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages()) { + let required = match self.compute_capacity(&packet.command) { Ok(required) => required, Err(error) => return Some(Err(error)), }; @@ -1997,8 +2053,7 @@ impl Backend { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { return Err(render::FALLBACK); } @@ -2074,8 +2129,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scratch.pages())?; + let required = self.compute_capacity(&packet.command)?; if let Some(result) = self.compute_direct(packet, required, deferred) { return result; } @@ -2115,7 +2169,7 @@ impl Backend { // admission and publication without changing this logical queue. if !bound { demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); - Backend::ensure_compute(self, scratch.pages(), &admission)?; + Backend::ensure_compute(self, required, &admission)?; admission.check()?; } let lease = self.shared.preparations.enter_until(&admission)?; @@ -2340,7 +2394,7 @@ impl super::Firmware { // states) still ends a scheduler wait that has no timeout of its own. if freed || (shared.channels.has_waiters() - && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + && self.queues.compute_readiness(0, None) == Readiness::Ready) { shared.channels.notify(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 10085c3fcb922d..2b501e9c723985 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -169,16 +169,44 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn supports_submission(&self) -> bool { + self.shared.submission_enabled + } + + fn supports_engine(&self, engine: crate::g17::job::Engine) -> bool { + !self.shared.firmware_test || engine == crate::g17::job::Engine::Compute + } + + fn prepare_command(&self, command: &mut crate::g17::command::Validated) -> Result { + use crate::g17::command::Validated; + let Some(policy) = self.shared.render_policy else { return Ok(()) }; + let priority = self.context.policy().priority(); + match command { + Validated::Render { resources, plan, .. } => { + let programs = resources.as_deref().ok_or(EINVAL)?; + *plan = Some(policy.plan(programs, priority)?); + } + Validated::Compute { resources, plan, m5_scratch, .. } => { + if let Some(programs) = resources { + *plan = Some(policy.plan_compute_scs(programs, priority, m5_scratch.layout())?); + } else if m5_scratch.enabled() { + // SCS needs the dispatch storage declarations to size its pool. + return Err(EINVAL); + } + } + } + Ok(()) + } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) } - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, command: &crate::g17::command::Validated) -> Result { // A reverse timeline chain must be submitted before the host signal. // Validate here, but do not consume physical capacity behind unsignalled // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.compute_blocks, self.shared.active_cores, scs_pages) - .map(|_| ()) + self.compute_capacity(command).map(|_| ()) } fn fail_vm(&self, error: Error) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 62ca7895a86f8e..cef991aff64a07 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -259,7 +259,7 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global, None)?; Pair::new( alloc, &resources.objects, @@ -340,7 +340,7 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global, None)?; let prepared = parked.prepare( alloc, &resources.objects, @@ -456,7 +456,7 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global, None)?; pair.prepare_bind( alloc, &resources.objects, From 990bbd25121a17a7802a6199e3eb2c9f14afa46c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:41 -0800 Subject: [PATCH 068/171] drm/asahi: g17: M5: encode M5 render commands Add the M5 tiling and fragment command encodings, their register programs and kick assembly, the M5 block geometry and buffer layout, and the per-generation render state views. Nothing submits them yet. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..77dc6eb34210 (6 commits); SPTM/APIF removed; adapted to integration's fw::render arguments and register programs; interim forms until the render submission commit; comments and names reworded; original 6613f5ab7b7e, 3d2524b7e48a, e25ba8a6fb2b, 5a71513221de, bd4a3d7a0c97, 77dc6eb34210 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit 96caf7397e424a871fe3ad398b3ee15cc56d3c8a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer.rs | 1 + drivers/gpu/drm/asahi/g17/buffer/layout.rs | 105 +++ drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 9 + drivers/gpu/drm/asahi/g17/fw/m5/render.rs | 643 ++++++++++++++++++ .../drm/asahi/g17/fw/m5/render_assembly.rs | 87 +++ .../drm/asahi/g17/fw/m5/render_registers.rs | 587 ++++++++++++++++ .../gpu/drm/asahi/g17/fw/m5/render_tail.rs | 70 ++ drivers/gpu/drm/asahi/g17/fw/m5/target.rs | 59 ++ .../gpu/drm/asahi/g17/fw/render/storage.rs | 8 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + .../drm/asahi/g17/queue/render/generation.rs | 138 ++++ 11 files changed, 1707 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/buffer/layout.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/render_tail.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/target.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/generation.rs diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 3c4e4bf73a394c..801b4d9d36f106 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -11,6 +11,7 @@ use crate::hw::t8140::resources as cfg; pub(crate) mod target; mod geometry; pub(crate) use geometry::BlockGeometry; +mod layout; use kernel::prelude::*; diff --git a/drivers/gpu/drm/asahi/g17/buffer/layout.rs b/drivers/gpu/drm/asahi/g17/buffer/layout.rs new file mode 100644 index 00000000000000..ca4ad547506582 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/layout.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Wire geometry for the shared parameter-buffer owner and growth machinery. + +use super::{fw, BlockGeometry}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(super) struct Layout { + pub(super) pages: usize, + pub(super) page_bytes: usize, + pub(super) control: usize, + pub(super) counter: usize, + pub(super) scenes: usize, + pub(super) blocks: usize, + pub(super) bytes: usize, + pub(super) scratch_slots: usize, + pub(super) scratch_stride: usize, + pub(super) metric_stride: usize, + pub(super) normal_blocks: usize, +} + +fn align(value: usize, alignment: usize) -> Result { + value.checked_add(alignment - 1).map(|n| n & !(alignment - 1)).ok_or(EOVERFLOW) +} + +impl Layout { + pub(super) const NEO: Self = Self { + pages: fw::PAGE_LIST, page_bytes: fw::PAGE_LIST_BYTES, + control: fw::CONTROL, counter: fw::COUNTER, scenes: fw::SCENE_TABLE, + blocks: fw::BLOCK_TABLE, bytes: fw::SIZE, + scratch_slots: fw::SCRATCH_SLOTS, scratch_stride: fw::SCRATCH_STRIDE as usize, + metric_stride: 4, normal_blocks: fw::NORMAL_MAX_BLOCKS, + }; + + pub(super) fn m5(geometry: BlockGeometry) -> Result { + geometry.validate()?; + let blocks = 0x4000; + let pages = align(geometry.capacity.checked_mul(8) + .and_then(|n| n.checked_add(blocks)).ok_or(EOVERFLOW)?, 0x4000)?; + let page_bytes = align(geometry.capacity.checked_mul(geometry.pages()) + .and_then(|n| n.checked_mul(4)).ok_or(EOVERFLOW)?, 0x4000)?; + let scratch_stride = crate::g17::fw::m5::pool::parameter_scratch_stride( + geometry.capacity, 0).map_err(|_| EINVAL)?; + Ok(Self { + pages, page_bytes, control: 0x300, counter: 0x200, + scenes: 0x400, blocks, bytes: pages.checked_add(page_bytes).ok_or(EOVERFLOW)?, + scratch_slots: fw::SCENES, scratch_stride, metric_stride: 64, + normal_blocks: geometry.capacity / 3, + }) + } + + pub(super) fn initialize(self, geometry: BlockGeometry, args: &fw::GraphArgs<'_>, + out: &mut [u8]) -> Result { + geometry.validate()?; + if out.len() < self.bytes || args.generation == 0 || args.blocks.is_empty() + || args.blocks.len() > geometry.max_blocks() { return Err(EINVAL); } + let scratch = args.scratch.checked_sub(geometry.base).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(geometry.base).ok_or(EINVAL)?; + let scratch_end = scratch.checked_add((self.scratch_slots * self.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + if scratch_end > u32::MAX as u64 || discard > u32::MAX as u64 { return Err(EINVAL); } + let pages = u32::try_from(args.blocks.len().checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let maximum = u32::try_from(geometry.capacity.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + let normal = u32::try_from(self.normal_blocks.checked_mul(geometry.pages()).ok_or(EOVERFLOW)?)?; + // Every object is unpublished here. Keep all unused words zero on reuse. + out[..self.bytes].fill(0); + fn put(out: &mut [u8], at: usize, value: [u8; N]) -> Result { + out.get_mut(at..at.checked_add(N).ok_or(EOVERFLOW)?).ok_or(EINVAL)? + .copy_from_slice(&value); + Ok(()) + } + let address = |offset: usize| args.fw_va.checked_add(offset as u64).ok_or(EOVERFLOW); + for (index, &block) in args.blocks.iter().enumerate() { + let first = geometry.compact_page(block)?; + put(out, self.blocks + index * 8, first.to_le_bytes())?; + for page in 0..geometry.pages() { + put(out, self.pages + (index * geometry.pages() + page) * 4, + first.checked_add(page as u32).ok_or(EOVERFLOW)?.to_le_bytes())?; + } + } + for index in 0..fw::SCENES { + let at = self.scenes + index * size_of::(); + let metric = (index * self.metric_stride) as u64; + put(out, at, args.metrics.client.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 8, args.metrics.firmware.checked_add(metric).ok_or(EOVERFLOW)?.to_le_bytes())?; + put(out, at + 0x28, (scratch + (index % self.scratch_slots * self.scratch_stride) as u64).to_le_bytes())?; + put(out, at + 0x40, address(self.control + 0x40)?.to_le_bytes())?; + } + put(out, self.scenes + fw::FIRST_SCENE * size_of::() + 0x4c, args.generation.to_le_bytes())?; + for (at, value) in [ + (0xc, args.buffer_id), (0x30, u32::try_from(self.page_bytes)?), (0x34, pages), + (0x38, u32::try_from(geometry.capacity)?), (0x3c, args.blocks.len() as u32), + (0x54, pages - 1), (0x58, u32::try_from(geometry.block_bytes)?), + (0x7c, maximum), (0x80, normal), (self.control, args.blocks.len() as u32), + (self.control + 4, args.blocks.len() as u32), (self.control + 0x60, 1), + ] { put(out, at, value.to_le_bytes())?; } + for (at, value) in [ + (0x20, address(self.pages)?), (0x28, args.page_list_client), + (0x44, address(self.blocks)?), (0x4c, address(self.control)?), + (0x64, address(self.counter)?), (0x84, discard), + ] { put(out, at, value.to_le_bytes())?; } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index 54f5b2eab41f82..5c3eef8c54e22a 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -26,3 +26,12 @@ pub(crate) mod queue; pub(crate) mod compute; pub(crate) mod render_demand; + +pub(crate) mod render; +pub(crate) mod render_tail; + +pub(crate) mod render_registers; + +pub(crate) mod render_assembly; + +pub(crate) mod target; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs new file mode 100644 index 00000000000000..f53956102b610b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 tiling and fragment command encoders, with shared kick bindings. + +#![allow(dead_code)] + +use super::super::kick::{KickTimestamp, RegisterWrite, RegisterArrayBinding}; + +use super::render_tail::{self as tail, RenderStage, RenderTail}; + +pub(crate) const M5_TA_COMMAND_BYTES: usize = 0x990; +pub(crate) const M5_3D_COMMAND_BYTES: usize = 0x2240; +pub(crate) const M5_TA_SELECTOR: u32 = 0; +pub(crate) const M5_3D_SELECTOR: u32 = 1; + +pub(crate) const M5_TA_PROGRAM_OFFSET: u32 = 0x60; + +pub(crate) const M5_3D_PROGRAM_OFFSETS: [u32; 4] = [0xa0, 0x7c0, 0xee0, 0x1600]; + +const PROGRAM_AREA: usize = 0x700; +const REGISTER_BYTES: usize = 12; +pub(crate) const M5_PROGRAM_CAPACITY: usize = PROGRAM_AREA / REGISTER_BYTES; + +const TA_DISPATCH_REGISTER: u32 = 0x1c898; + +const FRAGMENT_TILE_REGISTER: u32 = 0x10038; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum M5RenderCommandError { + ProgramTooLong, + ProgramOffset, + RceCount, + MissingDispatchMirror, + AddressOverflow, + InvalidCompletionState, + InvalidUsageHint, + Kick, + SharedRing, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5ProgramRef<'a> { + pub registers: &'a [RegisterWrite], + + pub rce_count: u16, + pub command_offset: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5FragmentProgramRefs<'a> { + + pub main: M5ProgramRef<'a>, + + pub main_mid_count: u16, + pub store: M5ProgramRef<'a>, + pub resume: M5ProgramRef<'a>, + pub load: M5ProgramRef<'a>, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5StageCompletion { + + pub stamp_addresses: [u64; 2], + + pub stamp: u32, + + pub queue_index: u32, + + pub ordinal: u32, + + pub has_wait: bool, + + pub completion: u64, + + pub key: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct M5HardwareStamp { + pub stamp: u32, + + pub entries: u32, + + pub generation: u64, + + pub base: u64, +} + +pub(crate) fn m5_hardware_stamp(p: &M5HardwareStamp) -> Result { + if p.entries == 0 { + return Err(M5RenderCommandError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +pub(crate) fn m5_render_completion( + p: &M5HardwareStamp, + pass: u8, +) -> Result { + Ok(m5_hardware_stamp(p)? | u64::from(pass) << 40) +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaLinked3d { + + pub queue_index: u32, + + pub stamp: u32, + + pub flag: u8, + + pub completion: u64, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaPolicy { + pub policy_a: u32, + pub policy_b: u8, + pub policy_c: u32, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5ExtraAddresses { + + pub count: u32, + pub addresses: [u64; 4], + + pub policy: u8, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M5TaCommandFields { + + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pb_slot: u32, + + pub ta_word_a: u32, + + pub pb_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub ta_word_b: u32, + pub ta_word_e: u32, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: u64, + pub tpc_va: u64, + pub tpc_size: u64, + pub linked_3d: M5TaLinked3d, + pub ta_word_c: u64, + pub ta_word_d: u64, + pub deflake_state_va: u64, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub timestamp_va: u64, + + pub diagnostic_ta_timestamp_upper: u64, + pub ta_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub ta_word_f: u64, + /// The shifted usage hint: the seven bytes of the tail's descriptor value after + /// its first byte. The byte after them starts the resolved usage pointer. + pub shifted_usage_hint: u64, + + pub uma_state: u64, + + pub tail_flag: u8, + pub pb_growth_request: u8, + pub shader_write_class: u8, + pub policy: M5TaPolicy, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct M53dCommandFields { + pub sequence: u64, + pub completion: M5StageCompletion, + + pub pass: u8, + pub context_token: u64, + pub tail_descriptor: u64, + pub resolved_tail_value: Option, + pub fragment_word_l: u32, + + pub fragment_flag_a: bool, + + pub short_partial_programs: u8, + + pub tail_flag: u8, + + pub pm_state: u64, + + pub pm_scene: u64, + + pub scene_end_va: u64, + pub tilemap_va: u64, + pub msaa_control: u64, + pub fragment_word_a: u32, + pub sample_count: u32, + pub utile_grid: u32, + pub merge_bounds: [u64; 2], + pub fragment_word_b: u64, + pub tile_count: u32, + pub depth_bias_va: u64, + pub scissor_va: u64, + pub occlusion_query_va: u64, + pub fragment_word_c: u64, + + pub binding_pairs: [[u64; 2]; 23], + pub depth_dimensions: u64, + pub depth_va: u64, + pub depth_va_pair: [u64; 2], + pub stencil_va: u64, + pub stencil_va_pair: [u64; 2], + pub fragment_word_d: u64, + pub fragment_word_e: u64, + pub depth_stride: u64, + pub stencil_stride: u64, + pub depth_compression_va: u64, + pub depth_compression_stride: u64, + pub stencil_compression_va: u64, + pub stencil_compression_stride: u64, + pub sample_aux_flags: [u64; 2], + pub size_aux_control: [u64; 2], + pub eot_program: [u64; 2], + pub partial_eot_program: [u64; 2], + pub fragment_word_f: u64, + pub fragment_word_g: u32, + pub fragment_word_h: u64, + pub fragment_word_i: u32, + pub zs_clear_values: u64, + pub fragment_word_j: u64, + pub fragment_word_k: u32, + pub sample_size: u32, + pub fragment_pass_flag: u8, + pub fragment_word_n: u32, + pub fragment_flag_c: u8, + pub stage_flag_a: u8, + pub predecessor_id: u32, + + pub stage_word_a: u32, + pub stage_word_b: u32, + pub fragment_pair_a: [u64; 2], + + pub timestamp_counts: [u32; 2], + + pub fragment_word_m: u64, + + pub uma_state: u64, + + pub timestamp_va: [u64; 2], + + pub empty_tile_flags: [u8; 4], + pub fragment_flag_b: u8, + pub extra: M5ExtraAddresses, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct M5RenderCommandInputs { + + pub context_id: u32, + + pub scheduler_record: u64, + pub ta: M5TaCommandFields, + pub fragment: M53dCommandFields, +} + +impl Default for M5RenderCommandInputs { + fn default() -> Self { + + Self { + context_id: 0, + scheduler_record: 0, + ta: M5TaCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + fragment: M53dCommandFields { stage_word_b: u32::MAX, ..Default::default() }, + } + } +} + +fn put8(out: &mut [u8], offset: usize, value: u8) { + out[offset] = value; +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put128(out: &mut [u8], offset: usize, value: [u64; 2]) { + put64(out, offset, value[0]); + put64(out, offset + 8, value[1]); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn completion_tail(out: &mut [u8], base: usize, c: &M5StageCompletion) { + put64(out, base, c.stamp_addresses[0]); + put64(out, base + 8, c.stamp_addresses[1]); + put32(out, base + 0x10, c.stamp); + put32(out, base + 0x14, c.queue_index); + put32(out, base + 0x18, c.ordinal); + put32(out, base + 0x1c, u32::from(c.has_wait)); +} + +/// Fill publication-time completion values without replacing the retained +/// resource, lifecycle or register-program fields of either command. +pub(crate) fn link_completions( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + kicks: [KickTimestamp; 2], + pass: u8, + fragment_busy: bool, +) { + put32(ta, 0x7a4, u32::from(fragment_busy)); + put64(ta, 0x7a8, kicks[1].completion_value(pass) & 0xff_ffff_ffff); + put8(ta, 0x7b0, pass); + put8(ta, 0x7b1, pass); + put64(ta, 0x8c6, kicks[0].completion_value(pass)); + put8(fragment, 0x90, pass); + put64(fragment, 0x2160, kicks[1].completion_value(pass)); +} + +/// Bind the command-state slot in each engine's completion tail. +pub(crate) fn bind_work_slot( + ta: &mut [u8; M5_TA_COMMAND_BYTES], + fragment: &mut [u8; M5_3D_COMMAND_BYTES], + slot: u32, +) -> Result<(), M5RenderCommandError> { + if slot >= 4 { return Err(M5RenderCommandError::InvalidCompletionState); } + put32(ta, 0x8be, slot); + put32(fragment, 0x2158, slot); + Ok(()) +} + +fn extra_tail(out: &mut [u8], stage: RenderStage, e: &M5ExtraAddresses) { + tail::encode_extra(out, stage, &tail::ExtraAddresses { + count: e.count, policy: e.policy, addresses: e.addresses, + }).expect("fixed render command extent"); +} + +fn check_program(p: &M5ProgramRef<'_>, offset: u32) -> Result<(), M5RenderCommandError> { + if p.command_offset != offset { + return Err(M5RenderCommandError::ProgramOffset); + } + if p.registers.len() > M5_PROGRAM_CAPACITY { + return Err(M5RenderCommandError::ProgramTooLong); + } + if usize::from(p.rce_count) > p.registers.len() || p.rce_count > 0x7f { + return Err(M5RenderCommandError::RceCount); + } + Ok(()) +} + +fn place_program(out: &mut [u8], low_alias: u64, p: &M5ProgramRef<'_>) -> Result { + let offset = p.command_offset as usize; + let address = low_alias + .checked_add(u64::from(p.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + put16(out, offset + PROGRAM_AREA + 8, 0); + put64(out, offset + PROGRAM_AREA, address); + for (index, register) in p.registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + let header = (read32(out, entry) & 0xfffc0006) | (register.reg & 0x3fff9); + put32(out, entry, header); + put64(out, entry + 4, register.value); + } + put16(out, offset + PROGRAM_AREA + 8, p.registers.len() as u16); + Ok(address) +} + +/// The TA command's external-write class: whether the render's graphics +/// programs declare writes to memory, as the compiler reports it. +pub(crate) const fn shader_write_class(declared: bool) -> u8 { + // Forwarded as declared; the TA policy word is not inferred from it. + declared as u8 +} + +/// The shifted usage hint of a TA command, by programmed mesh class. The hint +/// covers only the seven bytes before the resolved usage pointer: what looks +/// like a high "pass rotation" bit in a wider read is that pointer's first +/// byte, not part of the hint. Other ordinary vertex programs have different +/// hints. +pub(crate) fn mesh_class_usage_hint(mesh_class: u8) -> u64 { + match mesh_class { + 0 => 0xc0c0, + 1 => 0x10ac0, + _ => 0x12b40, + } +} + +pub(crate) fn encode_m5_ta_command( + out: &mut [u8; M5_TA_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + ta: &M5ProgramRef<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + check_program(ta, M5_TA_PROGRAM_OFFSET)?; + let dispatch = ta + .registers + .iter() + .find(|r| r.reg & !1 == TA_DISPATCH_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + low_alias.checked_add(u64::from(ta.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + let t = &i.ta; + let descriptor_value = tail::tiling_descriptor_value(t.tail_descriptor, t.shifted_usage_hint) + .ok_or(M5RenderCommandError::InvalidUsageHint)?; + put32(out, 0, M5_TA_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Tiling, &RenderTail { + descriptor_value, resolved_value: Some(t.resolved_tail_value), + trailing_value: t.context_token, + }).expect("fixed TA command extent"); + put32(out, 0x8d6, t.ta_word_e); + put32(out, 0x48, t.ta_word_b); + put32(out, 0x18, t.pb_slot); + put32(out, 0x1c, t.ta_word_a); + put64(out, 0x20, t.pb_state); + put64(out, 0x28, t.pm_scene); + put64(out, 0x30, t.scene_end_va); + put64(out, 0x10, i.scheduler_record); + put64(out, 0x780, t.tpc_va); + put64(out, 0x788, t.tpc_size); + put32(out, 0x79c, t.linked_3d.queue_index); + put32(out, 0x7a0, t.linked_3d.stamp); + put32(out, 0x7a4, u32::from(t.linked_3d.flag & 1)); + put64(out, 0x7a8, t.linked_3d.completion & 0xff_ffff_ffff); + let pass = (t.linked_3d.completion >> 40) as u8; + put8(out, 0x7b1, pass); + put8(out, 0x7b0, pass); + put64(out, 0x7b6, t.ta_word_c); + put64(out, 0x7be, t.ta_word_d); + put64(out, 0x7d6, t.deflake_state_va); + put128(out, 0x7de, [0, 0]); + put64(out, 0x7ee, 0); + put32(out, 0x7b2, 0); + put32(out, 0x86a, u32::from(t.stage_flag_a & 1)); + put32(out, 0x86e, t.predecessor_id); + put32(out, 0x872, t.stage_word_a); + put32(out, 0x876, t.stage_word_b); + put128(out, 0x87a, [0, 0]); + put128(out, 0x90e, t.ta_pair_a); + put32(out, 0x91e, t.timestamp_counts[0]); + put32(out, 0x922, t.timestamp_counts[1]); + put64(out, 0x926, t.ta_word_f); + put64(out, 0x92e, t.uma_state); + put8(out, 0x936, t.tail_flag & 1); + put64(out, 0x8fe, t.timestamp_va); + put64(out, 0x906, t.diagnostic_ta_timestamp_upper); + put32(out, 0x892, u32::from(t.pb_growth_request & 1)); + put32(out, 0x89a, u32::from(t.shader_write_class & 1)); + put32(out, 0x89e, u32::from(m5_ta_policy_bit(&t.policy, t.shader_write_class))); + put32(out, 0x8a2, 0); + put64(out, 0x8f6, 0); + completion_tail(out, 0x8a6, &t.completion); + put64(out, 0x8c6, t.completion.completion); + put64(out, 0x8ce, t.completion.key); + + put32(out, 0x38, 0); + put16(out, 0x3c, 0); + put32(out, 0x40, 0); + let address = place_program(out, low_alias, ta)?; + put16(out, 0x38, ta.rce_count); + put16(out, 0x3a, ta.registers.len() as u16); + put16(out, 0x3c, ta.registers.len() as u16); + put64(out, 0x85e, dispatch); + extra_tail(out, RenderStage::Tiling, &t.extra); + let empty = None; + Ok([Some(RegisterArrayBinding::new(address, ta.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?), empty, empty, empty]) +} + +pub(crate) fn encode_m5_3d_command( + out: &mut [u8; M5_3D_COMMAND_BYTES], + low_alias: u64, + i: &M5RenderCommandInputs, + f: &M5FragmentProgramRefs<'_>, +) -> Result<[Option; 4], M5RenderCommandError> { + let programs = [f.main, f.store, f.resume, f.load]; + for (program, offset) in programs.iter().zip(M5_3D_PROGRAM_OFFSETS) { + check_program(program, offset)?; + low_alias.checked_add(u64::from(program.command_offset)) + .ok_or(M5RenderCommandError::AddressOverflow)?; + } + for partial in &programs[1..] { + if usize::from(partial.rce_count) != partial.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + } + if f.main_mid_count < f.main.rce_count || usize::from(f.main_mid_count) > f.main.registers.len() { + return Err(M5RenderCommandError::RceCount); + } + let tile = f + .main + .registers + .iter() + .find(|r| r.reg & !1 == FRAGMENT_TILE_REGISTER) + .ok_or(M5RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.fragment; + put32(out, 0, M5_3D_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + tail::encode(out, RenderStage::Fragment, &RenderTail { + descriptor_value: t.tail_descriptor, resolved_value: t.resolved_tail_value, + trailing_value: t.context_token, + }).expect("fixed fragment command extent"); + put32(out, 0x2170, t.fragment_word_l); + put32(out, 0x2174, u32::from(t.fragment_flag_a)); + put32(out, 0x2100, u32::from(t.short_partial_programs & 1)); + put32(out, 0x20f8, 0); + + put128(out, 0x2178, [0, 0]); + put64(out, 0x2188, 0); + put8(out, 0x21d0, t.tail_flag & 1); + put64(out, 0x20, i.scheduler_record); + put64(out, 0x28, t.pm_state); + put64(out, 0x30, t.pm_scene); + put64(out, 0x38, t.scene_end_va); + put32(out, 0x58, t.fragment_word_a); + put64(out, 0x40, t.tilemap_va); + put64(out, 0x48, t.msaa_control); + put32(out, 0x50, t.sample_count); + put32(out, 0x54, t.utile_grid); + put128(out, 0x60, t.merge_bounds); + put64(out, 0x70, t.fragment_word_b); + put32(out, 0x78, t.tile_count); + put64(out, 0x1d30, t.scissor_va); + put64(out, 0x1d20, t.depth_bias_va); + put64(out, 0x1d40, t.occlusion_query_va); + for (index, pair) in t.binding_pairs.iter().enumerate() { + put128(out, 0x1d58 + index * 16, *pair); + } + put64(out, 0x1ec8, t.depth_va); + put64(out, 0x1ed0, t.depth_stride); + put64(out, 0x1ed8, t.depth_compression_stride); + put64(out, 0x1f28, t.fragment_word_d); + put128(out, 0x1ee0, t.depth_va_pair); + put64(out, 0x1ef0, t.depth_compression_va); + put64(out, 0x1ef8, t.stencil_va); + put64(out, 0x1f00, t.stencil_stride); + put64(out, 0x1f08, t.stencil_compression_stride); + put64(out, 0x1f30, t.fragment_word_e); + put128(out, 0x1f10, t.stencil_va_pair); + put64(out, 0x1f20, t.stencil_compression_va); + put64(out, 0x1f68, t.fragment_word_f); + put32(out, 0x1f70, t.fragment_word_g); + put128(out, 0x1f48, t.size_aux_control); + put128(out, 0x1f74, t.eot_program); + put64(out, 0x1f88, t.fragment_word_h); + put32(out, 0x1f90, t.fragment_word_i); + put128(out, 0x1f94, t.partial_eot_program); + put64(out, 0x1fa8, t.zs_clear_values); + put32(out, 0x1fb0, t.sample_size); + put128(out, 0x1f38, t.sample_aux_flags); + put64(out, 0x1d50, t.fragment_word_c); + put64(out, 0x1fc0, t.fragment_word_j); + put32(out, 0x1fc8, t.fragment_word_k); + put64(out, 0x1fd0, t.depth_dimensions); + out[0x1fdc..0x20dc].fill(0); + put32(out, 0x20dc, 0); + put128(out, 0x20e0, [0, 0]); + put32(out, 0x1fd8, 0); + put32(out, 0x20f0, 0); + put8(out, 0x90, t.pass); + put32(out, 0x20fc, u32::from(t.fragment_pass_flag & 1)); + put32(out, 0x2209, t.fragment_word_n); + put8(out, 0x220d, t.fragment_flag_c); + put32(out, 0x2104, u32::from(t.stage_flag_a & 1)); + put32(out, 0x2108, t.predecessor_id); + put32(out, 0x210c, t.stage_word_a); + put32(out, 0x2110, t.stage_word_b); + put128(out, 0x2114, [0, 0]); + put128(out, 0x21a8, t.fragment_pair_a); + put32(out, 0x21b8, t.timestamp_counts[0]); + put32(out, 0x21bc, t.timestamp_counts[1]); + put64(out, 0x21c0, t.fragment_word_m); + put64(out, 0x21c8, t.uma_state); + put128(out, 0x2198, t.timestamp_va); + completion_tail(out, 0x2140, &t.completion); + put64(out, 0x2160, t.completion.completion); + put64(out, 0x2190, 0); + for (index, flag) in t.empty_tile_flags.iter().enumerate() { + put32(out, 0x2124 + index * 4, u32::from(flag & 1)); + } + put32(out, 0x2138, u32::from(t.fragment_flag_b & 1)); + put64(out, 0x2168, t.completion.key); + extra_tail(out, RenderStage::Fragment, &t.extra); + + put32(out, 0x80, 0); + put16(out, 0x84, 0); + put32(out, 0x88, 0); + let mut rce = [None; 4]; + for (slot, program) in rce.iter_mut().zip(programs.iter()) { + let address = place_program(out, low_alias, program)?; + *slot = Some(RegisterArrayBinding::new(address, program.rce_count as u8).map_err(|_| M5RenderCommandError::ProgramOffset)?); + } + put64(out, 0x1f58, tile); + put64(out, 0x1f60, 0); + put16(out, 0x80, f.main.rce_count); + put16(out, 0x82, f.main_mid_count); + put16(out, 0x84, f.main_mid_count); + put16(out, 0x88, f.main.registers.len() as u16); + Ok(rce) +} + +fn m5_ta_policy_bit(p: &M5TaPolicy, shader_write_class: u8) -> bool { + if p.policy_a == 0 || !(1..=2).contains(&p.policy_b) || p.policy_c == 3 { + return false; + } + match p.policy_a { + 1 | 4 => shader_write_class & 1 == 0, + 3 => true, + _ => false, + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs new file mode 100644 index 00000000000000..1966ab22f4878d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_assembly.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Assemble an M5 render pair from shared render resources. +use kernel::prelude::*; +use super::{render_registers as regs, render as cmd}; +use super::render_registers::{Workspace, M5RenderControls}; +use super::super::{render::RenderArgs, kick::{RegisterArrayBinding, RegisterWrite}}; + +pub(crate) const FRAGMENT_OFFSET: usize = 0x4000; +pub(crate) const COMMAND_STORAGE_BYTES: usize = 0x8000; + +pub(crate) struct EncodedRender { + pub(crate) rce: [[Option; 4]; 2], + pub(crate) register_counts: [u16; 5], + pub(crate) kick_flags: super::super::kick::StageKickFlags, +} + +fn commands(storage: &mut [u8]) -> Result<( + &mut [u8; cmd::M5_TA_COMMAND_BYTES], + &mut [u8; cmd::M5_3D_COMMAND_BYTES], +)> { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let (ta, fragment) = storage.split_at_mut(FRAGMENT_OFFSET); + Ok(((&mut ta[..cmd::M5_TA_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + (&mut fragment[..cmd::M5_3D_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?)) +} + +pub(crate) fn link_completions(storage: &mut [u8], + kicks: [super::super::kick::KickTimestamp; 2], pass: u8, fragment_busy: bool) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::link_completions(ta, fragment, kicks, pass, fragment_busy); + Ok(()) +} + +pub(crate) fn bind_work_slot(storage: &mut [u8], slot: u32) -> Result { + let (ta, fragment) = commands(storage)?; + cmd::bind_work_slot(ta, fragment, slot).map_err(|_| EINVAL) +} + +#[inline(never)] +pub(crate) fn encode_into(storage: &mut [u8], low: u64, + args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: regs::TaExtra, dispatch: regs::FragmentDispatch, + inputs: &cmd::M5RenderCommandInputs, workspace: &mut Workspace) + -> Result { + if storage.len() < COMMAND_STORAGE_BYTES { return Err(EINVAL); } + let fragment_low = low.checked_add(FRAGMENT_OFFSET as u64).ok_or(EINVAL)?; + fragment_low.checked_add(cmd::M5_3D_COMMAND_BYTES as u64).ok_or(EINVAL)?; + if inputs.context_id != u32::from(args.resources.context_id) { return Err(EINVAL); } + workspace.write(args, controls, extra, dispatch, inputs)?; + let short_partial = [RegisterWrite::new(0x160a8, 0x20), RegisterWrite::new(0x16068, 0x4_0000)]; + let (store, resume, load): (&[_], &[_], &[_]) = if inputs.fragment.short_partial_programs & 1 != 0 { + (&short_partial, &short_partial, &short_partial) + } else { (&workspace.store, &workspace.resume, &workspace.load) }; + let partial_counts = [store.len() as u16, resume.len() as u16, load.len() as u16]; + let count = workspace.ta.registers().len() as u16; + let ta_program = cmd::M5ProgramRef { registers: workspace.ta.registers(), + rce_count: count - 3, command_offset: cmd::M5_TA_PROGRAM_OFFSET }; + let view = |registers, rce_count, command_offset| cmd::M5ProgramRef { + registers, rce_count, command_offset }; + let fragment_programs = cmd::M5FragmentProgramRefs { + main: view(&workspace.fragment[..workspace.fragment_count], + workspace.fragment_count as u16 - 4, 0xa0), + main_mid_count: workspace.fragment_count as u16 - 2, + store: view(store, partial_counts[0], 0x7c0), + resume: view(resume, partial_counts[1], 0xee0), + load: view(load, partial_counts[2], 0x1600), + }; + let ta = &mut workspace.encoded_ta; + let fragment = &mut workspace.encoded_fragment; + ta.fill(0); fragment.fill(0); + let fragment_rce = cmd::encode_m5_3d_command(fragment, fragment_low, inputs, &fragment_programs).map_err(|_| EINVAL)?; + let ta_rce = cmd::encode_m5_ta_command(ta, low, inputs, &ta_program).map_err(|_| EINVAL)?; + let (ta_storage, fragment_storage) = storage.split_at_mut(FRAGMENT_OFFSET); + ta_storage[..cmd::M5_TA_COMMAND_BYTES].copy_from_slice(ta); + fragment_storage[..cmd::M5_3D_COMMAND_BYTES].copy_from_slice(fragment); + // A kick sets header bit 61 for a stage whose register program has the + // SCS bracket and bit 63 for its matrix bracket, from the same flags that + // gate those brackets. The fragment matrix bracket is never emitted here. + let kick_flags = super::super::kick::StageKickFlags { + scratch: [extra.scs_bracket & 1 != 0 || controls.ta.scs.scratch.enabled(), + controls.fragment.scs.scratch.enabled()], + matrix: [extra.matrix_bracket & 1 != 0, false], + }; + Ok(EncodedRender { rce: [ta_rce, fragment_rce], + register_counts: [count, workspace.fragment_count as u16, partial_counts[0], partial_counts[1], partial_counts[2]], + kick_flags }) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs new file mode 100644 index 00000000000000..1400a687bbf593 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 register adaptation over the shared render-pass/resource lowering. +#![allow(dead_code)] +use kernel::prelude::*; +use super::super::{kick::RegisterWrite, render::{RenderArgs, RenderPass, M5RegisterPrograms as RegisterPrograms}}; +use super::render::M5RenderCommandInputs; +const DEFLAKE_STATE_OFFSET: u64 = 0x980; +const DEFLAKE_AUX_OFFSET: u64 = 0xc0; + +fn utile_config(p: &RenderPass) -> u64 { + (u64::from(p.utile_width / 16) << 12) | (u64::from(p.utile_height / 16) << 14) + | u64::from(p.samples_log2) +} +#[derive(Copy, Clone)] +pub(crate) struct M5RenderStageControls { + pub(crate) scs: crate::g17::render_resources::StageScratch, + + pub(crate) resource_flags: u64, + + pub(crate) resource_control: u64, + + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Copy, Clone)] +pub(crate) struct M5RenderControls { + /// Encoded event-command state for the initial fragment register list. + pub(crate) fragment_event_control: u64, + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + + pub(crate) fragment_tile_flags: u64, + + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_control: u64, +} + +struct RegisterWriter<'a> { + values: &'a mut [RegisterWrite], + count: usize, +} + +impl RegisterWriter<'_> { + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.values.get_mut(self.count) + .ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.count += 1; + Ok(()) + } + fn finish(self) -> Result { + if self.count != self.values.len() { + return Err(EINVAL); + } + Ok(()) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.reg == number) + .map(|r| r.value) + .ok_or(EINVAL) +} + +/// The event-command state of the initial fragment register list: 0 when +/// disabled, 3 for kind 1, otherwise 1. +pub(crate) const fn fragment_event_control(enabled: bool, kind: u32) -> u64 { + if !enabled { 0 } else if kind == 1 { 3 } else { 1 } +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &RenderPass, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(EINVAL); + } + + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(EINVAL); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((utile_config(p) & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 82], + common: &[RegisterWrite; 73], +) -> Result { + if controls.ta_dispatch_control & 1 != 0 { + return Err(EINVAL); + } + + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + for mut register in common.iter().copied() { + match register.reg { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + // A fixed word of the TA program. Sample locations are bound by + // their own register. + 0x10131 => register.value = 0x88, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(EINVAL); + } + register.value |= controls.ta_dispatch_control; + } + 0x101c9 => register.value = controls.ta.scs.scratch.size_word(), + 0xd471 => register.value = u64::from(controls.ta.scs.auxiliary_allocation), + 0x1a0f1 => register.value = controls.ta.scs.scratch.stack_word(), + 0x1a099 => register.value = controls.ta.scs.intersection.words()[0], + 0x1a0a1 => register.value = controls.ta.scs.intersection.words()[1], + 0x1a069 => register.value = controls.ta.scs.intersection.words()[2], + 0x1a071 => register.value = controls.ta.scs.intersection.words()[3], + 0x1a0c9 => register.value = controls.ta.scs.intersection.words()[4], + 0x1a0d1 => register.value = controls.ta.scs.intersection.words()[5], + 0x10799 => { + register.value = controls.ta.scs.scratch.window_word() + | (register.value & (1 << 25)); + out.push(0x1a149, controls.ta.scs.scratch.driver_word())?; + }, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers_into( + p: &RenderPass, + controls: &M5RenderControls, + free_list_slot: u32, + out: &mut [RegisterWrite; 90], + common: &[RegisterWrite; 89], +) -> Result { + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterWriter { values: out, count: 0 }; + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common.iter().copied() { + match register.reg { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.reg = 0x16469, + 0x100d9 => register.reg = 0x16471, + 0x100b8 => register.reg = 0x16138, + 0x10791 => { + let scratch = controls.fragment.scs.scratch; + register.value = if scratch.enabled() { + (controls.fragment_raster_control | (1 << 24)) & 0x200001000204 + | (scratch.window_word() & 0xff0000) + | (controls.fragment_raster_control & (1 << 25)) + } else { controls.fragment_raster_control }; + }, + 0x101c1 => register.value = controls.fragment.scs.scratch.size_word(), + 0xd469 => register.value = u64::from(controls.fragment.scs.auxiliary_allocation), + 0x1a0f9 => register.value = controls.fragment.scs.scratch.stack_word(), + 0x1a0a9 => register.value = controls.fragment.scs.intersection.words()[0], + 0x1a0b1 => register.value = controls.fragment.scs.intersection.words()[1], + 0x1a079 => register.value = controls.fragment.scs.intersection.words()[2], + 0x1a081 => register.value = controls.fragment.scs.intersection.words()[3], + 0x1a0d9 => register.value = controls.fragment.scs.intersection.words()[4], + 0x1a0e1 => register.value = controls.fragment.scs.intersection.words()[5], + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(EINVAL); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, controls.fragment.scs.scratch.driver_word())?, + 0x10211 => { + out.push(0x0a741, controls.fragment_event_control)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = free_list_slot as i32 as i64 as u64, + _ => {} + } + out.push(register.reg, register.value)?; + if register.reg == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.reg == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(EINVAL); + } + out.finish() +} + +#[derive(Copy, Clone)] +pub(crate) struct FragmentDispatch { + pub(crate) words: [u64; 2], +} + +impl FragmentDispatch { + + pub(crate) fn initial(self, descriptor_flag: u8) -> u64 { + if descriptor_flag & 1 != 0 { return 0x20; } + ((self.words[0] >> 27) & 0x1f_ffff_f860) + | (self.words[1].wrapping_shl(2) & 0x3_ffff_f87c) + | ((self.words[1] >> 32) << 7) + } +} + +#[derive(Copy, Clone)] +pub(crate) struct TaExtra { + pub(crate) gtp1: u64, + pub(crate) gtp2: u64, + pub(crate) gtp4: u64, + pub(crate) gtp5: u64, + pub(crate) gtp6: u64, + pub(crate) gtp7: u64, + pub(crate) segment_control: u64, + pub(crate) segment_index: u64, + pub(crate) min_free_tvb_pages: u64, + pub(crate) scs_bracket: u8, + pub(crate) matrix_bracket: u8, +} + +pub(crate) struct TaProgram { + registers: [RegisterWrite; 97], + length: usize, +} + +impl TaProgram { + pub(crate) const fn empty() -> Self { + Self { registers: [RegisterWrite {reg: 0, value: 0}; 97], length: 0 } + } + pub(crate) fn registers(&self) -> &[RegisterWrite] { &self.registers[..self.length] } + pub(crate) fn set_lifecycle(&mut self, value: u64) { + for r in &mut self.registers[..self.length] { + if matches!(r.reg, 0x1ca10 | 0x14a1 | 0xa349) { + r.value = value; + } + } + } + fn push(&mut self, number: u32, value: u64) -> Result { + let slot = self.registers.get_mut(self.length).ok_or(EINVAL)?; + *slot = RegisterWrite { reg: number, value }; + self.length += 1; + Ok(()) + } +} + +pub(crate) fn tiling_with_scratch_into(common: &[RegisterWrite], e: TaExtra, + mesh_storage: u64, out: &mut TaProgram) -> Result { + out.length = 0; + if e.scs_bracket & 1 != 0 { out.push(0x17e1, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17f1, 1)?; } + let mut deflake = false; + for r in common.iter().copied() { + if r.reg == 0x1748 { continue; } + if r.reg == 0x1740 { + if e.scs_bracket & 1 != 0 { out.push(0x17d9, 1)?; } + if e.matrix_bracket & 1 != 0 { out.push(0x17e9, 1)?; } + continue; + } + out.push(r.reg, r.value)?; + let register = r.reg; + match register { + // In a mesh-object TA program the mesh rendering buffer register + // follows the scratch-window register. + 0x10799 if mesh_storage != 0 => out.push(0x1ca48, mesh_storage)?, + 0x1c9c8 => { out.push(0x1c041, e.gtp1)?; out.push(0x1c9d0, e.gtp1)?; } + 0x1c880 if !deflake => { + deflake = true; + for (n,v) in [(0x1c948,e.gtp4),(0x1c888,e.gtp5), + (0x1c890,e.segment_control),(0x1c918,e.segment_index)] { out.push(n,v)?; } + } + 0x1c9d8 => { + for (n,v) in [(0x1c089,e.gtp2),(0x1c9e0,e.gtp2), + (0x16c41,e.gtp6 & !0x7f),(0x1ca40,e.gtp6 & !0x7f), + (0x1c9a8,e.min_free_tvb_pages),(0x1c920,e.gtp7 & !0x3f)] { out.push(n,v)?; } + } + _ => {} + } + } + if out.length != 92 + usize::from(e.scs_bracket & 1) * 2 + usize::from(e.matrix_bracket & 1) * 2 + + usize::from(mesh_storage != 0) { + return Err(EINVAL); + } + Ok(()) +} + +pub(crate) fn fragment_main_with_scratch_into(p: &RenderPass, common: &[RegisterWrite], + dispatch: FragmentDispatch, descriptor_flag: u8, scs: bool, out: &mut [RegisterWrite]) -> Result { + + let mut n = 0; + let mut tile = 0; + for mut r in common.iter().copied() { + + if r.reg == 0x10051 { + r.value = isp_sample_control(p); + } + if r.reg == 0x16068 { + if tile == 2 { return Err(EINVAL); } + out[n] = RegisterWrite { reg: 0x160a8, + value: if tile == 0 { dispatch.initial(descriptor_flag) } else { 0x20 } }; + n += 1; + tile += 1; + } + let slot = out.get_mut(n).ok_or(EINVAL)?; + *slot = r; + n += 1; + // The SCS controls bracket the fragment main list after entry and exit. + if scs && matches!(r.reg, 0x1739 | 0x1731) { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite { + reg: if r.reg == 0x1739 { 0x17e1 } else { 0x17d9 }, value: 1 }; + n += 1; + } + } + if n != out.len() || tile != 2 { return Err(EINVAL); } + Ok(()) +} + +/// ISP sample allocation, in the M5 register's units. +pub(crate) fn isp_sample_control(p: &RenderPass) -> u64 { + let samples = 1u32 << p.samples_log2; + if samples <= 1 { return 0; } + let sample_bytes = (u32::from(p.sample_size).max(8) + 7) & !7; + let groups = sample_bytes.div_ceil(32); + let units = (u32::from(p.utile_width) >> 3) * (u32::from(p.utile_height) >> 2) * groups; + let value = match samples { 2 => units.max(8) >> 3, 4 => units >> 1, 8 => units, _ => 0 }; + let value = if samples == 2 && p.utile_width == 16 && p.utile_height == 16 { value } else { value >> 1 }; + u64::from(value & 0x1f) << 16 +} + +fn partial(common: &[RegisterWrite], insert_after: u32, + c: &M5RenderControls, load: bool, out: &mut [RegisterWrite; N]) -> Result { + let mut w = RegisterWriter { values: out, count: 0 }; + for r in common.iter().copied() { + if r.reg == 0x15359 { continue; } + w.push(r.reg, r.value | if r.reg == 0x10039 { c.partial_tile_flags } else { 0 })?; + if r.reg == insert_after { + w.push(0x153b1, if load { c.store_pipeline_2_bind } else { c.partial_store_pipeline_2_bind })?; + w.push(0x153b9, if load { c.store_pipeline_2 } else { c.partial_store_pipeline_2 })?; + } + } + w.finish() +} + +/// Caller-owned assembly scratch. All resource addresses come from RenderArgs. +pub(crate) struct Workspace { + common: RegisterPrograms, + base: [RegisterWrite; 90], + pub(crate) ta: TaProgram, + pub(crate) fragment: [RegisterWrite; 94], + pub(crate) fragment_count: usize, + pub(crate) store: [RegisterWrite; 17], + pub(crate) resume: [RegisterWrite; 24], + pub(crate) load: [RegisterWrite; 12], + // Assemble in normal cached memory. Firmware command mappings on M5 + // are write-combined and must not be used for header read/modify/write. + pub(crate) encoded_ta: [u8; super::render::M5_TA_COMMAND_BYTES], + pub(crate) encoded_fragment: [u8; super::render::M5_3D_COMMAND_BYTES], +} +// SAFETY: Every field consists only of integer storage. +unsafe impl Zeroable for Workspace {} +impl Workspace { + #[inline(never)] + pub(crate) fn write(&mut self, args: &RenderArgs<'_>, controls: &M5RenderControls, + extra: TaExtra, dispatch: FragmentDispatch, inputs: &M5RenderCommandInputs) -> Result { + self.common.write_m5(args)?; + let r = args.resources; + let p = args.pass; + // The TA's two deflake sub-buffers sit at fixed offsets from the + // deflake base (DEFLAKE_STATE_OFFSET, DEFLAKE_AUX_OFFSET). + let base = r.deflake_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?; + let mut vdm_seen = false; + for reg in &mut self.common.ta { + let register = reg.reg; + reg.value = match register { + 0x1c880 if vdm_seen => base.checked_add(DEFLAKE_STATE_OFFSET).ok_or(EINVAL)?, + 0x10111 | 0x1c9b0 => base.checked_add(DEFLAKE_STATE_OFFSET).ok_or(EINVAL)?, + 0x10119 | 0x1c9b8 => base.checked_add(DEFLAKE_AUX_OFFSET).ok_or(EINVAL)?, + 0x1c079 | 0x1c9d8 if p.layers == 1 => r.heapmeta_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?, + _ => reg.value, + }; + if reg.reg == 0x1c880 { vdm_seen = true; } + } + for reg in &mut self.common.fragment { + let register = reg.reg; + reg.value = match register { + 0x16060 if p.layers == 1 => r.heapmeta_va, + 0x15021 => controls.aux_fb_flags, + 0x15049 => controls.aux_fb_control, + // Main tile-control bits 17..19 carry the empty-tile and + // depth/stencil attachment action flags. The ordinary M5 + // profile has one stored colour attachment. An unstored ZLS + // aspect or a pass that skips empty + // tiles needs its corresponding action bit. Partial phases + // clear these bits independently and retain their own policy. + 0x10039 => { + let skip_empty = !p.process_empty_tiles; + reg.value + | (u64::from(skip_empty && inputs.fragment.empty_tile_flags[3] & 1 == 0) << 17) + | (u64::from(p.depth.va != 0 + && (p.zls_control & (1 << 15) != 0 || p.zls_control & (1 << 19) == 0)) << 18) + | (u64::from(p.stencil.va != 0 + && (p.zls_control & (1 << 14) != 0 || p.zls_control & (1 << 18) == 0)) << 19) + } + _ => reg.value, + }; + } + let mut c = *controls; + c.ta.work_key = super::super::work_key(r.context_id, args.ta.state_word); + c.fragment.work_key = super::super::work_key(r.context_id, args.fragment.state_word); + let ta: &mut [RegisterWrite;82] = (&mut self.base[..82]).try_into().map_err(|_| EINVAL)?; + build_ta_registers_into(p, &c, r.free_list_slot, ta, &self.common.ta)?; + let mut extra = extra; + extra.scs_bracket |= u8::from(c.ta.scs.scratch.enabled()); + tiling_with_scratch_into(ta, extra, r.mesh_storage, &mut self.ta)?; + self.ta.set_lifecycle((u64::from(inputs.ta.predecessor_id) << 32) | u64::from(inputs.ta.completion.key as u32)); + c.fragment_tile_flags = (c.fragment_tile_flags & !0x7ffff) | 0x36011 + | (u64::from(inputs.fragment.empty_tile_flags[3] & 1) << 18); + build_fragment_registers_into(p, &c, r.free_list_slot, &mut self.base, &self.common.fragment)?; + self.fragment_count = 92 + usize::from(c.fragment.scs.scratch.enabled()) * 2; + fragment_main_with_scratch_into(p, &self.base, dispatch, inputs.fragment.empty_tile_flags[3], + c.fragment.scs.scratch.enabled(), &mut self.fragment[..self.fragment_count])?; + for reg in &mut self.fragment { + if matches!(reg.reg, 0x160e0 | 0x1499 | 0xa341) { + reg.value = (u64::from(inputs.fragment.predecessor_id) << 32) | u64::from(inputs.fragment.completion.key as u32); + } + } + partial(&self.common.store, 0x15381, &c, false, &mut self.store)?; + // The ordinary SPM store takes its own empty-tile flag, independent + // of main empty-tile processing and of the SPM load's flag. Stored + // ZLS surfaces still force empty-tile processing in the store. + for register in &mut self.store { + if register.reg == 0x10039 { + let empty = inputs.fragment.empty_tile_flags[1] & 1 != 0 + || p.depth.va != 0 || p.stencil.va != 0; + register.value = (register.value & !(1 << 16)) + | (u64::from(empty) << 16); + } + } + partial(&self.common.resume, 0x15381, &c, false, &mut self.resume)?; + // The load phase uses the primary normal store binding. + // Normal EOT values are in the main array, before any adaptation. + c.store_pipeline_2_bind = self.common.fragment[2].value << 32; + c.store_pipeline_2 = self.common.fragment[3].value; + partial(&self.common.load, 0x15309, &c, true, &mut self.load)?; + // The SPM load phase takes its empty-tile policy from its own flag, + // which the fragment command also carries. It differs from the main + // and partial-store policy for multisample resolve passes. + for register in &mut self.load { + if register.reg == 0x10039 { + register.value = (register.value & !(1 << 16)) + | (u64::from(inputs.fragment.empty_tile_flags[2] & 1) << 16); + } + } + Ok(()) + } +} + +/// Populate command-side copies from the same pass that produces the registers. +/// Queue identity, completion sequencing and resource budgets remain caller-owned. +pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, + i: &mut M5RenderCommandInputs) -> Result { + let p=args.pass;let r=args.resources; + if p.width==0 || p.width>32768 || p.height==0 || p.height>32768 + || p.layers==0 || p.layers>2048 || r.context_id==0 || r.context_id>63 || p.samples_log2>3 + || !matches!((p.utile_width,p.utile_height),(32,32)|(32,16)|(16,16)) { return Err(EINVAL); } + let usc=|v:u32|p.usc_base.checked_add(u64::from(v&!7)).ok_or(EINVAL); + let spec=|v:u64|if p.rsrc_spec_hi {v}else{v&u64::from(u32::MAX)}; + // Check every address before changing command inputs. + let bg=usc(p.bg.offset)?;let pbg=usc(p.partial_bg.offset)?; + let eot=usc(p.eot_offset)?;let peot=usc(p.partial_eot_offset)?; + let deflake=r.deflake_va.checked_sub(0x10_0000_0000) + .and_then(|offset|offset.checked_add(DEFLAKE_STATE_OFFSET)).ok_or(EINVAL)?; + i.context_id=u32::from(r.context_id); + i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; + let f=&mut i.fragment; + f.empty_tile_flags[0]=u8::from(p.process_empty_tiles); + // The admitted ordinary single-sample profile has one stored colour + // attachment. Its clear-intent flag follows empty-tile intent; a LOAD + // pass uses zero here, while partial-store ZLS addresses can + // independently force empty tiles in the register program. + if p.samples_log2 == 0 { + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles); + f.empty_tile_flags[2] = 0; + } else { + // Multisample stored-colour CLEAR and LOAD passes both process empty + // tiles in the SPM load. Only CLEAR sets the store flag; the BG + // resource specification publishes that intent in bit 6. An empty + // seed pass has no BG or empty-tile intent. + f.empty_tile_flags[1] = u8::from(p.process_empty_tiles && p.bg.rsrc_spec & (1 << 6) != 0); + f.empty_tile_flags[2] = u8::from(p.process_empty_tiles); + } + f.depth_bias_va=p.depth_bias_va;f.scissor_va=p.scissor_va;f.occlusion_query_va=p.occlusion_query_va; + f.binding_pairs[18]=[spec(p.bg.rsrc_spec),bg]; + f.binding_pairs[21]=[spec(p.partial_bg.rsrc_spec),pbg];f.binding_pairs[22][0]=p.zls_control; + f.depth_dimensions=p.depth_dimensions;f.depth_va=p.depth.va;f.depth_va_pair=[p.depth.va;2]; + f.stencil_va=p.stencil.va;f.stencil_va_pair=[p.stencil.va;2]; + f.depth_stride=p.depth.stride;f.stencil_stride=p.stencil.stride; + f.depth_compression_va=p.depth.compression_va;f.depth_compression_stride=p.depth.compression_stride; + f.stencil_compression_va=p.stencil.compression_va;f.stencil_compression_stride=p.stencil.compression_stride; + f.sample_aux_flags=[isp_sample_control(p),c.aux_fb_flags]; + f.size_aux_control=[u64::from(p.width)|(u64::from(p.height)<<32),c.aux_fb_control]; + f.eot_program=[spec(p.eot_rsrc_spec)<<32,eot]; + f.partial_eot_program=[spec(p.partial_eot_rsrc_spec)<<32,peot]; + f.zs_clear_values=u64::from(p.depth_clear)|(u64::from(p.stencil_clear|0x300)<<32); + f.tilemap_va=r.tilemap_va;f.msaa_control=p.msaa_control;f.sample_count=1u32< Option { + if context >= 64 || generation == 0 || generation >= (1u64 << 58) { + return None; + } + Some((u64::from(context) << 58) | generation) +} + +/// Merge the shifted usage hint into the TA tail's descriptor value. +/// +/// The value is eight bytes. Its first byte belongs to the descriptor; the +/// shifted hint fits only the following seven bytes. The next byte starts the +/// independent resolved usage pointer, so accepting an eighth hint byte would +/// overwrite that pointer. +pub(crate) fn tiling_descriptor_value(descriptor: u64, shifted_hint: u64) -> Option { + if shifted_hint > (u64::MAX >> 8) { + return None; + } + Some((descriptor & 0xff) | (shifted_hint << 8)) +} + +pub(crate) struct RenderTail { + + pub(crate) descriptor_value: u64, + + pub(crate) resolved_value: Option, + + pub(crate) trailing_value: u64, +} + +pub(crate) fn encode(out: &mut [u8], stage: RenderStage, value: &RenderTail) -> Option<()> { + let base = match stage { RenderStage::Tiling => 0x937, RenderStage::Fragment => 0x21d1 }; + let tail = out.get_mut(base..base + 24)?; + tail[..8].copy_from_slice(&value.descriptor_value.to_le_bytes()); + if let Some(resolved) = value.resolved_value { + tail[8..16].copy_from_slice(&resolved.to_le_bytes()); + } + tail[16..24].copy_from_slice(&value.trailing_value.to_le_bytes()); + Some(()) +} + +pub(crate) struct ExtraAddresses { + pub(crate) count: u32, + pub(crate) policy: u8, + pub(crate) addresses: [u64; 4], +} + +pub(crate) fn encode_extra(out: &mut [u8], stage: RenderStage, value: &ExtraAddresses) -> Option<()> { + let (base, end) = match stage { + RenderStage::Tiling => (0x96f, 0x990), + RenderStage::Fragment => (0x220e, 0x2233), + }; + let tail = out.get_mut(base..end)?; + tail[32] &= 0xf2; + if value.count != 0 || value.addresses.iter().any(|&address| address != 0) { + tail[32] |= 1 | ((value.policy & 3) << 2); + for (slot, address) in tail[..32].chunks_exact_mut(8).zip(value.addresses) { + slot.copy_from_slice(&address.to_le_bytes()); + } + } + if let RenderStage::Fragment = stage { tail[33..37].fill(0); } + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs new file mode 100644 index 00000000000000..2dc50a4eaabecf --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render-target suballocation geometry. + + +pub(crate) fn cluster_control(clusters: u32, pb_pages: u64) -> Option<[u64; 3]> { + let difference = match clusters { + 1 => return Some([1, 0, pb_pages]), + 2 => 4u64, + 4 => 5u64, + 8 => 6u64, + _ => return None, + }; + Some([0x40 | (difference << 8) | (difference << 19), 4, pb_pages]) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TargetSlices { + pub(crate) sizes: [usize; 8], + pub(crate) alignments: [usize; 8], + pub(crate) offsets: [usize; 8], + pub(crate) bytes: usize, +} + +impl TargetSlices { + pub(crate) fn new(tilemap_bytes: usize, tpc_used_bytes: usize, layers: u16, + kind: u32, segments: u32, pb_bytes: usize) -> Option { + if tilemap_bytes == 0 || tpc_used_bytes == 0 || layers == 0 || kind == 0 { + return None; + } + let mut sizes = [0; 8]; + let mut alignments = [0; 8]; + sizes[0] = tpc_used_bytes; + alignments[0] = 0x40; + if kind >= 2 { + if segments == 0 || pb_bytes == 0 { return None; } + let segments = segments as usize; + sizes[1] = tilemap_bytes.checked_mul(segments)?; + sizes[2] = if layers > 1 { 0x100usize.checked_mul(segments)? } else { 0 }; + sizes[4] = (kind as usize).checked_mul(0x300)?.checked_add(0x680)?; + sizes[5] = (kind as usize).checked_mul(0x5c0)?; + + let pages = pb_bytes.checked_add(0x7fff)? / 0x8000; + let groups = (pages.checked_add(0x17ff)? / 0x1800).checked_add(1)?; + sizes[6] = groups.checked_mul(8)?.checked_mul(segments)?; + sizes[7] = 0x80; + alignments = [0x40, 0x40, 0x100, 0, 0x40, 0x40, 0x80, 0x40]; + } + let mut bytes = 0usize; + let mut offsets = [0; 8]; + for i in 0..8 { + if alignments[i] != 0 { + bytes = bytes.checked_add(alignments[i] - 1)? & !(alignments[i] - 1); + } + offsets[i] = bytes; + bytes = bytes.checked_add(sizes[i])?; + } + Some(Self { sizes, alignments, offsets, bytes }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs index e1e6a10949fb44..e8349d0fcd0e2f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -113,7 +113,8 @@ impl Support { } } -/// The viewport words consumed from the deflake page at +0x900. +/// Viewport words shared by the tiling engines; their location and control +/// value are selected by the firmware profile. #[repr(C)] #[derive(Copy, Clone)] pub(crate) struct Viewport { @@ -129,6 +130,11 @@ pub(crate) struct Viewport { scale_z: u32, } impl Viewport { + pub(crate) fn with_control(mut self, control: u32) -> Self { + self.unk_0 = control; + self + } + /// Refresh only defined words; firmware-owned gaps retain their contents. pub(crate) fn apply(&self, destination: &mut Self) { destination.unk_0 = self.unk_0; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2b231de157a71b..c226388b4dd288 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -2,6 +2,7 @@ //! Physical paired tiling and fragment queues and their retained command storage. +pub(in crate::g17) mod generation; mod observation; pub(crate) use observation::Observed; mod dependencies; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs new file mode 100644 index 00000000000000..e067bcb85a3a9b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command tiling, fragment-status and auxiliary storage. Mappings precede +//! their backing owners; a generation is reusable only after paired retirement. + +use crate::g17::{ + buffer::{self, BlockGeometry}, + fw::render::{storage, RenderPass}, + object::{Allocator, CpuMap, KernelObject, Placement}, +}; +use crate::{hw::t8140::{dynamic, resources as cfg}, mmu}; +use kernel::prelude::*; + +/// Offsets within a single client allocation, not fixed GPU addresses. +#[derive(Copy, Clone)] +pub(in crate::g17) struct Layout { + pub(in crate::g17) bytes: usize, + pub(in crate::g17) deflake: usize, + pub(in crate::g17) status: usize, + pub(in crate::g17) auxiliary: usize, + pub(in crate::g17) viewport: usize, + pub(in crate::g17) viewport_control: u32, +} +impl Layout { + pub(in crate::g17) const NEO: Self = Self { + bytes: storage::STATE_SIZE, deflake: storage::DEFLAKE, + status: storage::TA_STATUS, auxiliary: storage::AUXILIARY, + viewport: 0x900, viewport_control: 0xc00, + }; + pub(in crate::g17) const M5: Self = Self { + bytes: 0x10000, deflake: 0, status: 0x4000, auxiliary: 0x8000, + viewport: 0x9c0, viewport_control: 0x3000, + }; + + pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { + if state.len() < self.bytes { return Err(EINVAL); } + let viewport = storage::Viewport::new(pass.width, pass.height)? + .with_control(self.viewport_control); + let offset = self.deflake + self.viewport; + let view = state.get_mut(offset..offset + size_of::()) + .ok_or(EINVAL)?.as_mut_ptr().cast::(); + if view.align_offset(align_of::()) != 0 { return Err(EINVAL); } + // SAFETY: The complete integer-only view is aligned and in bounds. + // The caller exclusively owns this unpublished command generation. + unsafe { viewport.apply(&mut *view) }; + state[self.auxiliary..self.auxiliary + storage::AUXILIARY_SIZE].fill(0); + Ok(()) + } +} + +enum Views { + Neo { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + fragment: KernelObject, + }, + M5 { status_fw: mmu::KernelMapping }, +} + +pub(in crate::g17) struct Generation { + views: Views, + state: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + let (range, layout) = if alloc.uat.is_m5() { + (BlockGeometry::M5_COMPACT, Layout::M5) + } else { (dynamic::CLIENT_LOWER, Layout::NEO) }; + KernelObject::new(alloc.dev, vm, Placement::In(range, cfg::TVB_PAGE), layout.bytes, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined) + } + + #[inline(never)] + pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { + let views = if alloc.uat.is_m5() { + let layout = Layout::M5; + let status_fw = state.map_range(alloc.uat.kernel_vm(), + layout.status..layout.status + mmu::UAT_PGSZ, + alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, 0)?; + Views::M5 { status_fw } + } else { + let deflake = buffer::compact_alias(&state, vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)?; + let auxiliary = state.map_range(vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + cfg::AUXILIARY, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, 0)?; + let fragment = alloc.kernel(mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?; + let fragment_client = buffer::compact_alias(&fragment, vm, 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + Views::Neo { deflake, auxiliary, fragment_client, fragment } + }; + Ok(KBox::new(Self { views, state, ordinal: None }, GFP_KERNEL)?) + } + + pub(in crate::g17) fn prepare(&mut self, ordinal: u64, pass: &RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } + let layout = match &mut self.views { + Views::Neo { fragment, .. } => { + fragment.bytes_mut().fill(0); + Layout::NEO + } + Views::M5 { .. } => { + let layout = Layout::M5; + self.state.bytes_mut()[layout.status..layout.status + mmu::UAT_PGSZ].fill(0); + layout + } + }; + layout.prepare(self.state.bytes_mut(), pass)?; + self.ordinal = Some(ordinal); + Ok(()) + } + + pub(in crate::g17) fn addresses(&self) -> [u64; 5] { + match &self.views { + Views::Neo { deflake, auxiliary, fragment_client, fragment } => [ + deflake.iova(), self.state.gpu_va() + storage::TA_STATUS as u64, + fragment_client.iova(), fragment.gpu_va(), auxiliary.iova(), + ], + Views::M5 { status_fw } => { + let base = self.state.gpu_va(); + [base + Layout::M5.deflake as u64, base + Layout::M5.status as u64, + base + Layout::M5.status as u64 + 8, status_fw.iova() + 8, + base + Layout::M5.auxiliary as u64] + } + } + } + + pub(in crate::g17) fn retire(&mut self, ordinal: u64) -> Result { + if self.ordinal != Some(ordinal) { return Err(EIO); } + self.ordinal = None; + Ok(()) + } +} From b854cf83708faadd46fc0d8d52a462d494c20bba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:56 -0800 Subject: [PATCH 069/171] drm/asahi: g17: M5: submit renders on M5 render pairs Build, park and reuse M5 render pairs and submit renders through them on the device-mutex path, renewing the parameter-buffer token when its binding changes. Attachment conflicts across pairs are ordered in firmware, and unrelated whole-attachment renders are not ordered. [ rebased onto integration/aurora: rewritten from t6050-gpu 6613f5ab7b7e..c3505169be1b (11 commits); SPTM/APIF removed; adapted to integration's render queue, buffer manager and device-mutex render path; the M5 features reported in GET_PARAMS except no query access; comments and names reworded; the no-query-access feature defined but not reported, as in the tested reference; M5 paths restored where the first rewrite lost them: a context's render channel moved to a new pair once its publications retire, and the replay of compute a render consumes, held until the owner's renders park; original 6613f5ab7b7e, e25ba8a6fb2b, 1d74fda93777, cda233bc91f7, 3cfa33f3237f, 77dc6eb34210, b0fdab9e3b55, 2f58522abec7, 2c5c77b6aaba, a507978ff411, c3505169be1b ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit c09d6d310e909140df7e5d430b743752c7cc8e08 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/asahi_core.rs | 8 + drivers/gpu/drm/asahi/file.rs | 8 - drivers/gpu/drm/asahi/g17/buffer.rs | 348 +++++++--- drivers/gpu/drm/asahi/g17/buffer/target.rs | 82 ++- drivers/gpu/drm/asahi/g17/context.rs | 36 +- drivers/gpu/drm/asahi/g17/context/identity.rs | 59 +- drivers/gpu/drm/asahi/g17/hazard.rs | 12 +- drivers/gpu/drm/asahi/g17/job.rs | 20 +- drivers/gpu/drm/asahi/g17/mod.rs | 17 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 19 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/render.rs | 620 +++++++++++++++--- .../asahi/g17/queue/render/configuration.rs | 41 ++ .../g17/queue/render/m5_render_submission.rs | 214 ++++++ .../gpu/drm/asahi/g17/queue/render/memory.rs | 434 ++++++++++-- .../gpu/drm/asahi/g17/queue/render/parked.rs | 25 +- .../drm/asahi/g17/queue/render/publication.rs | 61 +- .../drm/asahi/g17/queue/render/retirement.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime.rs | 5 +- .../drm/asahi/g17/runtime/compute_replay.rs | 45 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 92 ++- .../asahi/g17/runtime/render/dependencies.rs | 12 +- .../drm/asahi/g17/runtime/render/direct.rs | 22 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 2 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 33 +- .../drm/asahi/g17/runtime/render/prepare.rs | 244 ++++++- .../drm/asahi/g17/runtime/render/replay.rs | 23 + .../drm/asahi/g17/runtime/render/submit.rs | 42 +- .../drm/asahi/g17/runtime/render/teardown.rs | 8 +- drivers/gpu/drm/asahi/mmu/shared.rs | 3 +- 31 files changed, 2190 insertions(+), 362 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/configuration.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index a218270d6fc425..e1747a772e853c 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -91,6 +91,14 @@ module! { description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", params: { + m5_mesh_storage_chunks: u32 { + default: 0x200, + description: "M5 mesh rendering buffer bound to API task (object-stage) render kicks only, in 64 KiB chunks; 0 disables the binding and the task feature", + }, + m5_depth_msaa_compression: u32 { + default: 1, + description: "Enable single-layer D32 compressed 2x/4x ZLS admission", + }, debug_flags: u64 { default: 0, // permissions: 0o644, diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 237315c4e3d0f2..d0377ec32a6c28 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -66,29 +66,23 @@ use kernel::{ /// sink) instead of the GEM's first page. The GEM only anchors the GPUVA. pub(crate) const DRM_ASAHI_BIND_SPARSE_TILE: u32 = uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SPARSE_TILE; /// DRM_ASAHI_FEATURE_SPARSE_TILES: BIND_SPARSE_TILE accepted. -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_SPARSE_TILES: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SPARSE_TILES as u64; -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_COMPUTE_MATRIX: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_MATRIX as u64; /// DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: render programs accept /// DRM_ASAHI_RENDER_PROGRAM_TASK and such task (object-stage) kicks get the /// TA mesh rendering buffer, which backs the object stage's payload address. /// Reported only while m5_mesh_storage_chunks != 0. -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_TASK_MESH_STORAGE as u64; -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES as u64; /// D32 2x/4x ZLS admission; no compressed stencil or resolve bits. -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION as u64; /// DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: single-layer /// 2x/4x ZLS admission in the compressed Z/S form, stencil included. -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION as u64; #[allow(dead_code)] @@ -96,7 +90,6 @@ pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_SHADER_WRITES; /// Renders may carry separate tiling prefixes of both subqueues /// (DRM_ASAHI_RENDER_STAGE_PREFIXES). -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES as u64; /// The tiling half waits only on the render and compute prefixes of the command's stage-prefix @@ -106,7 +99,6 @@ pub(crate) const DRM_ASAHI_RENDER_STAGE_PREFIXES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_STAGE_PREFIXES; /// Renders may declare that they write whole attachments only /// (DRM_ASAHI_RENDER_WHOLE_ATTACHMENTS). -#[allow(dead_code)] pub(crate) const DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS: u64 = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS as u64; /// The render area covers every attachment's full extent, so its tile stores diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 801b4d9d36f106..c27b2d2e41a638 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -22,6 +22,8 @@ struct BufferId { references: u8, owner: bool, token: u64, + /// Hardware context and PB state named by this token's M5 binding. + bound: Option<(u32, u64)>, } impl BufferId { fn free(self) -> bool { @@ -43,6 +45,7 @@ impl BufferIds { references: 0, owner: false, token: 0, + bound: None, }), returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, @@ -91,6 +94,7 @@ impl BufferIds { references: 1, owner: false, token, + bound: None, }; Ok((id, token)) } @@ -104,6 +108,7 @@ impl BufferIds { if entry.free() { let token = self.fresh_token(); self.entries[usize::from(id)].token = token; + self.entries[usize::from(id)].bound = None; } self.entries[usize::from(id)].owner = true; self.remove_returned(id); @@ -117,6 +122,28 @@ impl BufferIds { entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; Ok((id, entry.token)) } + /// A repeated token must name the same M5 PB state and hardware context. + /// Renew it if an idle VM moved; other firmware profiles keep acquire_owner. + pub(crate) fn acquire_owner_bound(&mut self, id: u8, context: u32, pb_va: u64) + -> Result<(u8, u64)> { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + let references = entry.references.checked_add(1).ok_or(ENOSPC)?; + let previous = entry.bound; + let old_token = entry.token; + let token = if previous.is_some_and(|bound| bound != (context, pb_va)) { + self.fresh_token() + } else { + old_token + }; + let entry = &mut self.entries[usize::from(id)]; + entry.references = references; + entry.token = token; + entry.bound = Some((context, pb_va)); + Ok((id, token)) + } /// Preflight shared completion bookkeeping before any of its counters change. pub(crate) fn can_release(&self, id: u8) -> Result { let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; @@ -149,16 +176,22 @@ impl BufferIds { /// Each scene and its shorter scratch slot are one lifetime. A completed /// paired render releases both; merely publishing a later scene does not. pub(crate) struct Scenes { - owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + owners: [Option<(u64, u8)>; fw::SCENES], + scratch_slots: usize, next: usize, } impl Scenes { pub(crate) const fn new() -> Self { Self { - owners: [None; fw::SCRATCH_SLOTS], + owners: [None; fw::SCENES], + scratch_slots: fw::SCRATCH_SLOTS, next: fw::FIRST_SCENE, } } + fn with_slots(scratch_slots: usize) -> Result { + if scratch_slots == 0 || scratch_slots > fw::SCENES { return Err(EINVAL); } + Ok(Self { scratch_slots, ..Self::new() }) + } pub(crate) fn select(&self, ordinal: u64) -> Result { if let Some((_, scene)) = self .owners @@ -170,14 +203,14 @@ impl Scenes { } (0..fw::SCENES) .map(|offset| (self.next + offset) % fw::SCENES) - .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .find(|scene| self.owners[scene % self.scratch_slots].is_none()) .ok_or(ENOSPC) } pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { if self.select(ordinal)? != scene { return Err(EINVAL); } - let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + let slot = &mut self.owners[scene % self.scratch_slots]; if slot.is_none() { *slot = Some((ordinal, scene as u8)); self.next = (scene + 1) % fw::SCENES; @@ -203,16 +236,16 @@ pub(crate) struct Cursors { pub(crate) read: u32, } impl Cursors { - fn valid(self) -> bool { + fn valid(self, capacity: usize) -> bool { [self.total, self.committed, self.read] .into_iter() - .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + .all(|v| (v as usize) < capacity) } /// Firmware-paused growth permits consumption of the already committed /// prefix. Retired-host growth requires the complete snapshot unchanged. - pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { - if !self.valid() - || !after.valid() + fn permits_capacity(self, after: Self, firmware_request: bool, capacity: usize) -> bool { + if !self.valid(capacity) + || !after.valid(capacity) || self.total != after.total || self.committed != after.committed { @@ -221,7 +254,7 @@ impl Cursors { if !firmware_request { return self == after; } - let capacity = cfg::TVB_BLOCK_SLOTS as u32; + let capacity = capacity as u32; (after.read + capacity - self.read) % capacity <= (self.committed + capacity - self.read) % capacity } @@ -230,23 +263,25 @@ impl Cursors { /// Checked block-table indices for one bulk allocation. One ring slot stays /// empty even when all physical backing has been retained by the owner. pub(crate) struct Growth { + capacity: usize, before: Cursors, added: usize, retained: usize, target: usize, } impl Growth { - pub(crate) fn new(before: Cursors, added: usize) -> Result { - if !before.valid() || before.total != before.committed || added == 0 { + fn with_capacity(before: Cursors, added: usize, capacity: usize) -> Result { + if capacity < 2 || capacity > u32::MAX as usize + || !before.valid(capacity) || before.total != before.committed || added == 0 { return Err(EINVAL); } - let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) - % cfg::TVB_BLOCK_SLOTS; - if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + let retained = (before.total as usize + capacity - before.read as usize) % capacity; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= capacity { return Err(ENOSPC); } - let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + let target = (before.total as usize + added) % capacity; Ok(Self { + capacity, before, added, retained, @@ -266,32 +301,25 @@ impl Growth { if index >= self.added { return Err(EINVAL); } - Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.total as usize + index) % self.capacity) } pub(crate) fn available_slot(&self, index: usize) -> Result { if index >= self.available() { return Err(EINVAL); } - Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + Ok((self.before.read as usize + index) % self.capacity) } } -pub(crate) fn firmware_growth_target(current: usize) -> usize { - current - .saturating_add(fw::GROW_BLOCKS) - .min(cfg::TVB_MAX_BLOCKS) -} - /// The high-water sample estimates capacity, not current occupancy. A growth /// reserves three times the demand only once twice that demand no longer fits. -pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { - let maximum = fw::NORMAL_MAX_BLOCKS; +fn automatic_target_for(used_pages: usize, current: usize, pages: usize, maximum: usize) -> Option { if current >= maximum { return None; } let need = used_pages .saturating_mul(2) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum); if need <= current { return None; @@ -299,7 +327,7 @@ pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Opti Some( used_pages .saturating_mul(3) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .div_ceil(pages) .min(maximum), ) } @@ -343,7 +371,11 @@ pub(crate) struct TvbRelease { /// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. /// Firmware owns their contents; CPU zeroing occurs only on fresh backing. +/// +/// M5 blocks use their own geometry ([`BlockGeometry::m5`]): sparse, at the +/// geometry's backing stride, in the M5 compact window. pub(crate) struct Tvb { + geometry: BlockGeometry, addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. _primary: KernelObject, @@ -355,11 +387,27 @@ impl Tvb { fn bytes(blocks: usize) -> Result { blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } + fn backing_bytes(geometry: BlockGeometry, blocks: usize) -> Result { + if geometry == BlockGeometry::NEO { + return Self::bytes(blocks); + } + geometry.backing_bytes(blocks) + } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, + geometry: BlockGeometry, ) -> Result { + if geometry != BlockGeometry::NEO { + let start = relative.checked_mul(geometry.backing_stride).ok_or(EOVERFLOW)?; + let source = start..start.checked_add(geometry.block_bytes).ok_or(EOVERFLOW)?; + let guard = geometry.stride - geometry.block_bytes; + let mapping = object.map_range(vm, source, geometry.base..geometry.end, + BlockGeometry::PAGE as u64, mmu::PROT_GPU_SHARED_RW, guard)?; + geometry.compact_page(mapping.iova())?; + return Ok(mapping); + } let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( @@ -378,19 +426,29 @@ impl Tvb { vm: &mmu::Vm, blocks: usize, ) -> Result<(Self, KVec)> { - if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + Self::with_geometry(dev, vm, blocks, BlockGeometry::NEO) + } + pub(crate) fn with_geometry( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + blocks: usize, + geometry: BlockGeometry, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > geometry.max_blocks() { return Err(ERANGE); } - let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(geometry, blocks)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; for index in 0..blocks { - let mapping = Self::map_block(&backing, vm, index)?; + let mapping = Self::map_block(&backing, vm, index, geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } Ok(( Self { + geometry, addresses, _primary: backing, extensions: KVec::new(), @@ -426,17 +484,18 @@ impl Tvb { if target <= self.blocks() { return Ok(None); } - if target > cfg::TVB_MAX_BLOCKS { + if target > self.geometry.max_blocks() { return Err(ERANGE); } let added = target - self.blocks(); self.extensions.reserve(1, GFP_KERNEL)?; self.addresses.reserve(added, GFP_KERNEL)?; - let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let backing = KernelObject::backing(dev, Self::backing_bytes(self.geometry, added)?, + CpuMap::WriteCombined)?; let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; for relative in 0..added { - let mapping = Self::map_block(&backing, vm, relative)?; + let mapping = Self::map_block(&backing, vm, relative, self.geometry)?; addresses.push(mapping.iova(), GFP_KERNEL)?; mappings.push(mapping, GFP_KERNEL)?; } @@ -452,10 +511,14 @@ use super::object::Allocator; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::sync::Arc; +/// Bytes per scene of an M5 metrics entry (the M5 graph layout's metric stride). +const M5_METRIC_STRIDE: usize = 64; + /// First-free metrics entries are shared by all parameter-buffer graphs. /// An entry returns only when the graph and its firmware references retire. pub(crate) struct MetricsIds { used: [AtomicU64; 2], + scene_stride: usize, } impl MetricsIds { pub(crate) fn new() -> Self { @@ -464,8 +527,34 @@ impl MetricsIds { AtomicU64::new(0), AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), ], + scene_stride: size_of::(), + } + } + /// The allocator of the firmware's metrics entries: G17P's fixed entries, + /// or M5's `M5_METRIC_STRIDE`-byte scene entries. + pub(crate) fn for_firmware(m5: bool) -> Result { + if m5 { + Self::with_stride(M5_METRIC_STRIDE) + } else { + Ok(Self::new()) } } + /// The M5 metrics entries: `scene_stride` bytes per scene, bounded by the + /// backing each lease is taken from. + pub(crate) fn with_stride(scene_stride: usize) -> Result { + if scene_stride < 4 || scene_stride % 4 != 0 { return Err(EINVAL); } + scene_stride.checked_mul(fw::SCENES).ok_or(EOVERFLOW)?; + Ok(Self { + used: [AtomicU64::new(0), AtomicU64::new(0)], + scene_stride, + }) + } + /// Backing required for every entry owned by this allocator. + pub(crate) fn storage_bytes(&self) -> Result { + self.scene_stride.checked_mul(fw::SCENES) + .and_then(|bytes| bytes.checked_mul(self.used.len() * u64::BITS as usize)) + .ok_or(EOVERFLOW) + } } pub(crate) struct MetricsLease { @@ -473,20 +562,39 @@ pub(crate) struct MetricsLease { allocator: Arc, slot: usize, addresses: fw::MetricsAddresses, + scene_stride: usize, } impl MetricsLease { + /// M5: the client view of the metrics backing is placed per VM. + pub(crate) fn set_client_base(&mut self, base: u64) -> Result { + let offset = self.slot.checked_mul(self.scene_stride) + .and_then(|bytes| bytes.checked_mul(fw::SCENES)).ok_or(EOVERFLOW)?; + self.addresses.client = base.checked_add(offset as u64).ok_or(EOVERFLOW)?; + Ok(()) + } /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, bases: fw::MetricsAddresses, ) -> Result { + let scene_stride = ids.scene_stride; + let bytes = scene_stride.checked_mul(fw::SCENES).ok_or(EOVERFLOW)?; + // The fixed G17P entries are bounded by their reserved bits; an M5 + // allocator by the backing the lease is taken from. + let slots = if scene_stride == size_of::() { ids.used.len() * 64 } else { + // The M5 backing covers every entry of its allocator. + if object.size() < ids.storage_bytes()? { + return Err(EINVAL); + } + (object.size() / bytes).min(ids.used.len() * 64) + }; let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { let mut value = used.load(Ordering::Acquire); loop { let bit = value.trailing_ones(); - if bit == 64 { + if bit == 64 || bank * 64 + bit as usize >= slots { break; } match used.compare_exchange_weak( @@ -514,15 +622,16 @@ impl MetricsLease { allocator: ids.clone(), slot, addresses: fw::MetricsAddresses::default(), + scene_stride, }; - let offset = slot * fw::METRICS_STRIDE; + let offset = slot * bytes; lease.addresses = fw::MetricsAddresses { firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, }; - let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + let words = object.pointer(offset, bytes)?.cast::(); object.word(offset)?; - for index in 0..fw::SCENES { + for index in 0..bytes / size_of::() { // SAFETY: The complete naturally aligned metrics entry is inside // the retained object; the exclusive lease excludes other owners. unsafe { words.add(index).write_volatile(0) }; @@ -551,9 +660,12 @@ pub(crate) struct ManagerArgs { pub(crate) struct Manager { _page_list: mmu::KernelMapping, graph: KernelObject, + layout: layout::Layout, tvb: Tvb, metrics: MetricsLease, - scenes: Scenes, + /// Boxed: the per-scene owner table would otherwise make every `Manager` value large, and + /// one is built and moved through the frames of the submit path that creates a pair. + scenes: KBox, page_list_client: u64, scratch: u64, buffer_id: u8, @@ -619,41 +731,56 @@ impl Manager { metrics: MetricsLease, args: ManagerArgs, ) -> Result { + let geometry = tvb.geometry; + let neo = geometry == BlockGeometry::NEO; + let layout = if neo { layout::Layout::NEO } else { layout::Layout::m5(geometry)? }; + if metrics.scene_stride != layout.metric_stride { + return Err(EINVAL); + } let mut graph = alloc.kernel( - fw::SIZE, + layout.bytes, cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; - let page_list = compact_alias( - &graph, - vm, - source, - cfg::TVB_PAGE, - mmu::PROT_GPU_FW_SHARED_RW, - mmu::UAT_PGSZ, - )?; + let source = layout.pages..layout.pages + layout.page_bytes; + let page_list = if neo { + compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )? + } else { + graph.map_range(vm, source, geometry.base..geometry.end, + cfg::TVB_PAGE, mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ)? + }; let address = graph.gpu_va(); - graph.initialize::(0, |view| { - view.init(&fw::GraphArgs { - fw_va: address, - page_list_client: page_list.iova(), - metrics: metrics.addresses, - blocks: tvb.addresses(), - scratch: args.scratch, - discard: args.discard, - buffer_id: args.buffer_id.into(), - generation: args.generation, - }) - })?; + let graph_args = fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }; + if neo { + graph.initialize::(0, |view| view.init(&graph_args))?; + } else { + layout.initialize(geometry, &graph_args, graph.bytes_mut())?; + } fence(Ordering::SeqCst); - let mut scenes = Scenes::new(); + let mut scenes = KBox::new(Scenes::with_slots(layout.scratch_slots)?, GFP_KERNEL)?; scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; let page_list_client = page_list.iova(); Ok(Self { _page_list: page_list, graph, + layout, tvb, metrics, scenes, @@ -680,6 +807,13 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + pub(crate) fn page_count(&self) -> Result { + self.blocks().checked_mul(self.tvb.geometry.pages()).ok_or(EOVERFLOW)? + .try_into().map_err(|_| EOVERFLOW) + } + pub(crate) fn firmware_target(&self) -> usize { + self.blocks().saturating_add(fw::GROW_BLOCKS).min(self.tvb.geometry.max_blocks()) + } /// Read-only diagnostic while firmware is halted. In particular this /// does not reset the high-water counter as automatic_target() does. pub(crate) fn report_recovery(&self) { @@ -692,8 +826,8 @@ impl Manager { read(core::mem::offset_of!(fw::State, block_capacity)), read(core::mem::offset_of!(fw::State, producer)), read(core::mem::offset_of!(fw::State, consumer)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + read(self.layout.control + core::mem::offset_of!(fw::Control, max_pages)), + read(self.layout.control + core::mem::offset_of!(fw::Control, reset)), self.initialized); } pub(crate) fn state_va(&self) -> u64 { @@ -703,20 +837,33 @@ impl Manager { self.page_list_client } pub(crate) fn trailer_va(&self) -> u64 { - self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + self.graph.gpu_va() + self.layout.scenes as u64 + (fw::SCENES * size_of::()) as u64 } pub(crate) fn scene_va(&self, index: usize) -> Result { if index >= fw::SCENES { return Err(EINVAL); } - Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + Ok(self.graph.gpu_va() + self.layout.scenes as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses.client) + if self.tvb.geometry == BlockGeometry::NEO { + return fw::scene_registers(index, self.scratch, self.metrics.addresses.client); + } + if index >= fw::SCENES { return Err(EINVAL); } + let scratch = self.scratch.checked_sub(self.tvb.geometry.base).ok_or(EINVAL)? + .checked_add((index % self.layout.scratch_slots * self.layout.scratch_stride) as u64) + .ok_or(EOVERFLOW)?; + let metric = self.metrics.addresses.client.checked_sub(self.tvb.geometry.base) + .ok_or(EINVAL)? + .checked_add((index * self.layout.metric_stride) as u64).ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || metric > u32::MAX as u64 || metric & 3 != 0 { + return Err(EINVAL); + } + Ok((scratch, metric | 1)) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; - let offset = fw::SCENE_TABLE + scene * size_of::(); + let offset = self.layout.scenes + scene * size_of::(); let used = self .graph .word(offset + core::mem::offset_of!(fw::Scene, used))?; @@ -741,7 +888,7 @@ impl Manager { self.scenes.release(ordinal) } pub(crate) fn prepare_submission(&mut self) -> Result> { - let word = self.graph.word(fw::COUNTER)?; + let word = self.graph.word(self.layout.counter)?; Ok(Submitted { word, next: word.load(Ordering::Relaxed).wrapping_add(1), @@ -752,10 +899,10 @@ impl Manager { fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, max_pages))?; let reset = self .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + .word(self.layout.control + core::mem::offset_of!(fw::Control, reset))?; fence(Ordering::Acquire); let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); @@ -767,7 +914,8 @@ impl Manager { pub(crate) fn automatic_target(&mut self) -> Result> { self.sample_max_pages()?; let pages = core::mem::take(&mut self.shrink.growth_max); - Ok(automatic_growth_target(pages as usize, self.blocks())) + Ok(automatic_target_for(pages as usize, self.blocks(), self.tvb.geometry.pages(), + self.layout.normal_blocks)) } /// Records the blocks a pass layout requires, the floor of any shrink. /// A pass is being prepared at `now`; the heap counts as busy. @@ -785,12 +933,13 @@ impl Manager { self.tvb.primary_blocks().max(self.shrink.required).max(recent) } /// Whether the heap holds growth beyond its primary backing. + /// M5 heaps are never shrunk. pub(crate) fn grown(&self) -> bool { - !self.tvb.extensions.is_empty() + self.tvb.geometry == BlockGeometry::NEO && !self.tvb.extensions.is_empty() } /// Whether the heap holds growth that a shrink could give back. pub(crate) fn shrinkable(&self) -> bool { - self.tvb.extensions.last().is_some_and(|extension| { + self.tvb.geometry == BlockGeometry::NEO && self.tvb.extensions.last().is_some_and(|extension| { self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() }) } @@ -839,9 +988,9 @@ impl Manager { } fn cursors(&self) -> Result { Ok(Cursors { - total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), - committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), - read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + total: self.graph.word(self.layout.control)?.load(Ordering::Relaxed), + committed: self.graph.word(self.layout.control + 4)?.load(Ordering::Relaxed), + read: self.graph.word(self.layout.control + 8)?.load(Ordering::Relaxed), }) } } @@ -864,12 +1013,14 @@ impl Manager { /// Validate every retained sparse block and its reserved trailing guard in /// one VM snapshot before constructing more aliases. pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { - if self.blocks() < cfg::TVB_INITIAL_PAGES.len() - || self.blocks() > cfg::TVB_MAX_BLOCKS + let geometry = self.tvb.geometry; + let minimum = if geometry == BlockGeometry::NEO { cfg::TVB_INITIAL_PAGES.len() } else { 1 }; + if self.blocks() < minimum + || self.blocks() > geometry.max_blocks() || !vm.covers_sparse_blocks_with_guards( self.tvb.addresses(), - cfg::TVB_BLOCK_SIZE as u64, - cfg::TVB_BLOCK_STRIDE, + geometry.block_bytes as u64, + geometry.stride as u64, ) { return Err(EFAULT); @@ -896,23 +1047,23 @@ impl Manager { return Ok(None); }; let before = self.cursors()?; - let plan = Growth::new(before, target - previous)?; + let plan = Growth::with_capacity(before, target - previous, self.tvb.geometry.capacity)?; let firmware_request = descriptor.is_none(); let pages = target - .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .checked_mul(self.tvb.geometry.pages()) .ok_or(EOVERFLOW)? as u32; let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; for (index, &address) in growth.addresses.iter().enumerate() { - let first = fw::compact_page(address)?; + let first = self.tvb.geometry.compact_page(address)?; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; } let rebuilt_count = if firmware_request { 0 } else { - plan.available() * cfg::TVB_PAGES_PER_BLOCK + plan.available() * self.tvb.geometry.pages() }; let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; if !firmware_request { @@ -923,15 +1074,15 @@ impl Manager { let first = if ordinal < plan.retained() { let slot = plan.available_slot(ordinal)?; self.graph - .word(fw::BLOCK_TABLE + slot * size_of::())? + .word(self.layout.blocks + slot * size_of::())? .load(Ordering::Relaxed) } else { added[ordinal - plan.retained()].1 }; first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .checked_add(self.tvb.geometry.pages() as u32 - 1) .ok_or(EOVERFLOW)?; - for page in 0..cfg::TVB_PAGES_PER_BLOCK { + for page in 0..self.tvb.geometry.pages() { rebuilt.push(first + page as u32, GFP_KERNEL)?; } } @@ -965,7 +1116,7 @@ impl Manager { return Err(EBUSY); } let firmware_request = candidate.descriptor.is_none(); - let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + let base = self.graph.pointer(0, self.layout.bytes)?.cast::(); // Bounds, actual alignment and every reserved host entry are checked // before transferring aliases or consuming the caller-owned plan. self.graph.word(0)?; @@ -975,7 +1126,8 @@ impl Manager { { return Err(EIO); } - if !candidate.before.permits(self.cursors()?, firmware_request) { + if !candidate.before.permits_capacity(self.cursors()?, firmware_request, + self.tvb.geometry.capacity) { return Err(EAGAIN); } // The callback retains its input on error. The caller can therefore @@ -1016,13 +1168,13 @@ impl Manager { unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; }; for (slot, first) in added.into_iter() { - let at = fw::BLOCK_TABLE + slot * size_of::(); + let at = self.layout.blocks + slot * size_of::(); write(at, first); write(at + 4, 0); } if descriptor.is_some() { for (entry, value) in rebuilt.into_iter().enumerate() { - write(fw::PAGE_LIST + entry * 4, value); + write(self.layout.pages + entry * 4, value); } write(core::mem::offset_of!(fw::State, pages), pages); write(core::mem::offset_of!(fw::State, producer), plan.target()); @@ -1030,9 +1182,9 @@ impl Manager { write(core::mem::offset_of!(fw::State, last_page), pages - 1); } append_reserved(&mut self.tvb.extensions, extension); - write(fw::CONTROL, plan.target()); + write(self.layout.control, plan.target()); fence(Ordering::SeqCst); - write(fw::CONTROL + 4, plan.target()); + write(self.layout.control + 4, plan.target()); fence(Ordering::SeqCst); Ok(descriptor) } @@ -1072,7 +1224,7 @@ impl Manager { let before = self.cursors()?; let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; - if !before.valid() || before.total != before.committed || retained != blocks { + if !before.valid(cfg::TVB_BLOCK_SLOTS) || before.total != before.committed || retained != blocks { return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1161,8 +1313,10 @@ impl BufferIds { } impl MetricsIds { pub(crate) fn used(&self) -> u32 { + let reserved = if self.scene_stride == size_of::() { 128 - fw::METRICS_SLOTS as u32 } + else { 0 }; self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() - - (128 - fw::METRICS_SLOTS as u32) + - reserved } } diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bd96ecc9d460e7..b1ef1a16ed00bf 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -39,6 +39,7 @@ const GTP6_SIZE: usize = GTP_SLOTS /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { + pub(crate) tilemap_size: usize, pub(crate) layer_offset: usize, pub(crate) heap_offset: usize, pub(crate) used: usize, @@ -49,6 +50,8 @@ pub(crate) struct Layout { gtp: [Option; 8], /// Bytes of the TPC object the command uses. tpc_end: usize, + /// The M5 tiler slices of the TPC object and their cluster controls. + m5: Option<(crate::g17::fw::m5::target::TargetSlices, [u64; 3])>, } impl Layout { pub(crate) fn new( @@ -134,6 +137,7 @@ impl Layout { tpc_end = position; } Ok(Self { + tilemap_size, layer_offset, heap_offset, used, @@ -141,9 +145,31 @@ impl Layout { tvb_blocks, gtp, tpc_end, + m5: None, }) } + pub(crate) fn with_m5_slices(mut self, pass: &RenderPass, groups: u32, + segments: u32, parameter_bytes: usize) -> Result { + let slices = crate::g17::fw::m5::target::TargetSlices::new( + self.tilemap_size, self.tpc_size, pass.layers, groups, segments, parameter_bytes) + .ok_or(EINVAL)?; + let control = crate::g17::fw::m5::target::cluster_control(groups, u64::from(segments)) + .ok_or(EINVAL)?; + self.m5 = Some((slices, control)); + Ok(self) + } + pub(crate) fn is_m5(&self) -> bool { self.m5.is_some() } + fn check_capacity(&self, stride: usize) -> Result { + if self.m5.is_none() && stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + stride.checked_mul(DEPTH).ok_or(EOVERFLOW)?; + Ok(()) + } pub(crate) fn tpc_capacity(&self) -> Result { + if let Some((slices, _)) = self.m5 { + return aligned(slices.bytes, mmu::UAT_PGSZ); + } // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object // also contains GTP sub-buffers, notably 64 times the tile map. Applying // the cache limit to their combined extent rejects valid large targets. @@ -157,7 +183,8 @@ impl Layout { Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { - Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + let bytes = aligned(self.used, mmu::UAT_PGSZ)?; + Ok(if self.m5.is_some() { bytes } else { bytes.max(cfg::TARGET_RECORD_MIN) }) } } @@ -167,10 +194,15 @@ struct Object { backing: KernelObject, } impl Object { - fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize) -> Result { + fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, size: usize, m5: bool) -> Result { let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; - let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + let mapping = if m5 { + backing.map_range(vm, 0..size, super::BlockGeometry::M5_COMPACT, + cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + } else { + compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)? + }; Ok(Self { address: mapping.iova(), _mapping: mapping, @@ -196,6 +228,7 @@ pub(crate) struct Target { stride: usize, slots: [Option; DEPTH], retired: KVec, + m5: bool, } impl Target { /// Keep this allocation frame separate from the rest of the render graph. @@ -203,11 +236,10 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } - let records = Object::new(dev, vm, stride * DEPTH)?; - let tpc = Object::new(dev, vm, tpc)?; + layout.check_capacity(stride)?; + let m5 = layout.m5.is_some(); + let records = Object::new(dev, vm, stride * DEPTH, m5)?; + let tpc = Object::new(dev, vm, tpc, m5)?; KBox::new( Self { records, @@ -215,6 +247,7 @@ impl Target { stride, slots: [None; DEPTH], retired: KVec::new(), + m5, }, GFP_KERNEL, ) @@ -224,6 +257,27 @@ impl Target { self.tpc.address } + /// The M5 tiling words naming the TPC object's slices of `layout`. + pub(crate) fn m5_extra(&self, layout: &Layout) + -> Result { + if !self.m5 { return Err(EINVAL); } + let (slices, control) = layout.m5.ok_or(EINVAL)?; + if slices.bytes > self.tpc.backing.size() { return Err(ERANGE); } + let base = self.tpc.address.checked_sub(super::BlockGeometry::M5_COMPACT.start) + .ok_or(EINVAL)?; + let mut pointers = [0; 8]; + for index in 0..8 { + if slices.sizes[index] != 0 { + pointers[index] = base.checked_add(slices.offsets[index] as u64).ok_or(EOVERFLOW)?; + } + } + Ok(crate::g17::fw::m5::render_registers::TaExtra { + gtp1: pointers[1], gtp2: pointers[2], gtp4: pointers[4], gtp5: pointers[5], + gtp6: pointers[6], gtp7: pointers[7], segment_control: control[0], segment_index: control[1], + min_free_tvb_pages: control[2], scs_bracket: 0, matrix_bracket: 0, + }) + } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty /// or absent one. The TPC object holds them after the TPC. pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { @@ -243,6 +297,7 @@ impl Target { /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { let slot = ordinal as usize % DEPTH; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } if self.slots[slot].is_some_and(|owner| owner != ordinal) { return Err(EBUSY); } @@ -285,15 +340,14 @@ impl Target { ) -> Result { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; + if self.m5 != layout.m5.is_some() { return Err(EINVAL); } + layout.check_capacity(stride)?; let mut grew = false; if tpc > self.tpc.backing.size() { - Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners, self.m5)?; grew = true; } if stride > self.stride { - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } Self::replace( &mut self.retired, &mut self.records, @@ -301,6 +355,7 @@ impl Target { vm, stride * DEPTH, owners, + self.m5, )?; self.stride = stride; self.slots = [None; DEPTH]; @@ -315,12 +370,13 @@ impl Target { vm: &mmu::Vm, size: usize, owners: &[Option; DEPTH], + m5: bool, ) -> Result { let retain = owners.iter().any(Option::is_some); if retain { retired.reserve(1, GFP_KERNEL)?; } - let new = Object::new(dev, vm, size)?; + let new = Object::new(dev, vm, size, m5)?; let previous = core::mem::replace(old, new); if retain { super::append_reserved( diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index ef7934585e4983..7ac04bb34710ed 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -16,6 +16,10 @@ use super::{ use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; + +/// Both of a context's render channels are bound to other pairs whose publications have not +/// all retired: the publisher waits for that retirement. +pub(crate) const RENDER_CHANNEL_PENDING: Error = ERECALLCONFLICT; use super::job::Engine; use kernel::{ new_mutex, @@ -604,9 +608,35 @@ impl WorkStateLease { .change_count(self.word(), kicks, false) } - pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or_else(|| - super::capacity::refused(super::capacity::Site::RenderWorkSlots)) + /// The render channel of `pair` for a publication. When both channels are bound to other + /// pairs, the least recently used one whose every publication `retired(qids, ordinal)` + /// moves to `pair`; otherwise [`RENDER_CHANNEL_PENDING`]: the caller waits for retirement + /// instead of failing the client. + pub(crate) fn render_channel( + &self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Result { + let (index, previous) = self + .context + .work + .lock() + .render + .bind(pair, retired) + .ok_or(RENDER_CHANNEL_PENDING)?; + if let Some(previous) = previous { + static REBINDS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + let n = REBINDS.fetch_add(1, Ordering::Relaxed) + 1; + if n <= 8 || n.is_power_of_two() { + pr_info!("asahi: render channel {} of context {} moved from pair {} to pair {} (rebind {})\n", + index, self.context.id(), previous, pair, n); + } + } + Ok(index) + } + /// Records a publication through `pair`'s channel once it reached firmware. + pub(crate) fn channel_published(&self, pair: u8, qids: [u8; 2], ordinal: u64) { + self.context.work.lock().render.published(pair, qids, ordinal); } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7586f8474e602f..b3ae8831a6523b 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -172,19 +172,35 @@ impl Order { } } -/// Two stable logical ordinals, independent of physical pair numbering. +/// Two stable logical ordinals (render channels), independent of physical pair numbering. A +/// context's render work can move between physical pairs (sibling pairs, lending, relocation, +/// QID recycling), so a channel is rebound to a new pair once every publication this context +/// made through it has retired. A context's channel counts its passes up for the whole +/// process, so a rebound channel continues its pass sequence. pub(super) struct RenderSlots { pairs: [Option; 2], origins: [Option; 2], + /// Pass of the channel's last publication on its previous pair. + bases: [u8; 2], + /// QIDs and visibility target (ordinal + 1) of the last publication made through each + /// channel. + last: [Option<([u8; 2], u64)>; 2], + /// Recency of each channel's last publication. + used: [u64; 2], + epoch: u64, } impl RenderSlots { pub(super) const fn new() -> Self { Self { pairs: [None; 2], origins: [None; 2], + bases: [0; 2], + last: [None; 2], + used: [0; 2], + epoch: 0, } } - pub(super) fn get(&mut self, pair: u8) -> Option { + fn get(&mut self, pair: u8) -> Option { if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { return Some(index as u32); } @@ -192,10 +208,45 @@ impl RenderSlots { self.pairs[index] = Some(pair); Some(index as u32) } + /// The channel of `pair`, rebinding the least recently used channel whose publications + /// have all retired (`retired(qids, ordinal)`) when both are bound to other pairs. `None`: + /// no channel can move yet. + pub(super) fn bind( + &mut self, + pair: u8, + retired: impl Fn([u8; 2], u64) -> bool, + ) -> Option<(u32, Option)> { + if let Some(index) = self.get(pair) { + return Some((index, None)); + } + let index = (0..2) + .filter(|index| self.last[*index].is_none_or(|(qids, ordinal)| retired(qids, ordinal))) + .min_by_key(|index| self.used[*index])?; + let previous = self.pairs[index]; + // `last` holds the ordinal after the last publication (its visibility target), so that + // publication's pass is base + (next - origin). + if let (Some(origin), Some((_, next))) = (self.origins[index], self.last[index]) { + self.bases[index] = self.bases[index].wrapping_add(next.wrapping_sub(origin) as u8); + } + self.pairs[index] = Some(pair); + self.origins[index] = None; + self.last[index] = None; + Some((index as u32, previous)) + } pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { - let index = self.get(pair)? as usize; + let index = self.pairs.iter().position(|value| *value == Some(pair))?; let origin = *self.origins[index].get_or_insert(ordinal); - Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + Some(self.bases[index].wrapping_add(ordinal.checked_sub(origin)?.wrapping_add(1) as u8)) + } + /// Records a publication that reached firmware through `pair`'s channel. + pub(super) fn published(&mut self, pair: u8, qids: [u8; 2], ordinal: u64) -> bool { + let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) else { + return false; + }; + self.epoch = self.epoch.wrapping_add(1); + self.used[index] = self.epoch; + self.last[index] = Some((qids, ordinal)); + true } } diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 4d9e99d9a9cc0c..ca4cd5e30f0630 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -32,10 +32,13 @@ pub(crate) struct Footprint { spans: [Span; MAX_SPANS], count: usize, shared: bool, + /// The render writes whole attachments: its tile stores reach no pixel + /// outside its own render area. + whole: bool, } impl Footprint { - pub(crate) fn new(vm: u64) -> Self { + pub(crate) fn new(vm: u64, whole: bool) -> Self { Self { vm, spans: [Span { @@ -45,6 +48,7 @@ impl Footprint { }; MAX_SPANS], count: 0, shared: false, + whole, } } @@ -75,6 +79,12 @@ impl Footprint { if self.vm != other.vm && !(self.shared && other.shared) { return false; } + // Tile-granular stores of two whole-attachment renders write only + // pixels both renders own; unordered, they race as the application + // asked. + if self.whole && other.whole { + return false; + } self.spans[..self.count].iter().any(|a| { other.spans[..other.count] .iter() diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 3e80f72ad8b953..93761c025e4c35 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -41,6 +41,9 @@ impl Engine { pub(crate) struct Order { pub(crate) sequence: u64, pub(crate) wait_through: [Option; 2], + /// What a render's tiling half waits on: `wait_through` unless the command + /// carries its own, never later, tiling prefixes. + pub(crate) tiling_wait_through: [Option; 2], } impl Order { @@ -81,9 +84,22 @@ impl Order { Ok(Self { sequence: sequences[engine.index()], wait_through, + tiling_wait_through: wait_through, }) } + /// Give the tiling half the narrower prefixes of `tiling`, which resolves the + /// command's tail against the same ioctl position. + pub(crate) fn narrow_tiling(&mut self, tiling: &Self) -> Result { + for index in 0..2 { + if tiling.wait_through[index].unwrap_or(0) > self.wait_through[index].unwrap_or(0) { + return Err(EINVAL); + } + } + self.tiling_wait_through = tiling.wait_through; + Ok(()) + } + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { sequence != 0 && prefix.is_some_and(|last| sequence <= last) } @@ -115,8 +131,8 @@ impl Packet { firmware: KVec, ) -> Result> { let footprint = match &command { - Validated::Render { attachments, .. } => { - context.vm().render_footprint(&attachments[1])? + Validated::Render { attachments, pass, .. } => { + context.vm().render_footprint(&attachments[1], pass.whole_attachments)? } Validated::Compute { .. } => None, }; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9e26d47a309265..6234eb18d337c4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1807,6 +1807,23 @@ impl gpu::Gpu for Gpu { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); } + if variant.m5_commands { + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_RESOURCES) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2) + | crate::file::DRM_ASAHI_FEATURE_SPARSE_TILES + | crate::file::DRM_ASAHI_FEATURE_COMPUTE_MATRIX + | crate::file::DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES + | crate::file::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES + | crate::file::DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS; + if queue::render::m5_render_submission::depth_msaa_compression_enabled() { + params.features |= crate::file::DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION + | crate::file::DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION; + } + if crate::module_parameters::m5_mesh_storage_chunks.value() != 0 { + params.features |= crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; + } + } params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0fb686001cd091..2d99e4a2505590 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -602,6 +602,8 @@ pub(crate) struct Queue { replay_retirement: bool, replay_error: Option, replay_prepared: bool, + /// Since when the replay front waits for its render consumers (see compute_replay). + consumer_wait_since: Option, exit: Option, /// Since when the kill of this failed queue's context waits for other contexts' compute, /// and whether that wait was reported. @@ -683,6 +685,7 @@ impl Queue { replay_retirement: false, replay_error: None, replay_prepared: false, + consumer_wait_since: None, exit: None, kill_wait: None, installed: false, @@ -782,6 +785,13 @@ impl Queue { pub(crate) fn in_flight(&self) -> bool { (self.active.len != 0 || self.configuration.is_some()) && !self.quarantined } + /// Per-queue consumer wait (see compute_replay). + pub(crate) fn consumer_wait(&mut self, now: u64) -> u64 { + now.saturating_sub(*self.consumer_wait_since.get_or_insert(now)) + } + pub(crate) fn clear_consumer_wait(&mut self) { + self.consumer_wait_since = None; + } /// Cold M5 installation publishes one configuration before the first kick. /// A prepared claim prevents reassignment while these credits are owned. #[allow(dead_code)] // Read by M5 lazy admission, a follow-up. @@ -1195,11 +1205,12 @@ impl Queue { // M5 command words stay zero even for executed commands. The // front needs its exact zero-duration ring record; a later // command, chained to its predecessor by the kick parent, must - // not show a record with a real duration. As on G17P, work a - // render consumes through a firmware dependency is never spared. + // not show a record with a real duration. A render consumer of + // this work (firmware dependency) is checked by the replay pass: + // it republishes only while every render of the owner is parked + // or idle and none executed since recovery. self.active.iter().enumerate().all(|(index, active)| { - !active.packet.completion.has_firmware_consumer() - && !active.packet.completion.signalled() + !active.packet.completion.signalled() && match active.observed { // Only reached for spared work quarantined after a // recovery: every retained command crossed it. diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index c0a916a5a559e3..32a7e9e19fd258 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -442,6 +442,9 @@ impl Queue { command.payload.prior(), engine, )?; + if let Some((barriers, prior)) = command.payload.tiling_prefixes() { + order.narrow_tiling(&Order::new(base, preceding, barriers, prior, engine)?)?; + } preceding[engine.index()] += 1; let destinations = match &payload { Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c226388b4dd288..81aa4c711c8fb0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -3,6 +3,8 @@ //! Physical paired tiling and fragment queues and their retained command storage. pub(in crate::g17) mod generation; +mod configuration; +pub(in crate::g17) mod m5_render_submission; mod observation; pub(crate) use observation::Observed; mod dependencies; @@ -39,14 +41,20 @@ use retirement::{Ticket, Tracker}; /// are built: the covered earlier renders (their fragment completions) and the /// covered compute completion. Which kick waits for the render prefix is the /// command's choice (`RenderPass::fragment_barriers`); compute always orders -/// the tiling kick, since compute output may feed vertex work. +/// the tiling kick, since compute output may feed vertex work. A stage-prefix +/// command (`RenderPass::stage_prefixes`) also names the earlier renders its +/// tiling half waits for (`render_tiling`, whole-render completions) and, when +/// its fragment half waits on later compute work, that compute completion. #[derive(Copy, Clone)] pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) render_tiling: [KickDependency; 2], + pub(crate) render_tiling_count: usize, pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + pub(crate) fragment_compute: Option, /// Fragment kicks of other pairs' overlapping same-VM renders. pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], pub(crate) hazard_count: usize, @@ -56,9 +64,12 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + render_tiling: [KickDependency::ZERO; 2], + render_tiling_count: 0, tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + fragment_compute: None, hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], hazard_count: 0, }; @@ -77,12 +88,16 @@ pub(crate) struct RenderAddresses { pub(crate) struct PublicationLease { pub(crate) free_list_generation: u64, pub(crate) buffer_token: u64, + /// Hardware context for which the M5 token was selected. + pub(crate) buffer_context: Option, } /// Host services borrow the device-locked firmware state. Resource construction /// and command geometry preparation do not borrow this interface. pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; + /// The host's accepted recovery generation (no device access). + fn generation(&self) -> u64; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; /// Publishes `owners` in order with one pass over the QoS table. fn qos_publish_pair( @@ -92,7 +107,7 @@ pub(crate) trait Host { policy: crate::g17::fw::queue::Policy, ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; - fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn install_pair(&mut self, registration: &Registration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; /// The next publication number, shared with compute (timeout ordering). fn next_publication(&mut self) -> Result; @@ -104,6 +119,13 @@ pub(crate) trait Host { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]>; + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &WorkSlot, + tiling: &WorkSlot, + prepare: impl FnOnce(), + ) -> Result; fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, @@ -127,6 +149,9 @@ struct Prepared { packet: Arc, ordinal: u64, mcache: Option, + /// M5 kick register arrays and header flags, encoded with the commands. + register_arrays: Option<[[Option; 4]; 2]>, + kick_flags: Option, } struct Active { /// Publication number, shared with compute (crate::g17::timeout). @@ -151,9 +176,18 @@ struct Active { /// apart before it is republished; a single sample may straddle an update. const ABANDON_GRACE_NS: u64 = 2_000_000; +/// The firmware-specific host transaction; queue ownership remains shared. +pub(crate) enum Registration { + Neo(RenderRegistration), + M5(crate::g17::fw::m5::queue::HardwareSubmission), +} + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { + /// The firmware identity of an M5 render pool, pinned while this pair + /// uses the pool. G17P pools have fixed identities and need no pin. + pool_pin: Option, kick_aliases: Option<[Option; 2]>, memory: KBox, kicks: [kick::Queue; 2], @@ -187,6 +221,8 @@ pub(crate) struct Pair { /// Retired, never-executed groups awaiting republication on this pair, in /// their original order. Nothing else is prepared on the pair before them. replay: Tracker>, + /// Recovery generation of this pair's last executed retirement. + executed_generation: Option, /// When the oldest parked group was parked or the last one republished. replay_since: Option, /// Scheduler page of the context of the last published command: the @@ -198,6 +234,19 @@ pub(crate) struct Pair { /// Completion values of the two kicks of the last publication, kept with the configuration /// cache: what each queue holds once that publication completed. published_completions: Option<[u64; 2]>, + /// M5: the queues' firmware identity (logical context, scheduler page, + /// owner) changed since the last configuration firmware consumed. Only + /// this sets the M5 configuration's context and QoS update bits; + /// `context_update` stays the host witness that retires `previous` and + /// gates the completion seed. Cleared once a publication carried it. + identity_update: bool, + /// Owner of the identities named by an M5 cold-install configuration + /// pair, until firmware consumed it or an Active took them over. + configuration: Option, + /// Hardware context named by the configurations firmware last took. + configured_context: Option, + /// QoS slot named by those configurations. + configured_qos: Option, } impl Pair { #[inline(never)] @@ -223,6 +272,10 @@ impl Pair { descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, ) -> Result> { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS || owner == 0 @@ -239,14 +292,11 @@ impl Pair { } return Err(error); } - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters, variant.programs)?; + let layout = m5_render_submission::layout(&first.command, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; @@ -264,6 +314,7 @@ impl Pair { variant, )?; storage.write_init(kernel::try_init!(Self { + pool_pin, kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], @@ -291,11 +342,16 @@ impl Pair { previous: None, displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: 0, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -319,15 +375,22 @@ impl Pair { pub(crate) fn buffer_id(&self) -> u8 { self.memory.manager.buffer_id() } + /// The parameter-buffer state an M5 PB token is bound to. + pub(crate) fn m5_pb_state_va(&self) -> Option { + self.memory.graph.is_m5().then(|| self.memory.manager.state_va()) + } pub(crate) fn manager(&mut self) -> &mut buffer::Manager { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.len() != 0 + self.active.len() != 0 || self.replay.len() != 0 || self.configuration.is_some() } - /// A replay or an abandonment grace period needs the worker to poll. + /// Configuration consumption, a replay or an abandonment grace period + /// needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.configuration.is_some() + || self.replay.len() != 0 + || self.active.iter().any(|a| a.abandoned_since.is_some()) } pub(crate) fn replay_stalled(&self, now: u64) -> bool { self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) @@ -380,6 +443,8 @@ impl Pair { pass, attachments, ray_query, + resources: programs, + plan, .. } = &packet.command else { @@ -388,13 +453,22 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + // The TA mesh rendering buffer is bound only to kicks with an API task + // (object) stage. Resolve it before any ordinal state is claimed so an + // allocation failure fails this kick cleanly. + let mesh_storage = if programs.as_ref().is_some_and(|p| p.uses_task()) { + self.memory.mesh_storage_register(alloc)? + } else { + 0 + }; let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)?; let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory @@ -415,10 +489,12 @@ impl Pair { self.memory.lifecycle }; let (scratch, metric) = self.memory.manager.scene_registers(scene)?; - let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp_offset = self.memory.timestamp_offset(ordinal); let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; let resources = abi::RenderResources { context_id: packet.context.id().try_into()?, + // On M5 too: a directly owned execution root has one generation, which the + // render tail token names. context_generation: packet.context.generation(), pb_slot: self.buffer_id().into(), free_list_slot: self.free_list.id().into(), @@ -437,7 +513,7 @@ impl Pair { gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, - mesh_storage: 0, + mesh_storage, fragment_status_va: fragment_status, fragment_status_fw_va: fragment_fw, aux_fb_va: auxiliary, @@ -455,14 +531,14 @@ impl Pair { ordinal, ta: abi::RenderStage { qid: qids[0], - descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + descriptor_va: self.memory.graph.descriptor_client(0, ordinal)?, state_word: word, stamp_va: addresses.stamps[0][0], aux_stamp_va: addresses.stamps[0][1], }, fragment: abi::RenderStage { qid: qids[1], - descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + descriptor_va: self.memory.graph.descriptor_client(1, ordinal)?, state_word: word, stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], @@ -477,24 +553,37 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = + let result = if self.memory.graph.is_m5() { self.memory - .write_descriptors(&args, attachments, packet.context.id().try_into()?); - let mcache = match result { - Ok(mcache) => mcache, + .write_m5_commands( + &args, + &layout, + programs.as_ref().and(*plan), + programs.as_ref().map_or(0, |p| p.mesh_class()), + ) + .map(|(arrays, flags)| (None, Some(arrays), Some(flags))) + } else { + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?) + .map(|mcache| (mcache, None, None)) + }; + let (mcache, register_arrays, kick_flags) = match result { + Ok(encoded) => encoded, Err(error) => { self.memory.graph.release(ordinal)?; - self.memory.generation(ordinal)?.ordinal = None; + self.memory.generation(ordinal)?.clear(); self.memory.target.cancel(ordinal); return Err(error); } }; - self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.clear_timestamps(ordinal)?; self.memory.manager.claim_scene(ordinal, scene)?; self.prepared = Some(Prepared { packet: packet.clone(), ordinal, mcache, + register_arrays, + kick_flags, }); Ok(()) } @@ -504,28 +593,46 @@ impl Pair { /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the /// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix /// orders the tiling kick. A compute dependency always orders the tiling kick. +/// The completion classes name, for M5 kicks, what each barrier waits for: +/// 1 the tiling encoder, 2 a full channel or compute completion, 3 the +/// fragment encoder; the last class is the implicit parent's. struct RenderBarriers { - tiling: [KickDependency; 3], + tiling: [KickDependency; 5], tiling_count: usize, - fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], + fragment: [KickDependency; 4 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, + tiling_types: [u8; 6], + fragment_types: [u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS], } fn render_barriers( fragment_stage: bool, + stage_prefixes: bool, dependencies: &RenderDependencies, own_tiling: KickDependency, ) -> RenderBarriers { - let mut tiling_barriers = [KickDependency::ZERO; 3]; + let fragment_stage = fragment_stage || stage_prefixes; + let mut tiling_barriers = [KickDependency::ZERO; 5]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; + let mut fragment_barriers = [own_tiling; 4 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; + let mut tiling_types = [1; 6]; + let mut fragment_types = [3; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[0] = 1; + for dependency in &dependencies.render_tiling[..dependencies.render_tiling_count] { + tiling_barriers[tiling_count] = *dependency; + tiling_types[tiling_count] = 2; + tiling_count += 1; + } for dependency in dependencies.render() { if fragment_stage { fragment_barriers[fragment_count] = *dependency; fragment_count += 1; } else { tiling_barriers[tiling_count] = *dependency; + // A full channel barrier uses the tiling encoder's class 2. + // Class 3 belongs to the fragment encoder. + tiling_types[tiling_count] = 2; tiling_count += 1; } } @@ -542,8 +649,31 @@ fn render_barriers( } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; + tiling_types[tiling_count] = 2; tiling_count += 1; } + if let Some(compute) = dependencies.fragment_compute { + fragment_barriers[fragment_count] = compute; + fragment_types[fragment_count] = 2; + fragment_count += 1; + } + if stage_prefixes { + // Only identical words of the same class are interchangeable. + let mut kept = 0; + for read in 0..tiling_count { + if (0..kept).any(|i| { + tiling_barriers[i].word() == tiling_barriers[read].word() + && tiling_types[i] == tiling_types[read] + }) { + continue; + } + tiling_barriers[kept] = tiling_barriers[read]; + tiling_types[kept] = tiling_types[read]; + kept += 1; + } + tiling_count = kept; + tiling_types[kept] = 1; + } // Only tile stores overlap: the tiling kick stays concurrent. for hazard in &dependencies.hazards[..dependencies.hazard_count] { fragment_barriers[fragment_count] = *hazard; @@ -554,6 +684,8 @@ fn render_barriers( tiling_count, fragment: fragment_barriers, fragment_count, + tiling_types, + fragment_types, } } @@ -573,12 +705,24 @@ impl Pair { channels::WorkSlot, kick::{KickArgs, KickQos}, queue::{ - self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, - TilingBinding, + FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, TilingBinding, }, }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + let m5 = self.memory.graph.is_m5(); + let context_id = packet.context.id(); + let qos_slot = packet.context.qos_id(); + if m5 { + if self.configured_context.is_some_and(|configured| configured != context_id) + || self.configured_qos.is_some_and(|configured| configured != qos_slot) + { + self.identity_update = true; + } + if lease.buffer_context.is_some_and(|bound| bound != context_id) { + return Err(EBUSY); + } + } self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; @@ -593,7 +737,13 @@ impl Pair { let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; - let work_slot = work.render_slot(self.slot)?; + // A channel moves to this pair only after every publication made through it is + // visible on both of its rings (the stamps cover ordinal + 1). + let work_slot = work.render_channel(self.slot, |old, published| { + host.stamps(old).is_ok_and(|stamps| { + stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) + }) + })?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; let stamps = host.stamps(qids)?; @@ -627,9 +777,11 @@ impl Pair { data_master: DataMaster::Fragment as u8, }, ]; + // M5 carries the identity update bits only on a changed identity. + let identity_update = if m5 { self.identity_update } else { self.context_update }; let mut configs = [None; 2]; for stage in 0..2 { - configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + let args = QueueConfigArgs { target: if stage == 0 { QueueConfigTarget::Tiling(Some(TilingBinding { pb_va: self.memory.manager.state_va(), @@ -644,11 +796,13 @@ impl Pair { kick_ring_fw_va: self.kicks[stage].firmware_va(), qid: qids[stage], install: !self.installed[stage], - completion_seed: covered[stage].then(|| { - kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)) - }), + completion_seed: (covered[stage] + && (!m5 || !self.installed[stage] || self.context_update)) + .then(|| { + kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)) + }), context_id: context.id().try_into()?, free_list: FreeListBinding { va: self.free_list.control_va(), @@ -658,12 +812,35 @@ impl Pair { scheduler_va: context.scheduler_va(), policy, qos_slot: qos, - qos_update: self.context_update, + qos_update: identity_update, owner_pid: context.owner_pid(), - context_update: self.context_update, - })?); + context_update: identity_update, + }; + configs[stage] = Some(if m5 { + crate::g17::fw::m5::queue::config( + &args, + crate::g17::fw::m5::queue::ConfigProfile { + soft_fault_enabled: true, + extended: false, + parameter_manager: (stage == 0).then(|| u32::from(self.buffer_id())), + }, + )? + } else { + QueueConfig::new(&args)? + }); } let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + if m5 && self.registered.is_none() { + if self.installed == [true; 2] { + // The cold-install configuration pair must be consumed first. + let cursors = self.memory.graph.cursors()?; + if cursors.iter().zip(self.cursors).any(|(c, end)| c.read != end) { + return Err(EBUSY); + } + } else { + return self.publish_m5_configuration(&packet, configs, kicks, policy, ring, host); + } + } let cache = if self.recovery_generation == epoch { self.configs } else { @@ -695,10 +872,7 @@ impl Pair { if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); } - let queues = qids.map(|qid| { - self.memory.graph.queues.gpu_va() - + u64::from(qid) * size_of::() as u64 - }); + let queues = qids.map(|qid| self.memory.graph.queue_va(qid)); let Validated::Render { pass, ray_query, .. } = &packet.command @@ -706,13 +880,25 @@ impl Pair { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let barriers = + render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; + let barrier_types = [ + &barriers.tiling_types[..barriers.tiling_count + 1], + &barriers.fragment_types[..barriers.fragment_count + 1], + ]; + let prepared = self.prepared.as_ref().ok_or(EIO)?; + let m5_kick = match (prepared.register_arrays, prepared.kick_flags) { + (Some(arrays), Some(flags)) if m5 => Some((arrays, flags)), + (None, None) if !m5 => None, + _ => return Err(EIO), + }; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], parent: parents[stage], barriers, + dependency_types: m5.then_some(barrier_types[stage]), descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), queue_va: queues[stage], qos: KickQos { @@ -724,38 +910,60 @@ impl Pair { }, }, mcache, - event_mask: abi::RENDER_KICK_EVENT_MASK, + // M5 tiling kicks also retain the last event of the mask. + event_mask: if m5 && stage == 0 { + [0, 0, 0, 1u64 << 62] + } else { + abi::RENDER_KICK_EVENT_MASK + }, register_arrays: arrays, - scs_enabled: ray_query[stage].scratch.enabled(), - dependency_types: None, - compute_matrix: false, + scs_enabled: match m5_kick { + Some((_, flags)) => flags.scratch[stage], + None => ray_query[stage].scratch.enabled(), + }, + compute_matrix: m5_kick.is_some_and(|(_, flags)| flags.matrix[stage]), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings( - self.memory.graph.descriptor_client(0, ordinal), - ray_query[0].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[0], + None => abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal)?, + ray_query[0].scratch.enabled(), + self.variant.programs, + )?, + }, None, ); let fragment_args = args( 1, &barriers.fragment[..barriers.fragment_count], - abi::FragmentDescriptor::register_bindings( - self.memory.graph.descriptor_client(1, ordinal), - ray_query[1].scratch.enabled(), - self.variant.programs, - )?, + match m5_kick { + Some((arrays, _)) => arrays[1], + None => abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal)?, + ray_query[1].scratch.enabled(), + self.variant.programs, + )?, + }, mcache, ); - let registration = RenderRegistration::new( - (qids[1], self.kicks[1].low_va()), - (qids[0], self.kicks[0].low_va()), - policy.priority(), - )?; + let registration = if m5 { + Registration::M5(crate::g17::fw::m5::queue::HardwareSubmission::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + true, + )?) + } else { + Registration::Neo(RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + policy.priority(), + )?) + }; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; @@ -807,10 +1015,16 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; - let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); - let inner = memory.graph.support.word(0x28000)?; - let next_inner = (next_ordinal as u32).wrapping_mul(2); + // M5 graphs keep no G17P event and inner-producer words. + let legacy_state = if m5 { + None + } else { + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + Some((event_slot, next_event, inner, next_inner)) + }; let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; let published = host.next_publication()?; @@ -828,7 +1042,11 @@ impl Pair { previous, }, kicks, - context_update: self.context_update, + // On M5 the cold-install configuration may already have + // cleared context_update: the first publication after a + // rebind still retires an unpublished `previous`. + context_update: self.context_update + || (m5 && self.previous.as_ref().is_some_and(|p| p.publication.is_none())), failed: false, drained_result: None, abandoned_since: None, @@ -837,9 +1055,25 @@ impl Pair { .map_err(|_| EBUSY)?; self.prepared = None; packet.mark_published(); - lanes[1].publish_prefix(); + work.channel_published(self.slot, qids, next_ordinal); + // The Active now retains this Packet and its identity and QoS pins; it + // replaces the configuration-only identity owner. + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } + // M5 fills both lanes now; the host-port transaction precedes either + // producer, which publish_inner advances. + if m5 { + for lane in &lanes { + lane.prepare(); + } + } else { + lanes[1].publish_prefix(); + } fragment_kick.commit(); - lanes[1].publish_kick(); + if !m5 { + lanes[1].publish_kick(); + } tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { @@ -864,7 +1098,15 @@ impl Pair { pm_submission.commit(); Ok(()) }, - || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + || match legacy_state { + Some((event_slot, next_event, inner, next_inner)) => { + lanes[0].publish_tiling(event_slot, next_event, inner, next_inner) + } + None => { + lanes[1].publish_kick(); + lanes[0].publish_kick(); + } + }, ) })(); let outer_next = match result { @@ -907,6 +1149,13 @@ impl Pair { self.recovery_generation = epoch; self.installed = [true; 2]; self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); + if m5 { + // Both configurations of this publication carried any update bits, + // so firmware applies them before this kick. + self.configured_context = Some(context_id); + self.configured_qos = Some(qos_slot); + self.identity_update = false; + } self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -927,6 +1176,90 @@ impl Pair { ) } + /// M5 cold install: the queues consume their initial configuration + /// records, published as one configuration pair, before the first kick + /// pair. Returns EBUSY so the command publishes once they are consumed. + fn publish_m5_configuration( + &mut self, + packet: &Arc, + configs: [crate::g17::fw::queue::QueueConfig; 2], + kicks: [KickTimestamp; 2], + policy: crate::g17::fw::queue::Policy, + ring: u8, + host: &mut impl Host, + ) -> Result { + use crate::g17::fw::queue::ITEM_RING_ENTRIES; + if self.configuration.is_some() { + return Err(EBUSY); + } + let ordinal = self.ordinal; + let qids = self.qids(); + let lanes = [ + publication::Lane::new( + &self.memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + true, + kicks[0], + false, + )?, + publication::Lane::new( + &self.memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + true, + kicks[1], + false, + )?, + ]; + let ends = self.cursors.map(|base| (base + 1) % ITEM_RING_ENTRIES); + let slot = |stage: usize| -> Result { + WorkSlot::new( + DataMaster::Compute, + self.memory.graph.queue_va(qids[stage]), + qids[stage], + ends[stage].try_into()?, + true, + ) + }; + let (tiling, fragment) = (slot(0)?, slot(1)?); + self.configuration = Some(configuration::Configuration::new( + &packet.context, + ends, + host.generation(), + )?); + let configuration = self.configuration.as_mut().ok_or(EIO)?; + let configured = + host.publish_configuration_pair(policy.priority(), &fragment, &tiling, || { + // Ownership exists before either item producer can become visible. + configuration.mark_visible(); + for lane in &lanes { + lane.publish_prefix(); + } + }); + if let Err(error) = configured { + if self.configuration.as_ref().is_some_and(|owner| !owner.visible()) { + self.configuration.take().ok_or(EIO)?.release(); + } + return Err(error); + } + self.cursors = ends; + self.installed = [true; 2]; + self.configured_context = Some(packet.context.id()); + self.configured_qos = Some(packet.context.qos_id()); + self.configs = [Some(configs[0]), Some(configs[1])]; + self.context_update = false; + self.identity_update = false; + self.outer_started |= ring; + packet.context.mark_published(); + host.notify_render(policy.priority())?; + Err(EBUSY) + } + /// Everything that must hold before a publication changes any state: no alias work /// pending, a healthy device epoch, room in the active ring, this slot's packet, the /// prepared descriptors for this ordinal, a clean completion status and a registration @@ -1043,6 +1376,9 @@ impl retirement::Memory for Observer<'_, H> { fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { self.memory.pass(ordinal) } + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + self.memory.stage_timestamps(ordinal) + } } impl Pair { @@ -1225,10 +1561,9 @@ impl Pair { /// Sample PM feedback only after this pair's exact retirement. Busy pairs /// keep their current capacity and let firmware request growth as needed. pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { - let Validated::Render { pass, .. } = &packet.command else { - return Err(EINVAL); - }; - let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; + let required = + m5_render_submission::layout(&packet.command, self.clusters, self.variant.programs)? + .tvb_blocks; self.memory .manager .note_required(required, crate::g17::now_ns()); @@ -1398,6 +1733,19 @@ impl Pair { self.retire_front(None, state, Some(observed), host, defer) } + /// The M5 cold-install configuration pair was consumed exactly: its + /// identity owner is released. + fn retire_configuration(&mut self, host: &impl Host) -> Result { + let Some(configuration) = self.configuration.as_ref() else { + return Ok(false); + }; + if !configuration.consumed(self.memory.graph.cursors()?, host.generation()) { + return Ok(false); + } + self.configuration.take().ok_or(EIO)?.release(); + Ok(true) + } + fn retire_front( &mut self, masks: Option<[u64; 2]>, @@ -1405,6 +1753,19 @@ impl Pair { observed: Option, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let configuration_retired = state == 0 && self.retire_configuration(host)?; + let retired = self.retire_front_command(masks, state, observed, host, defer)?; + Ok(retired || configuration_retired) + } + + fn retire_front_command( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { if self.quarantined || state != 0 || !self.selected(masks) { return Ok(false); @@ -1486,9 +1847,25 @@ impl Pair { ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + self.executed_generation = Some(host.generation()); Ok(true) } + /// An executed group of this pair retired in `generation` or later. + pub(crate) fn executed_since(&self, generation: u64) -> bool { + self.executed_generation.is_some_and(|executed| executed >= generation) + } + /// Published work of this pair is still live: in flight and not parked. + pub(crate) fn live_unparked(&self) -> bool { + self.replay.len() == 0 && self.active.len() != 0 + } + /// The oldest parked group precedes a command waiting through `through`. + pub(crate) fn replay_through(&self, through: Option) -> bool { + self.replay + .front() + .is_some_and(|packet| crate::g17::job::Order::contains(through, packet.order.sequence)) + } + /// Every retained group was retired and unlinked by firmware without /// executing (job list empty, stamps cover it, no recovery, fragment end /// never written): after another VM's recovery the firmware drops an @@ -2039,6 +2416,9 @@ impl Pair { /// Both processors have stopped before the caller reaches this method. /// VM ownership settles before quarantined admission is rehabilitated. pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(configuration) = self.configuration.take() { + configuration.release(); + } if let Some(prepared) = self.prepared.as_ref() { defer(Deferred::Unpublished( prepared.packet.completion.clone(), @@ -2120,9 +2500,7 @@ fn install_memory( descriptors.word(offset + 12)?, ]; let previous = words.map(|word| word.load(Ordering::Relaxed)); - let pages: u32 = (memory.manager.blocks() - * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) - .try_into()?; + let pages = memory.manager.page_count()?; let next = crate::g17::fw::buffer::reload_descriptor( previous, memory.manager.page_list_va(), @@ -2162,6 +2540,7 @@ fn install_memory( /// A binding prepared while the registry exclusively lends the idle pair. /// Its frontier prevents a stale off-lock result from replacing another owner. pub(crate) struct PreparedBinding { + pool_pin: Option, previous_context: Arc, previous_owner: Option, ordinal: u64, @@ -2183,8 +2562,14 @@ pub(crate) struct RetiredBinding { buffer_id: u8, publication: Option, ready: bool, + /// An M5 same-context pool rebind: the old pool stays until firmware + /// acknowledged its release. + wait_for_pool_release: bool, } impl RetiredBinding { + pub(crate) fn wait_for_pool_release(&self) -> bool { + self.wait_for_pool_release + } pub(crate) fn context(&self) -> &Arc { &self.context } @@ -2202,6 +2587,7 @@ impl Pair { return; } self.context_update = false; + self.identity_update = false; if let Some(previous) = self.previous.as_mut() { if previous .publication @@ -2330,6 +2716,28 @@ impl Pair { && previous_ready && self.idle_for_context(self.context())?) } + /// An M5 pool may be replaced without changing the pair's live logical + /// owner. Commands already using the pair must drain, and an earlier + /// replacement must have retired, unless nothing was published since it: + /// that binding never reached firmware and is superseded directly. + pub(crate) fn bindable(&self, owner: u64, context: &Arc) -> Result { + if self.memory.graph.is_m5() + && self.owner == Some(owner) + && Arc::ptr_eq(context, self.context()) + { + return Ok(self.displaced.is_none() + && self.previous_supersedable() + && self.idle_for_context(context)?); + } + self.reusable() + } + /// No `previous`, or one that nothing has been published after: a warm + /// (memoryless) replacement with no publication ordinal and not ready. + fn previous_supersedable(&self) -> bool { + self.previous.as_ref().is_none_or(|p| { + p.memory.is_none() && p.publication.is_none() && !p.ready && self.context_update + }) + } /// Logical close leaves the physical channel identity and backing retained. pub(crate) fn release_owner(&mut self) { self.owner = None; @@ -2351,7 +2759,11 @@ impl Pair { pm_generation: Option, first: &Arc, ) -> Result { - if !self.reusable()? + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } + if !self.bindable(owner, &context)? || owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() @@ -2359,8 +2771,15 @@ impl Pair { return Err(EBUSY); } alloc.uat.vm_context_mask(context.vm())?; - let same_vm = Arc::ptr_eq(self.context().status(), context.status()); - let (memory, aliases, kicks) = if same_vm { + // On M5 only the same logical context (a pool rebind) keeps the graph: + // any other context gets its own graph and queue records, and a live + // queue record is never retargeted. + let keep_graph = if self.memory.graph.is_m5() { + Arc::ptr_eq(self.context(), &context) + } else { + Arc::ptr_eq(self.context().status(), context.status()) + }; + let (memory, aliases, kicks) = if keep_graph { if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { return Err(EINVAL); } @@ -2369,10 +2788,8 @@ impl Pair { let metrics = metrics.ok_or(EINVAL)?; let buffer_id = buffer_id.ok_or(EINVAL)?; let pm_generation = pm_generation.ok_or(EINVAL)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = + m5_render_submission::layout(&first.command, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -2393,6 +2810,7 @@ impl Pair { (Some(memory), Some(persistent), Some(aliases)) }; Ok(PreparedBinding { + pool_pin, previous_context: self.context().clone(), previous_owner: self.owner, ordinal: self.ordinal, @@ -2411,7 +2829,7 @@ impl Pair { alloc: &Allocator<'_>, descriptors: &crate::g17::object::KernelObject, ) -> Result { - if !self.reusable()? + if !self.bindable(binding.owner, &binding.context)? || binding.ordinal != self.ordinal || binding.previous_owner != self.owner || !Arc::ptr_eq(&binding.previous_context, self.context()) @@ -2420,20 +2838,44 @@ impl Pair { return Err(EBUSY); } let foreign = binding.memory.is_some(); + let same_context = Arc::ptr_eq(self.context(), &binding.context); if let Some(memory) = binding.memory.as_mut() { - install_memory( + // A VM holds one graph's pool mappings per slot. On M5 a new + // context of the same VM gets its own graph: move the drained + // outgoing graph's mappings out first, and back if the install fails. + let same_vm = Arc::ptr_eq(self.context().status(), binding.context.status()); + let moved = if self.memory.graph.is_m5() && same_vm { + self.memory.take_pool_mappings() + } else { + None + }; + let installed = install_memory( alloc, descriptors, self.slot, memory, &mut binding.aliases, &mut binding.kicks, - )?; + ); + if let Err(error) = installed { + if let Some((slot, mappings)) = moved { + self.memory.put_pool_mappings(slot, mappings)?; + } + return Err(error); + } + drop(moved); } else { - self.memory.set_owner(self.qids(), &binding.context)?; - } + // The same logical context keeps its M5 queue records: their host + // words already name it, and a rewrite would store firmware-owned + // words back while the previous kick may still execute. + self.memory.set_owner(self.qids(), &binding.context, !same_context)?; + } + let wait_for_pool_release = self.memory.graph.is_m5() + && same_context + && !Arc::ptr_eq(&self.free_list, &binding.pool); let old_context = self.context().clone(); let old_buffer = self.buffer_id(); + self.pool_pin = binding.pool_pin.take(); let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); let old_memory = binding .memory @@ -2446,6 +2888,7 @@ impl Pair { buffer_id: old_buffer, publication: None, ready: foreign, + wait_for_pool_release, }; if foreign { self.displaced = Some(retired); @@ -2459,6 +2902,9 @@ impl Pair { self.graph_started = false; } self.context_update = true; + if foreign || !same_context { + self.identity_update = true; + } if foreign { self.cursors = [0; 2]; self.configs = [None; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs new file mode 100644 index 00000000000000..856a82b82a7677 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/configuration.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initial M5 configuration can reach firmware before an Active command. +//! Its captured identities therefore have an independent owner. No aggregate +//! work-state node is referenced by this configuration-only item prefix. + +use super::{retirement::Cursors, Context}; +use crate::mmu::ContextPin; +use kernel::{prelude::*, sync::Arc}; + +pub(super) struct Configuration { + context: Arc, + pin: ContextPin, + qos: u8, + targets: [u32; 2], + generation: u64, + visible: bool, +} +impl Configuration { + pub(super) fn new(context: &Arc, targets: [u32; 2], generation: u64) -> Result { + let pin = context.pin()?; + let qos = match context.pin_qos() { + Ok(qos) => qos, + Err(error) => { context.unpin(pin); return Err(error); } + }; + Ok(Self { context: context.clone(), pin, qos, targets, generation, visible: false }) + } + pub(super) fn mark_visible(&mut self) { self.visible = true; } + pub(super) fn visible(&self) -> bool { self.visible } + pub(super) fn consumed(&self, cursors: [Cursors; 2], generation: u64) -> bool { + self.visible && self.generation == generation && cursors.iter().zip(self.targets).all(|(cursor, target)| { + cursor.done == target && cursor.read == target && cursor.write == target + }) + } + /// Only before visibility, after exact configuration consumption, after + /// transfer to an Active owning the same pinned Packet, or processor stop. + pub(super) fn release(self) { + self.context.unpin_qos(self.qos); + self.context.unpin(self.pin); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs new file mode 100644 index 00000000000000..443b8d0d0d5b32 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5 render admission, target layout and command inputs on the shared submission path. + +use kernel::prelude::*; +use crate::g17::fw::m5::{render as cmd, render_registers as regs}; +use crate::g17::{buffer::target, command::Validated, fw::render::{RenderArgs, RenderPass, RegisterPrograms}}; + +/// Compressed single-layer MSAA ZLS admission: D32 depth with optional plain +/// stencil, and the compressed Z/S form. +pub(crate) fn depth_msaa_compression_enabled() -> bool { + crate::module_parameters::m5_depth_msaa_compression.value() == 1 +} + +pub(crate) fn validate(pass: &RenderPass) -> Result { + // The hardware carries the sample stride in eight bits. The 64-byte limit + // belonged to the first qualified profile, not the M5 imageblock. + let single_sample = pass.samples_log2 == 0 + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size % 8 == 0 + && u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) <= 65536 + && (pass.sample_size != 0 || pass.layers == 1); + // ZLS addresses/actions and page strides are sample-independent. + // Retain the checked plain layout; compression is not admitted here, + // except in the layered compressed Z/S form (every enabled Z/S load or + // store compressed, a compression address per compressed plane, depth + // and stencil strides as supplied by userspace). The same form in + // single-layer 2x/4x MSAA depth/stencil passes is admitted only with the + // opt-in. + let plain_zls = pass.zls_control == 0 + || (pass.zls_control & 0x154 == 0 + && pass.depth.compression_va == 0 && pass.stencil.compression_va == 0); + let multisample = (plain_zls || layered_compressed_zls(pass) + || (depth_msaa_compression_enabled() && (single_layer_compressed_depth_zls(pass) + || single_layer_compressed_zls(pass)))) + && matches!(pass.samples_log2, 1..=3) + && matches!((pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16)) + && pass.sample_size >= 8 && pass.sample_size % 8 == 0 + && (u32::from(pass.utile_width) * u32::from(pass.utile_height) + << pass.samples_log2) <= 2048 + && (u32::from(pass.sample_size) * u32::from(pass.utile_width) + * u32::from(pass.utile_height) << pass.samples_log2) <= 65536; + if !(single_sample || multisample) + || (pass.zls_control != 0 && pass.depth.va == 0 && pass.stencil.va == 0) { + return Err(ENOTSUPP); + } + Ok(()) +} + +/// Layered multisample depth/stencil in the compressed Z/S form. +fn layered_compressed_zls(pass: &RenderPass) -> bool { + pass.layers > 1 && compressed_zls_form(pass) +} + +/// Single-layer 2x/4x depth/stencil in the compressed Z/S form, the same form +/// as the layered one, compressed stencil included. +fn single_layer_compressed_zls(pass: &RenderPass) -> bool { + pass.layers == 1 && matches!(pass.samples_log2, 1 | 2) && compressed_zls_form(pass) +} + +/// Multisample depth/stencil in the compressed Z/S form: each enabled load or +/// store of a plane is compressed (ZLS bits: Z load 15 / compress 2, S load 14 +/// / compress 4, Z store 19 / compress 6, S store 18 / compress 8; the depth +/// format bits 25:26 are allowed), a plane +/// with any compression has a compression address, and no other ZLS bits. +fn compressed_zls_form(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_LOAD_COMPRESS: u64 = 1 << 2; + const S_LOAD_COMPRESS: u64 = 1 << 4; + const Z_STORE_COMPRESS: u64 = 1 << 6; + const S_STORE_COMPRESS: u64 = 1 << 8; + /// Depth format (bits 25:26) is a property of the plane, not of layering. + const Z_FORMAT: u64 = 3 << 25; + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_LOAD_COMPRESS | S_LOAD_COMPRESS + | Z_STORE_COMPRESS | S_STORE_COMPRESS | Z_FORMAT; + let control = pass.zls_control; + let z_compressed = control & (Z_LOAD_COMPRESS | Z_STORE_COMPRESS) != 0; + let s_compressed = control & (S_LOAD_COMPRESS | S_STORE_COMPRESS) != 0; + control != 0 + && control & !KNOWN == 0 + && (control & Z_LOAD == 0 || control & Z_LOAD_COMPRESS != 0) + && (control & S_LOAD == 0 || control & S_LOAD_COMPRESS != 0) + && (control & Z_STORE == 0 || control & Z_STORE_COMPRESS != 0) + && (control & S_STORE == 0 || control & S_STORE_COMPRESS != 0) + && (!z_compressed || (pass.depth.va != 0 && pass.depth.compression_va != 0)) + && (!s_compressed || (pass.stencil.va != 0 && pass.stencil.compression_va != 0)) + && (z_compressed || pass.depth.compression_va == 0) + && (s_compressed || pass.stencil.compression_va == 0) +} + +/// Single-layer D32 depth compression, with optional plain S8. +/// The compressed Z/S form is checked separately. Both depth compression bits +/// are required because partial rendering forces depth load and store even +/// when one of those operations is disabled in the final render pass. +fn single_layer_compressed_depth_zls(pass: &RenderPass) -> bool { + const Z_LOAD: u64 = 1 << 15; + const S_LOAD: u64 = 1 << 14; + const Z_STORE: u64 = 1 << 19; + const S_STORE: u64 = 1 << 18; + const Z_COMPRESS: u64 = (1 << 2) | (1 << 6); + const KNOWN: u64 = Z_LOAD | S_LOAD | Z_STORE | S_STORE | Z_COMPRESS; + pass.layers == 1 + && matches!(pass.samples_log2, 1 | 2) + && pass.zls_control & !KNOWN == 0 + && pass.zls_control & Z_COMPRESS == Z_COMPRESS + && pass.depth.va != 0 && pass.depth.compression_va != 0 + && pass.stencil.compression_va == 0 + && (pass.zls_control & (S_LOAD | S_STORE) == 0 || pass.stencil.va != 0) +} + +pub(crate) fn parameter_geometry(blocks: usize) -> Result { + let mut info = core::mem::MaybeUninit::::zeroed(); + // SAFETY: The complete sysinfo is initialized and writable. + unsafe { kernel::bindings::si_meminfo(info.as_mut_ptr()); } + // SAFETY: The object was initialized before si_meminfo filled it. + let info = unsafe { info.assume_init() }; + let ram = (info.totalram as u64).checked_mul(u64::from(info.mem_unit)).ok_or(EOVERFLOW)?; + let initial = blocks.checked_mul(0x80000).ok_or(EOVERFLOW)?; + crate::g17::buffer::BlockGeometry::m5(ram, initial as u64) +} + +/// The target layout of a render: G17P's for a command without an M5 plan. +pub(crate) fn layout(command: &Validated, clusters: u32, programs: RegisterPrograms) + -> Result { + let Validated::Render { pass, plan, .. } = command else { return Err(EINVAL); }; + let layout = target::Layout::new(pass, clusters, programs)?; + if plan.is_none() { return Ok(layout); } + validate(pass)?; + // One target group per cluster unless the pass asks for few primitives. + // Both target storage and the segment controls use the physical cluster count. + let geometry = parameter_geometry(layout.tvb_blocks)?; + // The auxiliary tables cover the complete parameter inventory, including + // future growth, rather than just the initially resident physical blocks. + let bytes = geometry.capacity.checked_mul(geometry.block_bytes).ok_or(EOVERFLOW)?; + let groups = if pass.few_primitives { 1 } else { clusters }; + layout.with_m5_slices(pass, groups, 64, bytes) +} + +pub(crate) fn controls(pass: &RenderPass) -> Result { + validate(pass)?; + let stage = |resource_flags| regs::M5RenderStageControls { + scs: crate::g17::render_resources::StageScratch::NONE, + resource_flags, resource_control: 0x200000001, ptc_max: 96, work_key: 0, + }; + Ok(regs::M5RenderControls { + fragment_event_control: regs::fragment_event_control(true, 1), + ta: stage(0x4054b), fragment: stage(0x104054b), + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: pass.eot_rsrc_spec << 32, + store_pipeline_2: pass.usc_base + u64::from(pass.eot_offset & !7), + partial_store_pipeline_2_bind: pass.partial_eot_rsrc_spec << 32, + partial_store_pipeline_2: pass.usc_base + u64::from(pass.partial_eot_offset & !7), + partial_tile_flags: 0x200000, + aux_fb_flags: 0x10c000 | u64::from(pass.few_primitives) + | (u64::from(pass.depth_bias_is_int) << 18), + aux_fb_control: 0x100000, + }) +} + +pub(crate) fn inputs(args: &RenderArgs<'_>, usage: [u64; 2]) -> Result { + let r = args.resources; + let mut fields = cmd::M5RenderCommandInputs::default(); + let token = crate::g17::fw::m5::render_tail::context_token(u32::from(r.context_id), + u64::from(r.context_generation)).ok_or(EINVAL)?; + fields.ta.context_token = token; + fields.fragment.context_token = token; + fields.ta.resolved_tail_value = usage[0]; + fields.fragment.resolved_tail_value = Some(usage[1]); + fields.scheduler_record = r.kick_record_va; + // The compiler's external-write class of the render's graphics programs. + fields.ta.shader_write_class = cmd::shader_write_class(args.pass.shader_writes_memory); + fields.ta.sequence = args.ordinal.checked_mul(2).and_then(|n| n.checked_add(1)).ok_or(EOVERFLOW)?; + fields.fragment.sequence = args.ordinal.checked_mul(2).ok_or(EOVERFLOW)?; + fields.fragment.pass = 1; + fields.ta.predecessor_id = r.ta_ids.predecessor; + fields.fragment.predecessor_id = r.fragment_ids.predecessor; + fields.fragment.fragment_word_n = 1; + fields.fragment.fragment_flag_c = 1; + fields.fragment.binding_pairs[22][1] = 0x04040404; + // Pass policy is filled by apply_pass alongside its register copies. + fields.fragment.empty_tile_flags = [0; 4]; + fields.ta.timestamp_va = r.timestamp_va; + fields.ta.diagnostic_ta_timestamp_upper = r.timestamp_va + 8; + fields.fragment.timestamp_va = [r.timestamp_va + 16, r.timestamp_va + 24]; + let stamp = (args.ordinal as u32).wrapping_add(1).wrapping_shl(8); + fields.ta.linked_3d = cmd::M5TaLinked3d { + queue_index: u32::from(args.fragment.qid), stamp, ..Default::default() + }; + let completion = |stage: &crate::g17::fw::render::RenderStage, key| cmd::M5StageCompletion { + stamp_addresses: [stage.stamp_va, stage.aux_stamp_va], stamp, + queue_index: u32::from(stage.qid), ordinal: args.ordinal as u32, + key, ..Default::default() + }; + fields.ta.completion = completion(&args.ta, u64::from(r.ta_ids.current)); + fields.fragment.completion = completion(&args.fragment, u64::from(r.fragment_ids.current)); + fields.ta.pb_slot = r.pb_slot; + fields.ta.pb_state = r.pb_va; + fields.fragment.pm_state = r.pb_va; + fields.ta.pm_scene = r.scene_va; + fields.fragment.pm_scene = r.scene_va; + fields.ta.scene_end_va = r.scene_end_va; + fields.fragment.scene_end_va = r.scene_end_va; + fields.ta.uma_state = r.free_list_control_va; + fields.fragment.uma_state = r.free_list_control_va; + fields.ta.tail_flag = 1; + fields.fragment.tail_flag = 1; + Ok(fields) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 90a11ca977684e..189b67010ec5c1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// Usage records of one M5 command, both stages. +const M5_USAGE_BYTES: usize = 0x40; +const M5_USAGE_PAIR_BYTES: usize = 2 * M5_USAGE_BYTES; /// The lead command of a stage (InitBM before tiling), after the stage's configuration record /// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; @@ -50,7 +53,7 @@ impl Aliases { vm: &mmu::Vm, source: core::ops::Range, ) -> Result { - let mut range = dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let size = (source.end - source.start) as u64; loop { // Descriptor aliases are read-only to GPU consumers; the canonical @@ -91,7 +94,8 @@ impl Aliases { /// Stable queue records and rotating descriptors/configuration records. pub(super) struct Graph { - aliases: [Aliases; 2], + /// One alias of all M5 command storage, or the G17P tiling and fragment ones. + aliases: [Option; 2], pub(super) descriptors: PooledObject, pub(super) queues: PooledObject, pub(super) config: PooledObject, @@ -99,8 +103,15 @@ pub(super) struct Graph { pub(super) owners: [Option; RECORDS], /// Host words of the pointer blocks in this firmware build. pub(super) pointers: fw::variant::PointerLayout, + m5: bool, } impl Graph { + pub(super) fn is_m5(&self) -> bool { self.m5 } + + /// Out of line: inlined, its temporaries stay reserved in `Memory::new`'s frame while the + /// rest of the pair (`buffer::Manager::new` and its allocations) is built below it, which + /// overflowed the submit ioctl's kernel stack. + #[inline(never)] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -109,21 +120,28 @@ impl Graph { pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; + let m5 = alloc.uat.is_m5(); + // M5 selects cached firmware attributes at initial mapping creation; + // write-combined CPU stores also support the uncached capacity fallback. + let cpu = if m5 { CpuMap::WriteCombined } else { CpuMap::WriteBack }; + let bytes = if m5 { RECORDS * fw::m5::render_assembly::COMMAND_STORAGE_BYTES } + else { TA_BYTES + FRAGMENT_BYTES }; let descriptors = pool.allocate( alloc, Family::Descriptor, - TA_BYTES + FRAGMENT_BYTES, - TA_BYTES + FRAGMENT_BYTES, + bytes, + bytes, prot, - CpuMap::WriteBack, + cpu, )?; - let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; - let fragment = Aliases::new( + let ta = Aliases::new(&descriptors, alloc, context.vm(), + 0..if m5 { bytes } else { TA_BYTES })?; + let fragment = if m5 { None } else { Some(Aliases::new( &descriptors, alloc, context.vm(), TA_BYTES..TA_BYTES + FRAGMENT_BYTES, - )?; + )?) }; let mut queues = pool.allocate( alloc, Family::QueueState, @@ -138,27 +156,33 @@ impl Graph { 0x8000, 0x8000, prot, - CpuMap::WriteBack, + cpu, )?; let support = alloc.kernel( - storage::SUPPORT_SIZE, + if m5 { mmu::UAT_PGSZ } else { storage::SUPPORT_SIZE }, mmu::UAT_PGSZ as u64, prot, CpuMap::WriteCombined, )?; let address = queues.gpu_va(); for stage in 0..2 { - queues.write( - qids[stage] as usize * size_of::(), - queue::QueueRecord::new(&queue::QueueRecordArgs { - pointers_va: address + POINTERS[stage] as u64, - items_va: address + ITEMS[stage] as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }), - )?; + let args = queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + policy: context.policy(), + }; + if m5 { + queues.write(qids[stage] as usize * size_of::(), + fw::m5::queue::QueueRecord::new(&args)?)?; + } else { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&args), + )?; + } queues.initialize::(POINTERS[stage], |view| { *view = queue::PointerBlock::new(&pointers)?; Ok(()) @@ -166,33 +190,67 @@ impl Graph { } queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; Ok(Self { - aliases: [ta, fragment], + aliases: [Some(ta), fragment], descriptors, queues, config, support, owners: [None; RECORDS], pointers, + m5, }) } - pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + pub(super) fn queue_offset(&self, qid: u8) -> usize { + usize::from(qid) * if self.m5 { size_of::() } + else { size_of::() } + } + pub(super) fn queue_va(&self, qid: u8) -> u64 { + self.queues.gpu_va() + self.queue_offset(qid) as u64 + } + pub(super) fn offset(&self, stage: usize, ordinal: u64) -> usize { let slot = ordinal as usize % RECORDS; - if stage == 0 { + if self.m5 { + slot * fw::m5::render_assembly::COMMAND_STORAGE_BYTES + + stage * fw::m5::render_assembly::FRAGMENT_OFFSET + } else if stage == 0 { slot * abi::TA_DESCRIPTOR_SIZE } else { TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE } } pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { - self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + self.descriptors.gpu_va() + self.offset(stage, ordinal) as u64 } - pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> Result { + if self.m5 { + return Ok(self.aliases[0].as_ref().ok_or(EIO)?.client.iova() + + self.offset(stage, ordinal) as u64); + } let stride = if stage == 0 { abi::TA_DESCRIPTOR_SIZE } else { abi::FRAGMENT_SLOT_SIZE }; - self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + Ok(self.aliases[stage].as_ref().ok_or(EIO)?.client.iova() + + (ordinal as usize % RECORDS * stride) as u64) + } + /// The command storage of `ordinal` on M5: both stages' commands. + pub(super) fn m5_storage(&mut self, ordinal: u64) -> Result<&mut [u8]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = self.offset(0, ordinal); + self.descriptors.bytes_mut() + .get_mut(start..start + fw::m5::render_assembly::COMMAND_STORAGE_BYTES) + .ok_or(EINVAL) + } + /// Reset the usage entries owned by this descriptor generation. They stay + /// resident with the graph until both stages release the descriptor slot. + pub(super) fn m5_usage(&mut self, ordinal: u64) -> Result<[u64; 2]> { + if !self.m5 || !self.owns(ordinal) { return Err(EINVAL); } + let start = ordinal as usize % RECORDS * M5_USAGE_PAIR_BYTES; + self.support.bytes_mut().get_mut(start..start + M5_USAGE_PAIR_BYTES) + .ok_or(EINVAL)?.fill(0); + let address = self.support.gpu_va() + start as u64; + Ok([address, address + M5_USAGE_BYTES as u64]) } pub(super) fn publish_recovery_targets(&self) -> Result { let Some(target) = self.pointers.recovery_target else { @@ -247,8 +305,24 @@ struct Compact { address: u64, _backing: KernelObject, } +/// The mesh storage chunk: chunk bases are 64 KiB aligned and the storage +/// register counts 64 KiB chunks. +const MESH_STORAGE_CHUNK: usize = 0x10000; + impl Compact { + /// M5 shared-RW object in the compact window with an explicit VA alignment. + fn new_aligned(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize, align: u64) -> Result { + let size = size.checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ - 1); + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = backing.map_range(vm, 0..size, buffer::BlockGeometry::M5_COMPACT, + align, mmu::PROT_GPU_SHARED_RW, mmu::UAT_PGSZ)?; + if mapping.iova() % align != 0 { return Err(EINVAL); } + Ok(Self { address: mapping.iova(), mapping: Some(mapping), _backing: backing }) + } fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + if alloc.uat.is_m5() { + return Self::new_aligned(alloc, vm, size, cfg::TVB_PAGE); + } let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; let mapping = buffer::compact_alias( &backing, @@ -266,14 +340,63 @@ impl Compact { } } -pub(super) struct Generation { +/// Per-command state of a pair: the G17P one below or the M5 one +/// ([`super::generation::Generation`]). +pub(super) enum Generation { + Neo(KBox), + M5(KBox), +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + if alloc.uat.is_m5() { + super::generation::Generation::allocate_state(alloc, vm) + } else { + NeoGeneration::allocate_state(alloc, vm) + } + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result { + Ok(if alloc.uat.is_m5() { + Self::M5(super::generation::Generation::new(alloc, vm, state)?) + } else { + Self::Neo(NeoGeneration::new(alloc, vm, state)?) + }) + } + pub(super) fn ordinal(&self) -> Option { + match self { + Self::Neo(generation) => generation.ordinal, + Self::M5(generation) => generation.ordinal, + } + } + /// Gives up the claim of an unpublished or retired command. + pub(super) fn clear(&mut self) { + match self { + Self::Neo(generation) => generation.ordinal = None, + Self::M5(generation) => generation.ordinal = None, + } + } + /// `viewport` applies to the G17P deflake buffer ([`abi::Deflake::viewport`]). + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass, viewport: bool) -> Result { + match self { + Self::Neo(generation) => generation.prepare(ordinal, pass, viewport), + Self::M5(generation) => generation.prepare(ordinal, pass), + } + } + pub(super) fn addresses(&self) -> [u64; 5] { + match self { + Self::Neo(generation) => generation.addresses(), + Self::M5(generation) => generation.addresses(), + } + } +} + +pub(super) struct NeoGeneration { deflake: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, pub(super) ordinal: Option, } -impl Generation { +impl NeoGeneration { fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { KernelObject::new( alloc.dev, @@ -368,19 +491,24 @@ impl Generation { } pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, + _fragment_view: Option, pub(super) graph: Graph, scratch: Compact, discard: Compact, + /// M5 mesh rendering buffer of the queue's task-stage kicks. Shaders read + /// its per-workgroup 64 KiB chunk base, so it is mapped in the context VM. + mesh_storage: Option, + mesh_storage_chunks: u32, pub(super) target: KBox, - generations: [Option>; DEPTH], + generations: [Option; DEPTH], generation_count: usize, pub(super) timestamps: KernelObject, pub(super) manager: buffer::Manager, pub(super) lifecycle: [fw::ObjectIds; 2], // The extra client views retain their allocation positions even though // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, + registers: Option, + m5_workspace: Option>, pub(super) context: Arc, pub(super) installed_pool: Option, } @@ -405,13 +533,27 @@ impl Memory { ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); + let m5 = alloc.uat.is_m5(); + if m5 != layout.is_m5() { return Err(EINVAL); } let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; - let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; - let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; - let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let (tvb, mappings, scratch, discard) = if m5 { + let geometry = super::m5_render_submission::parameter_geometry(layout.tvb_blocks)?; + let bytes = fw::m5::pool::parameter_scratch_stride(geometry.capacity, 0) + .map_err(|_| EINVAL)?.checked_mul(fw::m5::pool::SCENE_TABLE_ENTRIES) + .ok_or(EOVERFLOW)?; + let (tvb, mappings) = buffer::Tvb::with_geometry(alloc.dev, vm, layout.tvb_blocks, geometry)?; + let scratch = Compact::new(alloc, vm, bytes)?; + let discard = Compact::new(alloc, vm, 0x8000)?; + (tvb, mappings, scratch, discard) + } else { + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + (tvb, mappings, scratch, discard) + }; let count = DEPTH; let initial = if ordinal == 0 { 0 @@ -447,36 +589,47 @@ impl Memory { let predecessor = crate::g17::ids::OBJECTS.object()?; let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; let support_va = graph.support.gpu_va(); - graph.support.initialize::(0, |view| { - view.init( - support_va, - alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, - low::RENDER_RUN_LIST, - low::RENDER_PAGE_LIST, - word, - ) - })?; - let registers = KernelObject::new( + if !m5 { + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + } + let registers = if m5 { None } else { Some(KernelObject::new( alloc.dev, vm, crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), mmu::UAT_PGSZ, mmu::PROT_GPU_SHARED_RO, CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( + )?) }; + let m5_workspace = if m5 { + let mut workspace = KBox::::new_uninit(GFP_KERNEL)?; + // SAFETY: Workspace is integer-only and has a valid zero representation. + unsafe { core::ptr::write_bytes(workspace.as_mut_ptr(), 0, 1); } + // SAFETY: The entire value was initialized above. + Some(unsafe { workspace.assume_init() }) + } else { None }; + let fragment_view = if m5 { None } else { Some(graph.descriptors.map_range( vm, TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RO, 0, - )?; + )?) }; Ok(( storage.write_init(kernel::try_init!(Self { graph, scratch, discard: discard, + mesh_storage: None, + mesh_storage_chunks: 0, target, generations, generation_count: count, @@ -485,35 +638,82 @@ impl Memory { lifecycle: [tiling, fragment], _fragment_view: fragment_view, registers: registers, + m5_workspace, context, installed_pool: None, }))?, mappings, )) } + /// The mesh storage register value of a task-stage kick. The buffer is + /// allocated on the queue's first such kick and kept for the queue's lifetime, + /// so other contexts pay nothing. A failed allocation leaves no partial state. + /// A zero chunk parameter disables the binding. + pub(super) fn mesh_storage_register(&mut self, alloc: &Allocator<'_>) -> Result { + if self.mesh_storage.is_none() { + let chunks = crate::module_parameters::m5_mesh_storage_chunks.value(); + if chunks == 0 { return Ok(0); } + if chunks > 0xffff { return Err(EINVAL); } + let bytes = (chunks as usize).checked_mul(MESH_STORAGE_CHUNK).ok_or(EOVERFLOW)?; + let storage = Compact::new_aligned(alloc, self.context.vm(), bytes, + MESH_STORAGE_CHUNK as u64)?; + self.mesh_storage = Some(storage); + self.mesh_storage_chunks = chunks; + } + let storage = self.mesh_storage.as_ref().ok_or(ENOMEM)?; + Ok((storage.address & 0xffff_ffff_ffff) | (u64::from(self.mesh_storage_chunks) << 48)) + } pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { self.generations[ordinal as usize % self.generation_count] - .as_deref_mut() + .as_mut() .ok_or(EIO) } pub(super) fn owns(&self, ordinal: u64) -> bool { self.graph.owns(ordinal) && self.generations[ordinal as usize % self.generation_count] .as_ref() - .is_some_and(|generation| generation.ordinal == Some(ordinal)) + .is_some_and(|generation| generation.ordinal() == Some(ordinal)) + } + /// M5 commands record four stage timestamps, G17P ones two. + pub(super) fn timestamp_offset(&self, ordinal: u64) -> usize { + ordinal as usize % RECORDS * if self.graph.m5 { 32 } else { 16 } + } + pub(super) fn clear_timestamps(&mut self, ordinal: u64) -> Result { + let at = self.timestamp_offset(ordinal); + if self.graph.m5 { self.timestamps.write(at, [0u64; 4]) } + else { self.timestamps.write(at, [0u64; 2]) } } pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { if !self.owns(ordinal) { return Err(EIO); } - let offset = ordinal as usize % RECORDS * 16; + let offset = self.timestamp_offset(ordinal); + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + if self.graph.m5 { 24 } else { 8 })? + .load(Ordering::Relaxed), + ]) + } + /// Export the four M5 stage words only after the observer has acquired + /// both completion stamps. Keep the two-word G17P representation intact. + pub(super) fn stage_timestamps(&self, ordinal: u64) -> Result<[u64; 4]> { + if !self.graph.m5 { + let [start, end] = self.pass(ordinal)?; + return Ok([start, start, start, end]); + } + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = self.timestamp_offset(ordinal); Ok([ self.timestamps.dword(offset)?.load(Ordering::Relaxed), self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 16)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 24)?.load(Ordering::Relaxed), ]) } pub(super) fn owners(&self) -> [Option; DEPTH] { - core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal())) } pub(super) fn retire(&mut self, ordinal: u64) -> Result { if !self.owns(ordinal) { @@ -522,7 +722,7 @@ impl Memory { self.target.retire(ordinal)?; self.manager.retire_scene(ordinal)?; self.graph.release(ordinal)?; - self.generation(ordinal)?.ordinal = None; + self.generation(ordinal)?.clear(); Ok(()) } } @@ -547,7 +747,7 @@ impl Memory { let record = ordinal as usize % RECORDS; let config = size_of::(); let ranges = [ - (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.descriptors, self.graph.offset(0, ordinal), size_of::()), (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), ]; @@ -566,6 +766,42 @@ impl Memory { } } + /// Encodes both M5 commands of `args.ordinal` into its command storage and + /// returns the kicks' register arrays and header flags. + pub(super) fn write_m5_commands( + &mut self, + args: &abi::RenderArgs<'_>, + layout: &target::Layout, + plan: Option, + mesh_class: u8, + ) -> Result<([[Option; 4]; 2], fw::kick::StageKickFlags)> { + use super::m5_render_submission as render_submission; + use fw::m5::{render_assembly, render_registers}; + if !self.graph.m5 { + return Err(EINVAL); + } + let mut controls = render_submission::controls(args.pass)?; + if let Some(stages) = plan.and_then(|p| p.stages) { + controls.ta.scs = stages[0].scs; + controls.fragment.scs = stages[1].scs; + controls.ta.resource_flags = stages[0].resource_flags; + controls.ta.ptc_max = stages[0].ptc_max; + controls.fragment.resource_flags = stages[1].resource_flags; + controls.fragment.ptc_max = stages[1].ptc_max; + } + let usage = self.graph.m5_usage(args.ordinal)?; + let mut inputs = render_submission::inputs(args, usage)?; + inputs.ta.shifted_usage_hint = fw::m5::render::mesh_class_usage_hint(mesh_class); + render_registers::apply_pass(args, &controls, &mut inputs)?; + let extra = self.target.m5_extra(layout)?; + let low = self.graph.descriptor_client(0, args.ordinal)?; + let workspace = self.m5_workspace.as_deref_mut().ok_or(EIO)?; + let encoded = render_assembly::encode_into(self.graph.m5_storage(args.ordinal)?, low, + args, &controls, extra, render_registers::FragmentDispatch { words: [0; 2] }, + &inputs, workspace)?; + Ok((encoded.rce, encoded.kick_flags)) + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -576,7 +812,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(0, args.ordinal), + self.graph.offset(0, args.ordinal), size_of::(), )? .cast::(); @@ -584,7 +820,7 @@ impl Memory { .graph .descriptors .pointer( - Graph::offset(1, args.ordinal), + self.graph.offset(1, args.ordinal), size_of::(), )? .cast::(); @@ -626,15 +862,19 @@ impl Memory { if !self.owns(ordinal) { return Err(EIO); } + if self.graph.m5 { + return fw::m5::render_assembly::link_completions( + self.graph.m5_storage(ordinal)?, kicks, payload, fragment_busy); + } let tiling = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); // SAFETY: The complete views were bounds/alignment checked during // preparation. The unchanged full ordinal still exclusively owns both @@ -655,7 +895,7 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); // SAFETY: The view was checked during preparation and the ordinal still owns it; only // integer fields are read. @@ -677,21 +917,49 @@ impl Memory { impl Memory { /// Preserve the initial mapping order while moving VM-lifetime mappings /// out of the graph. Reserve all host storage before moving any mapping. + /// Moves this drained graph's pool mappings out of its VM slot, so a new + /// M5 graph of the same VM can install on the pair (see bind_owner). + pub(super) fn take_pool_mappings(&mut self) -> Option<(u8, mmu::RenderPoolMappings)> { + let slot = self.installed_pool?; + let id = u64::from(self.lifecycle[0].predecessor); + let mappings = self.context.vm().take_render_pool_mappings(slot, id)?; + self.installed_pool = None; + Some((slot, mappings)) + } + /// Returns mappings moved out by `take_pool_mappings` after a failed install. + pub(super) fn put_pool_mappings( + &mut self, + slot: u8, + mappings: mmu::RenderPoolMappings, + ) -> Result { + self.context.vm().put_render_pool_mappings(slot, mappings)?; + self.installed_pool = Some(slot); + Ok(()) + } + pub(super) fn take_persistent( &mut self, tvb: &mut KVec, ) -> Result> { if self.scratch.mapping.is_none() || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 + || (!self.graph.m5 && self.registers.as_ref().is_none_or(|r| r.gpu_va() == 0)) { return Err(EIO); } + if self.mesh_storage.as_ref().is_some_and(|m| m.mapping.is_none()) { + return Err(EIO); + } let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.reserve(tvb.len().checked_add(4).ok_or(EOVERFLOW)?, GFP_KERNEL)?; aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + if let Some(storage) = &mut self.mesh_storage { + aliases.push(storage.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + } + if let Some(registers) = &mut self.registers { + aliases.push(registers.take_mapping()?, GFP_KERNEL)?; + } for mapping in core::mem::take(tvb) { aliases.push(mapping, GFP_KERNEL)?; } @@ -704,6 +972,9 @@ impl Memory { if slot >= 4 || !self.owns(ordinal) { return Err(EINVAL); } + if self.graph.m5 { + return fw::m5::render_assembly::bind_work_slot(self.graph.m5_storage(ordinal)?, slot); + } let queues = [ self.graph .queues @@ -715,12 +986,12 @@ impl Memory { let ta = self .graph .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? + .pointer(self.graph.offset(0, ordinal), size_of::())? .cast::(); let fragment = self .graph .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? + .pointer(self.graph.offset(1, ordinal), size_of::())? .cast::(); for word in queues { word.store(slot, Ordering::Relaxed); @@ -738,7 +1009,40 @@ impl Memory { /// Retargets both retired queue records to `context`. A context of another /// scheduling profile also replaces the priority and policy words, after /// the owner fields; the same profile leaves them untouched. - pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + /// `rewrite`: store the M5 records even when their host-owned words + /// already name `context`; without it that no-op read-modify-write, which + /// would also store back firmware-owned words of a possibly running queue, + /// is skipped. G17P always retargets. + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context, rewrite: bool) -> Result { + if self.graph.m5 { + let records = qids.map(|qid| self.graph.queues.pointer(self.graph.queue_offset(qid), + size_of::())); + let [first, second] = records; + let pointers = [first?, second?].map(|p| p.cast::()); + // SAFETY: Both complete records are in bounds. The caller owns + // retired queues exclusively; build both updates before storing. + unsafe { + let mut values = pointers.map(|p| p.read_unaligned()); + if !rewrite { + let mut named = true; + for value in &values { + named &= value.names_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + if named { + return Ok(()); + } + } + for value in &mut values { + value.set_compute_owner(context.work_head_va(), context.scheduler_va(), + context.owner_pid(), context.policy())?; + } + for (pointer, value) in pointers.into_iter().zip(values) { + pointer.write_unaligned(value); + } + } + return Ok(()); + } let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; // Check every field before the first update, retaining the required diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index ca74d9524c698d..3c18c58f646cb0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -2,10 +2,9 @@ //! Installed kick storage retained after a logical scheduler release is acknowledged. -use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use super::{install_memory, memory, Pair, RetiredBinding, Tracker}; use crate::g17::{ buffer, - command::Validated, context::Context, freelist::RenderPool, job::Packet, @@ -42,6 +41,7 @@ pub(crate) struct ParkedPair { /// All new graph ownership stays separate from the installed kick storage until commit. pub(crate) struct Prepared { + pool_pin: Option, memory: Option>, pool: Arc, aliases: Option>, @@ -109,6 +109,7 @@ impl Pair { buffer_id, publication: None, ready: true, + wait_for_pool_release: false, }; (parked, retired) } @@ -143,19 +144,24 @@ impl ParkedPair { pm_generation: u64, first: &Arc, ) -> Result { + let pool_pin = free_list.pin_id()?; + if alloc.uat.is_m5() { + free_list.ensure_backing(alloc, context.vm(), context.vm())?; + } if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { return Err(EINVAL); } let kicks = self.kicks.as_ref().ok_or(EBUSY)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = super::m5_render_submission::layout( + &first.command, + self.clusters, + self.variant.programs, + )?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -174,6 +180,7 @@ impl ParkedPair { self.variant, )?; Ok(Prepared { + pool_pin, memory: Some(memory), pool: free_list, aliases: Some(persistent), @@ -221,6 +228,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + pool_pin: next.pool_pin.take(), graph_started: false, kick_aliases: None, memory, @@ -248,11 +256,16 @@ impl ParkedPair { previous: self.previous.take(), displaced: None, replay: Tracker::new(), + executed_generation: None, replay_since: None, context_update: true, executed: self.executed, switch: None, published_completions: None, + identity_update: true, + configuration: None, + configured_context: None, + configured_qos: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index d65c40c7cfadee..838d3b17bab862 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -5,6 +5,7 @@ use super::{memory, retirement::Window}; use crate::g17::fw::{ + m5, kick::KickTimestamp, queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, }; @@ -27,6 +28,12 @@ pub(in crate::g17::queue) fn required(previous: Option, config: Que || previous.map(identity) != Some(identity(config)) } +/// The kick announcement record: the G17P one, or the M5 notification. +enum Announcement { + Neo(KickAnnounce), + M5(m5::queue::Notification), +} + pub(super) struct Lane<'a> { items: [&'a AtomicU64; 4], producer: &'a AtomicU32, @@ -34,8 +41,8 @@ pub(super) struct Lane<'a> { config: QueueConfig, config_destination: *mut u64, config_present: bool, - announce: KickAnnounce, - announce_destination: *mut KickAnnounce, + announce: Announcement, + announce_destination: *mut u8, prefix: usize, prefix_end: u32, pub(super) window: Window, @@ -55,7 +62,9 @@ impl<'a> Lane<'a> { timestamp: KickTimestamp, lead: bool, ) -> Result { - let count = 2 + u32::from(present) + u32::from(lead); + let m5 = graph.is_m5(); + // M5 items carry no descriptor: [configuration], notification. + let count = u32::from(present) + u32::from(lead) + if m5 { 1 } else { 2 }; let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -69,8 +78,8 @@ impl<'a> Lane<'a> { } let announce_destination = graph .queues - .pointer(announce_offset, size_of::())? - .cast::(); + .pointer(announce_offset, if m5 { size_of::() } + else { size_of::() })?; let index = |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; let items = [ @@ -91,7 +100,9 @@ impl<'a> Lane<'a> { if lead { push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); } - push(descriptor); + if !m5 { + push(descriptor); + } if present { push(graph.config.gpu_va() + config_offset as u64); } @@ -105,16 +116,20 @@ impl<'a> Lane<'a> { config, config_destination, config_present: present, - announce: KickAnnounce::new( - config.qid.try_into()?, - if stage == 0 { - DataMaster::Tiling - } else { - DataMaster::Fragment - }, - timestamp, - config.priority as u8, - )?, + announce: if m5 { + Announcement::M5(m5::queue::notification(config.qid.try_into()?, timestamp)?) + } else { + Announcement::Neo(KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + config.priority as u8, + )?) + }, announce_destination, prefix: count as usize - 1, prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, @@ -146,9 +161,21 @@ impl<'a> Lane<'a> { // SAFETY: The full record is inside the retained queue graph and is // exclusively owned by this ordinal. Publication follows its copy. unsafe { - self.announce_destination.write_unaligned(self.announce); + match &self.announce { + Announcement::Neo(value) => self.announce_destination + .cast::().write_unaligned(*value), + Announcement::M5(value) => self.announce_destination + .cast::().write_unaligned(*value), + } } } + /// Fill the retained records without advancing the firmware-visible cursor. + /// M5's host-port transaction must occur between this and publish_kick. + pub(super) fn prepare(&self) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + } pub(super) fn publish_prefix(&self) { self.items(0..self.prefix); self.config(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 4377bbfe3e311d..19a2caabe68e10 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -39,7 +39,7 @@ pub(crate) struct Window { impl Window { pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { - if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + if base >= capacity || !(1..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { return Err(EINVAL); } Ok(Self { @@ -230,8 +230,7 @@ pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> R if !observation.complete { return Ok(Poll::Pending); } - let [start, end] = memory.pass(ticket.ordinal)?; - Ok(Poll::Complete([start, start, start, end])) + Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)) } /// Outstanding publications span less than half the 24-bit stamp sequence. @@ -255,6 +254,11 @@ pub(crate) trait Memory { fn snapshot(&mut self, ordinal: u64) -> Result; fn stamps(&mut self) -> Result<[u32; 2]>; fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; + /// Four stage timestamps: M5 records them, G17P derives them from its pass. + fn stage_timestamps(&mut self, ordinal: u64) -> Result<[u64; 4]> { + let [start, end] = self.pass(ordinal)?; + Ok([start, start, start, end]) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -314,8 +318,7 @@ fn complete_poll( // Refresh this generation's pass after observing the post-maintenance // stamps. The pre-stamp snapshot alone cannot authorize export. fence(Ordering::Acquire); - let [start, end] = memory.pass(ticket.ordinal)?; - return Ok(Poll::Complete([start, start, start, end])); + return Ok(Poll::Complete(memory.stage_timestamps(ticket.ordinal)?)); } ticket.previous = snapshot.cursors; Ok(Poll::Pending) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 15ec8897499d19..4a2a116f83bc80 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -174,6 +174,8 @@ pub(super) struct Registry { pub(super) compute_building: bool, compute_scheduler_building: bool, replay_outer_wait: Option, + /// A compute replay waits for its owner's renders to park or idle (compute_replay). + replay_consumer_wait: Option, pub(super) observations: compute::retirement::Batch, /// Completion-ring slot where the next compute scan starts. completion_cursor: usize, @@ -211,7 +213,7 @@ impl Registry { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + metrics: Arc::new(MetricsIds::for_firmware(variant.m5_commands)?, GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, @@ -226,6 +228,7 @@ impl Registry { compute_building: false, compute_scheduler_building: false, replay_outer_wait: None, + replay_consumer_wait: None, observations: compute::retirement::Batch::new()?, completion_cursor: 0, completion_seen: { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 32b9acef94c0fd..d16b22e35b896f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -6,6 +6,9 @@ use super::DeferredBatch; use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; +/// Bound for a compute replay waiting on its owner's render abandonment proof. +const CONSUMER_WAIT_NS: u64 = 300_000_000; + impl Firmware { /// Called only from an outer worker or client submission, never from event /// dispatch or a recovery callback nested within publication. All candidate @@ -18,6 +21,7 @@ impl Firmware { if state != 0 { return Ok(()); } + self.queues.replay_consumer_wait = None; let commands = self .queues .compute @@ -53,6 +57,43 @@ impl Firmware { else { break; }; + // A render consumer named this command through a firmware dependency. + // Republish only while no render of the owner has executed since this recovery + // (none consumed the dead token), after every live render of the owner is + // parked for replay; the render replay then depends on the new command. A + // parked render this command waits through is republished first, so the new + // command depends on it again. + let generation = self.recovery.generation(); + let parked_prefix = owner.is_some_and(|owner| { + self.queues.render.owner_parked_through(owner, packet.order.wait_through[0]) + }); + let (executed, live) = if packet.completion.has_firmware_consumer() { + owner.map_or((true, false), |owner| self.queues.render.owner_render_state(owner, generation)) + } else { + (false, false) + }; + let must_wait = parked_prefix || (!executed && live); + let now = super::now_ns(); + let waited = if must_wait { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .consumer_wait(now) + } else { + 0 + }; + if must_wait && waited < CONSUMER_WAIT_NS { + // The event worker polls while this waits. + self.queues.replay_consumer_wait = Some(now); + return Ok(()); + } + let consumer = (executed || must_wait).then_some(ENODATA); + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .clear_consumer_wait(); let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); @@ -66,7 +107,9 @@ impl Firmware { self.queues.replay_outer_wait = None; } - let result = if !outer_ready { + let result = if let Some(error) = consumer { + Err(error) + } else if !outer_ready { // This replay has no live firmware ownership. Persistent // backpressure uses the existing failed-replay retirement. Err(EBUSY) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index d6d811b410e89f..96a3de5776013c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -499,6 +499,7 @@ impl crate::g17::Shared { let backpressured = firmware.render_control_backpressured(); let polling = backpressured || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.replay_consumer_wait.is_some() || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index fc2e01be761aeb..4b62d96d7cd3cf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -524,6 +524,8 @@ impl Registry { struct Host<'a> { firmware: &'a mut crate::g17::Firmware, registration_powered: bool, + /// An M5 pair's installation, submitted with its first outer publication. + m5_registration: Option, } impl Host<'_> { fn release_registration(&mut self) { @@ -542,6 +544,9 @@ impl Host<'_> { } } impl crate::g17::queue::render::Host for Host<'_> { + fn generation(&self) -> u64 { + self.firmware.recovery.generation() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.firmware.recovery.generation(), @@ -571,8 +576,18 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. - fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + /// M5's host command port installs and kicks the pair in one transaction: + /// its registration is retained until the outer publication, without the + /// core-power hold. + fn install_pair(&mut self, registration: &crate::g17::queue::render::Registration) -> Result { use crate::g17::fw::variant::QueueRegistration; + let registration = match registration { + crate::g17::queue::render::Registration::M5(submission) => { + self.m5_registration = Some(*submission); + return Ok(()); + } + crate::g17::queue::render::Registration::Neo(registration) => registration, + }; match self.firmware.variant.registration { QueueRegistration::Firmware => return Ok(()), QueueRegistration::Mailbox => { @@ -611,12 +626,44 @@ impl crate::g17::queue::render::Host for Host<'_> { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]> { + if self.firmware.init.is_m5() { + let submission = match self.m5_registration.take() { + Some(submission) => submission, + None => crate::g17::fw::m5::queue::HardwareSubmission::advance_pair( + fragment.qid, + tiling.qid, + )?, + }; + return self.firmware.submit_m5_render_pair( + priority, + &[copy_slot(fragment), copy_slot(tiling)], + &submission, + commit, + late_tiling, + true, + ); + } self.firmware .init .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + prepare: impl FnOnce(), + ) -> Result { + let pair = self.firmware.init.m5_work_pair(priority, fragment, tiling)?; + pair.prepare(); + prepare(); + pair.publish(); + Ok(()) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { - self.release_registration(); + if !self.firmware.init.is_m5() { + self.release_registration(); + } let result = self.firmware.queues.render.pools.activate(id, generation); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); @@ -639,16 +686,28 @@ impl crate::g17::queue::render::Host for Host<'_> { } result } - /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. M5 renders + /// use the common compute work ring, pipe 2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); - crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { - self.firmware.bridge.regs.g17_core_power() - }); - let result = self - .firmware - .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); + let m5 = self.firmware.init.is_m5(); + if m5 { + // A paired publication is one tiling and one fragment submission. + for _ in 0..2 { + if let Err(error) = self.firmware.init.note_m5_submission_history() { + pr_warn!("asahi: M5 submission history skipped: {:?}\n", error); + } + } + } else { + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); + } + let result = self.firmware.primary.notify( + (0x83 << crate::g17::MSG_TYPE_SHIFT) + | u64::from(priority) << 2 + | if m5 { 2 } else { 0 }, + ); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } @@ -684,6 +743,18 @@ impl crate::g17::queue::render::Host for Host<'_> { } } +/// The M5 pair submission takes both work slots as one array. +fn copy_slot(slot: &crate::g17::fw::channels::WorkSlot) -> crate::g17::fw::channels::WorkSlot { + crate::g17::fw::channels::WorkSlot { + unk_00: slot.unk_00, + queue_va: slot.queue_va, + data_master: slot.data_master, + items_end: slot.items_end, + qid: slot.qid, + first: slot.first, + } +} + impl crate::g17::Firmware { /// The pair is returned even when transport, accounting or collection fails. /// No off-lock operation or callback which can re-enter the device runs here. @@ -703,6 +774,7 @@ impl crate::g17::Firmware { let mut host = Host { firmware: self, registration_powered: false, + m5_registration: None, }; let result = operation(&mut pair, &mut host); host.release_registration(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index e150a10e2c9bc7..b31a37dae1fd19 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -10,6 +10,14 @@ impl Registry { &self, owner: u64, packet: &Arc, + ) -> Result { + self.render_prefix_at(owner, packet, packet.order.wait_through[0]) + } + pub(in crate::g17::runtime) fn render_prefix_at( + &self, + owner: u64, + packet: &Arc, + prefix: Option, ) -> Result { let mut frontier = Frontier::new(); let compute = packet.engine() == crate::g17::job::Engine::Compute; @@ -18,7 +26,7 @@ impl Registry { pair.collect_dependencies( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; @@ -26,7 +34,7 @@ impl Registry { dependencies.collect( owner, packet.completion.status(), - packet.order.wait_through[0], + prefix, &mut frontier, compute, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index 409b948ee6aef8..a0626544002178 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -45,6 +45,9 @@ struct DirectHost<'a> { } impl crate::g17::queue::render::Host for DirectHost<'_> { + fn generation(&self) -> u64 { + self.generation + } fn next_publication(&mut self) -> Result { Ok(self.port.next_publication()) } @@ -65,7 +68,7 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.port.qos_cancel(publication) } - fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + fn install_pair(&mut self, _registration: &crate::g17::queue::render::Registration) -> Result { Err(EAGAIN) } fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { @@ -86,6 +89,16 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { self.port .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) } + fn publish_configuration_pair( + &mut self, + _priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + _prepare: impl FnOnce(), + ) -> Result { + // Only M5 publishes configuration pairs, never on this path. + Err(EINVAL) + } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { // The lease leaves preparation when the pair is returned; nothing can // observe it in between because the pair is lent. @@ -186,6 +199,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: None, }; let pair = match self.queues.render.take_pair(slot, owner, packet) { Ok(pair) => pair, @@ -263,8 +277,10 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Option { // A compute prefix needs the compute queue's state; leave it to the - // device-mutex path. - if packet.is_published() + // device-mutex path. M5 renders publish through the device mutex + // only: their host-port transaction is part of the publication. + if packet.context.is_m5() + || packet.is_published() || packet.cancel_requested() || packet.completion.status().get() != 0 || packet.order.wait_through[1].unwrap_or(0) != 0 diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 277534328406ee..032a73759733b9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -251,7 +251,7 @@ impl crate::g17::Shared { manager.prepare_growth( &dev, &vm, - crate::g17::buffer::firmware_growth_target(before), + manager.firmware_target(), None, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 1e3354d90dbfb9..ab94764580521a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -122,7 +122,17 @@ impl Pools { } fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool.m5_descriptor()?; + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + return Ok(()); + } + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; pool.repopulate()?; let descriptor = pool .page_pool_state()? @@ -240,8 +250,13 @@ impl Pools { /// A detached pool may capture firmware counters only after its last pair /// stopped using this registration. Callers check all retained pair owners. pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.is_m5() { + let words = descriptor_words::<8>(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + return pool.store_m5_descriptor(words.map(|word| word.load(Ordering::Relaxed))); + } let state = pool.page_pool_state()?; - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + let words = descriptor_words::<4>(init.page_pool_descriptor_table()?, pool.id())?; fence(Ordering::Acquire); let descriptor = PagePoolDescriptor { page_list: words[0].load(Ordering::Relaxed), @@ -256,14 +271,12 @@ impl Pools { } } -fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { - let offset = Pools::slot(id)? * size_of::(); - Ok([ - table.dword(offset)?, - table.dword(offset + 8)?, - table.dword(offset + 16)?, - table.dword(offset + 24)?, - ]) +/// The `N` descriptor words of pool `id`: four on G17P, eight on M5. +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; N]> { + let offset = Pools::slot(id)? * N * size_of::(); + // Check the full row before constructing the infallible array of views. + table.pointer(offset, N * size_of::())?; + Ok(core::array::from_fn(|i| table.dword(offset + i * 8).unwrap())) } impl Pools { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index cef991aff64a07..863e7a59e118e3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + freelist::{PoolPin, RenderPool}, preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, @@ -44,6 +45,28 @@ impl BuildResources { fn metrics(&self) -> Result { MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) } + /// The render pool geometry the command needs: M5 commands carry a plan. + fn pool_plan(&self, packet: &Packet) -> Result> { + let crate::g17::command::Validated::Render { plan, .. } = &packet.command else { + return Err(EINVAL); + }; + if self.uat.is_m5() != plan.is_some() { + return Err(EINVAL); + } + Ok(*plan) + } + /// M5 places the client view of the metrics backing per VM. Called during + /// off-lock preparation, after the context's bindings exist. + fn bind_metrics( + &self, + context: &crate::g17::context::Context, + metrics: &mut MetricsLease, + ) -> Result { + if self.uat.is_m5() { + metrics.set_client_base(context.metrics_client(&self.metrics)?)?; + } + Ok(()) + } } #[expect( @@ -78,6 +101,148 @@ struct BuildContext<'a> { } impl super::super::Backend { + /// Reserves scarce M5 pool identities before the pair build transfers a + /// physical render pair: the owner's fitting pool, or the context's pool + /// for the command's plan, pinned. The pin is held across the build; the + /// pair takes its own. + fn claim_render_pool( + &self, + first: &Arc, + lease: &Lease, + ) -> Result<(Arc, Option)> { + let dev = self.shared.drm()?; + let (resources, existing) = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + let resources = BuildResources::new(firmware)?; + let geometry = resources.pool_plan(first)?.map(|plan| plan.geometry); + let mut existing = None; + for entry in firmware.queues.render.entries.iter().flatten() { + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(geometry) + && !pair.pool().vacant() + { + existing = Some(pair.pool().clone()); + break; + } + } + (resources, existing) + }; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + let pool = match existing { + Some(pool) => pool, + None => self.context.render_pool( + &alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?, + }; + let pin = self.pin_render_pool(&pool, &resources, lease)?; + Ok((pool, pin)) + } + + /// Pins `pool`'s identity. When every M5 identity is taken, an identity + /// whose final free-list release firmware consumed is reclaimed and its + /// physical storage kept for `pool`. ENOSPC when none is reclaimable: the + /// caller waits for render capacity. + fn pin_render_pool( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + ) -> Result> { + let mut reclaimed_pages = None; + loop { + if !lease.is_current() { + return Err(EAGAIN); + } + match pool.pin_id() { + Ok(pin) => { + if let Some(reclaimed) = reclaimed_pages.take() { + pool.cache_reclaimed(reclaimed); + } + return Ok(pin); + } + Err(error) if error == ENOSPC && resources.uat.is_m5() => { + // Repeat the release observation after a failed claim: a + // concurrent pin may have taken the reclaimed identity. + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_one(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + // Dropped outside the device lock: its mappings may take + // memory-management locks. + drop(reclaimed_pages.take()); + if reclaimed.is_some() { + reclaimed_pages = reclaimed; + continue; + } + self.shared.queue_events(); + return Err(error); + } + Err(error) => return Err(error), + } + } + } + + /// Arms an M5 pool's backing for the pair build. A pool without storage + /// takes the physical storage of a released identity first, without + /// evicting a metadata-only owner. The caller holds the pool's pin. + fn prepare_pool_backing( + &self, + pool: &Arc, + resources: &BuildResources, + lease: &Lease, + client: &crate::mmu::Vm, + ) -> Result { + if resources.uat.is_m5() && pool.needs_storage() { + let reclaimed = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + firmware.queues.render.pools.observe(&firmware.init)?; + resources + .ids + .reclaim_backing(|id| firmware.queues.render.pools.released(u32::from(id), 0)) + }; + if let Some(reclaimed) = reclaimed { + pool.cache_reclaimed(reclaimed); + } + } + let dev = self.shared.drm()?; + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + pool.ensure_backing(&alloc, &resources.global, client) + } + fn build_render_pair( &self, slot: u8, @@ -87,6 +252,12 @@ impl super::super::Backend { ) -> Result { let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; + // M5 pool identities are scarce: reserve one before any pair moves. + let _pool_claim = if self.context.is_m5() { + Some(self.claim_render_pool(first, &lease)?) + } else { + None + }; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -130,7 +301,12 @@ impl super::super::Backend { ) } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; - if pair.matches(self.owner, &self.context)? { + // An M5 pool that no longer fits the command's geometry + // is replaced by a same-owner rebind below. + let plan = resources.pool_plan(first)?; + if pair.matches(self.owner, &self.context)? + && pair.pool().fits(plan.map(|p| p.geometry)) + { if !pair.pool().vacant() { return Ok(()); } @@ -140,9 +316,17 @@ impl super::super::Backend { dev: &dev, uat: &resources.uat, }; + if resources.uat.is_m5() { + return self.prepare_pool_backing( + &pool, + &resources, + &lease, + self.context.vm(), + ); + } return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } - if !pair.reusable()? { + if !pair.bindable(self.owner, &self.context)? { return Err(EBUSY); } // The release of the pair's scheduler is armed or still being @@ -152,7 +336,14 @@ impl super::super::Backend { if firmware.queues.teardown.releasing(pair.context()) { return Err(EBUSY); } - let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + // On M5 a different logical context, even of the same VM, + // gets its own graph; only a pool rebind of the same + // context keeps the graph and its queue records. + let foreign = if resources.uat.is_m5() { + !Arc::ptr_eq(pair.context(), &self.context) + } else { + !Arc::ptr_eq(pair.context().status(), self.context.status()) + }; if foreign { firmware .queues @@ -247,7 +438,7 @@ impl super::super::Backend { ctx: &BuildContext<'_>, reservation: Reservation, pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -257,9 +448,16 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global, None)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } Pair::new( alloc, &resources.objects, @@ -328,7 +526,7 @@ impl super::super::Backend { mut parked: crate::g17::queue::render::parked::ParkedPair, buffer: (u8, u64), pm_generation: u64, - metrics: MetricsLease, + mut metrics: MetricsLease, ) -> Result { let BuildContext { lease, @@ -338,9 +536,16 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global, None)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + resources.bind_metrics(&self.context, &mut metrics)?; + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } let prepared = parked.prepare( alloc, &resources.objects, @@ -444,7 +649,7 @@ impl super::super::Backend { mut pair: KBox, buffer: Option<(u8, u64)>, pm_generation: Option, - metrics: Option, + mut metrics: Option, ) -> Result { let BuildContext { lease, @@ -454,9 +659,18 @@ impl super::super::Backend { first, } = *ctx; let prepared = (|| { - let free_list = - self.context - .render_pool(alloc, &resources.ids, &resources.global, None)?; + let free_list = self.context.render_pool( + alloc, + &resources.ids, + &resources.global, + resources.pool_plan(first)?, + )?; + if let Some(metrics) = metrics.as_mut() { + resources.bind_metrics(&self.context, metrics)?; + } + if resources.uat.is_m5() { + self.prepare_pool_backing(&free_list, resources, lease, self.context.vm())?; + } pair.prepare_bind( alloc, &resources.objects, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 499e3858fcca18..aa78a7ae63c66d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -10,6 +10,29 @@ use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; use kernel::prelude::*; impl super::RenderState { + /// (an executed group of the owner retired in `generation` or later, + /// a published group of the owner is still live and not parked). + pub(in crate::g17) fn owner_render_state(&self, owner: u64, generation: u64) -> (bool, bool) { + let mut executed = false; + let mut live = false; + for pair in self.entries.iter().flatten().filter_map(|entry| entry.pair.as_ref()) { + if pair.owner() != Some(owner) { + continue; + } + executed |= pair.executed_since(generation); + live |= pair.live_unparked(); + } + (executed, live) + } + /// A parked render of the owner precedes a command waiting through `through`. + pub(in crate::g17) fn owner_parked_through(&self, owner: u64, through: Option) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.owner() == Some(owner) && pair.replay_through(through)) + } + pub(in crate::g17) fn replay_polling(&self) -> bool { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 970035547dfee2..24fea49f807d13 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -568,9 +568,22 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } - if dependencies.render_count != 0 - && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } - if pass.fragment_barriers) + let staged = matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.stage_prefixes); + if staged { + let tiling = self.render_prefix_at(owner, packet, packet.order.tiling_wait_through[0])?; + for dependency in tiling.as_slice() { + *dependencies + .render_tiling + .get_mut(dependencies.render_tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_tiling_count += 1; + } + } + if staged + || (dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers)) { let tiling = self.tiling_prefix(owner, packet)?; for dependency in tiling.as_slice() { @@ -585,9 +598,20 @@ impl Registry { dependencies.compute = queue.dependency( owner, packet.completion.status(), - packet.order.wait_through[1], + packet.order.tiling_wait_through[1], state, )?; + if packet.order.tiling_wait_through[1] != packet.order.wait_through[1] { + let whole = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )?; + if whole != dependencies.compute { + dependencies.fragment_compute = whole; + } + } } Ok(dependencies) } @@ -700,7 +724,14 @@ impl crate::g17::Firmware { } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); - let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + // An M5 PB token is bound to the hardware context and PB state it names. + let buffer_context = pair.m5_pb_state_va().map(|pb_va| (packet.context.id(), pb_va)); + let (buffer_id, buffer_token) = match buffer_context { + Some((context, pb_va)) => { + self.queues.buffers.acquire_owner_bound(buffer_id, context, pb_va)? + } + None => self.queues.buffers.acquire_owner(buffer_id)?, + }; let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { Ok(preparation) => preparation, Err(error) => { @@ -711,6 +742,7 @@ impl crate::g17::Firmware { let lease = PublicationLease { free_list_generation: preparation.generation, buffer_token, + buffer_context: buffer_context.map(|(context, _)| context), }; let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index fffa6d28dd289c..bbb2f9f1d08139 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -77,7 +77,13 @@ impl RenderState { } let mut index = 0; while index < self.retired.len() { - if self.retired[index].buffer_id().is_some() { + // An M5 pool replaced by a same-context rebind stays until + // firmware acknowledged its release. + let pool = self.retired[index].pool(); + if self.retired[index].buffer_id().is_some() + || (self.retired[index].wait_for_pool_release() + && !self.pools.released(u32::from(pool.id()), pool.control_va())) + { index += 1; continue; } diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index fec7a6b61ad8fb..a5ea381434d443 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -108,11 +108,12 @@ impl Vm { pub(crate) fn render_footprint( &self, attachments: &Attachments, + whole: bool, ) -> Result>> { if attachments.as_slice().is_empty() { return Ok(None); } - let mut footprint = Footprint::new(self.id); + let mut footprint = Footprint::new(self.id, whole); let bindings = self.shared_bindings.as_ref().and_then(|shared| { (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) }); From 4d1cd52bfe3e84dc3f418b59aa42d4e657819e1e Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 21:47:48 -0800 Subject: [PATCH 070/171] drm/asahi: g17: M5: Fix existing M5 Mesa losing the GPU device on staged renders On M5, Mesa that already sends staged renders under DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES sends them in a 368-byte form. This kernel accepts a staged render only as the full 480-byte struct drm_asahi_cmd_render, so the first such render failed with EINVAL and the application lost its device. That Mesa now leaves staged renders off and keeps working; Mesa that sends the full command finds them under a new bit. The kernel reports the full-command form as DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 (bit 31) and no longer reports bit 25, which stays defined for the shorter form. What the kernel accepts under DRM_ASAHI_RENDER_STAGE_PREFIXES is unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 2c22c8e7be7e4fc9242d8712c30ff9be37103b75 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/file.rs | 6 +++--- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- include/uapi/drm/asahi_drm.h | 13 +++++++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index d0377ec32a6c28..ccae606a84a075 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -89,9 +89,9 @@ pub(crate) const DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: u64 = pub(crate) const DRM_ASAHI_RENDER_SHADER_WRITES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_SHADER_WRITES; /// Renders may carry separate tiling prefixes of both subqueues -/// (DRM_ASAHI_RENDER_STAGE_PREFIXES). -pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: u64 = - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES as u64; +/// (DRM_ASAHI_RENDER_STAGE_PREFIXES), as the full render command. +pub(crate) const DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: u64 = + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 as u64; /// The tiling half waits only on the render and compute prefixes of the command's stage-prefix /// fields, its fragment half on the whole ones. #[allow(dead_code)] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6234eb18d337c4..401687c7be48af 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1814,7 +1814,7 @@ impl gpu::Gpu for Gpu { | crate::file::DRM_ASAHI_FEATURE_SPARSE_TILES | crate::file::DRM_ASAHI_FEATURE_COMPUTE_MATRIX | crate::file::DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES - | crate::file::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES + | crate::file::DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 | crate::file::DRM_ASAHI_FEATURE_WHOLE_ATTACHMENT_RENDERS; if queue::render::m5_render_submission::depth_msaa_compression_enabled() { params.features |= crate::file::DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 78a34e3d7ab4db..5ad3a6a128125f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -409,7 +409,9 @@ enum drm_asahi_feature { /** * @DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES: M5: render commands accept - * DRM_ASAHI_RENDER_STAGE_PREFIXES. + * DRM_ASAHI_RENDER_STAGE_PREFIXES in a shorter command form. The kernel + * no longer reports this bit; see + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. */ DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES = (1UL) << 25, @@ -440,6 +442,13 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_NO_QUERY_ACCESS as an ordering hint. */ DRM_ASAHI_FEATURE_COMPUTE_NO_QUERY_ACCESS = (1UL) << 30, + + /** + * @DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2: M5: render commands accept + * DRM_ASAHI_RENDER_STAGE_PREFIXES with the full struct + * drm_asahi_cmd_render. + */ + DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2 = (1UL) << 31, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -1369,7 +1378,7 @@ enum drm_asahi_render_flags { * whole ones. Earlier renders in the tiling prefix are waited on to * completion. The command must be the full struct drm_asahi_cmd_render. * Not combined with DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. Requires - * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES. + * DRM_ASAHI_FEATURE_RENDER_STAGE_PREFIXES_V2. */ DRM_ASAHI_RENDER_STAGE_PREFIXES = (1U << 23), From aabf80268191cc0b2e0b25f2fee60a64586c0b76 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 5 Nov 2026 00:15:59 -0800 Subject: [PATCH 071/171] drm/asahi: g17: M5: add the opt-in M5 thermal governor Add a host GPU power-budget governor for M5 (asahi.m5_thermal_governor, off by default). It holds the SoC hotspot near a target by lowering the budget the firmware reads, shares the heat with a CPU frequency ceiling, and ramps the fans with the hotspot under load. Publishers note GPU activity with or without the device mutex. [ rebased onto integration/aurora: rewritten from t6050-gpu 77dc6eb34210..60a4dd529842 (6 commits); SPTM/APIF removed; adapted to integration's Shared work items and the port's CLPC demand; comments and names reworded; original 77dc6eb34210, 0a75aae77faf, 284638c2a784, cc2c9850d389, f5de511cb487, 60a4dd529842 ] Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit fc1fadafdd0610c9c985c48a92c363f590205926 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/asahi_core.rs | 8 + drivers/gpu/drm/asahi/g17/initdata.rs | 46 + drivers/gpu/drm/asahi/g17/mod.rs | 110 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 4 + .../drm/asahi/g17/runtime/render/direct.rs | 1 + drivers/gpu/drm/asahi/g17/thermal.rs | 843 ++++++++++++++++++ 6 files changed, 1012 insertions(+) create mode 100644 drivers/gpu/drm/asahi/g17/thermal.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index e1747a772e853c..7d8f961f615101 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -95,6 +95,14 @@ module! { default: 0x200, description: "M5 mesh rendering buffer bound to API task (object-stage) render kicks only, in 64 KiB chunks; 0 disables the binding and the task feature", }, + m5_thermal_governor: u32 { + default: 0, + description: "M5 thermal flags: bit 0 enables the host GPU power budget controller, bit 2 lets the controller lower the CPU frequency ceiling while the GPU is budget-bound, bit 3 runs the fans at their maximum while it governs; default 0 preserves the existing policy", + }, + m5_thermal_target_mc: u32 { + default: 90000, + description: "M5 GPU power budget governor target for the SoC hotspot temperature computed by the SMC, in milli-degrees C (70000..104000)", + }, m5_depth_msaa_compression: u32 { default: 1, description: "Enable single-layer D32 compressed 2x/4x ZLS admission", diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 130e914d845104..6894277a3d416f 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1980,3 +1980,49 @@ impl super::recovery::Memory for InitData { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); } } + +/// M5 thermal governor inputs and the host power budget it publishes. +impl InitData { + /// M5 only: the live host GPU power budget word of Region C. + pub(super) fn m5_power_budget(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.region_c)?.word(super::thermal::REGION_C_POWER_BUDGET)?.load(Ordering::Relaxed)) + } + + /// M5 only: the GPU's current power draw in mW, as the firmware's power controller reports it + /// (integer word of the shared power-policy block). + pub(super) fn m5_gpu_power_mw(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + Ok(self.object(self.bundle)? + .word(super::fw::m5::initdata::M5_SHARED_LAYOUT.policy + super::thermal::POLICY_GPU_POWER)? + .load(Ordering::Relaxed)) + } + + /// M5 only: whether the firmware's emergency clamp is active, from its primary performance + /// limiter in the shared power-policy block. + pub(super) fn m5_firmware_clamped(&self) -> Result { + if !self.m5 { return Err(EINVAL); } + let block = self.object(self.bundle)?; + let base = super::fw::m5::initdata::M5_SHARED_LAYOUT.policy; + let mask = block.word(base + super::thermal::POLICY_LIMITER_MASK)?.load(Ordering::Relaxed); + let limit = block.word(base + super::thermal::POLICY_LIMITER_LIMIT)?.load(Ordering::Relaxed); + Ok(super::thermal::is_emergency_clamp(mask, limit)) + } + + /// M5 only: publishes the host GPU power budget like the effort input. + pub(super) fn publish_m5_power_budget(&self, budget_mw: u32) -> Result { + if !self.m5 { return Err(EINVAL); } + let region = self.object(self.region_c)?; + let offset = super::thermal::REGION_C_POWER_BUDGET; + let word = region.word(offset)?; + let line = region.pointer(offset, 4)?; + word.store(budget_mw, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the naturally aligned store. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 401687c7be48af..c90bf8c8b13409 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -64,6 +64,7 @@ pub(crate) mod stats; pub(crate) mod status; mod teardown; pub(crate) mod telemetry; +mod thermal; mod timeout; mod validation; @@ -686,6 +687,8 @@ struct Firmware { recovery: recovery::State, effort: power::Effort, hold: power::Hold, + /// M5: host GPU power budget governor (Region C's budget word). + governor: thermal::Governor, registration_power_release_pending: bool, variant: &'static fw::Variant, queues: runtime::Registry, @@ -798,10 +801,52 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { self.port.note_gpu_activity(); + self.primary.state.shared.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } + /// One governor step with the hotspot temperature reading; returns whether to run again and + /// what the CPU ceiling should do (applied by the caller without the device lock). + fn governor_tick(&mut self, temp_mc: Option, fans: bool) + -> (bool, thermal::CpuAdvice, bool) + { + let now = now_ns(); + // Work published since the last step, with or without the device mutex. + let published = self.primary.state.shared.governor_activity_ns.swap(0, Ordering::AcqRel); + if published != 0 { + self.governor.note_activity(published); + } + // Effort remains asserted until submitted work retires. A long job + // must not look idle merely because it has no subsequent submission. + if self.effort.applied() != 0 && thermal::requested() { + self.governor.note_activity(now); + } + let Ok(current) = self.init.m5_power_budget() else { + return (false, thermal::CpuAdvice::Release, false); + }; + self.governor.set_maximum(current); + let power = self.init.m5_gpu_power_mw().unwrap_or(0); + let clamped = self.init.m5_firmware_clamped().unwrap_or(false); + let (publish, again) = self.governor.tick(temp_mc, power, clamped, now); + if let Some(budget) = publish { + if self.init.publish_m5_power_budget(budget).is_err() { + return (false, thermal::CpuAdvice::Release, false); + } + } + let advice = self.governor.cpu_advice(power, now); + let want_fans = self.governor.fans_wanted(now); + // Keep ticking while the fans are still boosted, so the hold can expire. + (again || fans, advice, want_fans) + } + + /// Restore the calibrated maximum budget (shutdown). + fn governor_restore(&mut self) { + if let Some(budget) = self.governor.restore() { + let _ = self.init.publish_m5_power_budget(budget); + } + } + fn apply_effort(&mut self, effort: u32) -> Result { Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } @@ -941,6 +986,7 @@ const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; const FEED_WORK: u64 = 5; +const GOVERNOR_WORK: u64 = 6; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -997,6 +1043,14 @@ struct Shared { poll: DelayedWork, #[pin] idle: DelayedWork, + /// M5: the thermal governor runs (the firmware has the host power budget word). + governs: bool, + #[pin] + governor: DelayedWork, + governor_armed: AtomicBool, + /// When work was last published while the governor is requested (0: none since its last + /// step). Publishers without the device mutex set it too. + governor_activity_ns: AtomicU64, #[pin] grow: Work, #[pin] @@ -1011,6 +1065,10 @@ struct Shared { /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, + /// M5: the CPU frequency ceiling the thermal governor shares the SoC's heat with, and the + /// fan boost. Never taken with `state` held. + #[pin] + actuators: Mutex, #[pin] changed: CondVar, /// The port of the session in `state`; null before boot and once detachment begins. @@ -1027,6 +1085,30 @@ impl_has_delayed_work! { impl_has_delayed_work! { impl HasDelayedWork for Shared { self.idle } } +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.governor } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.governor_armed.store(false, Ordering::Release); + // Sensor reads may sleep: take them before the device lock. + let temp = thermal::hotspot_temperature_mc(); + let fans = this.actuators.lock().fans.boosted(); + let (again, advice, want_fans) = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => firmware.governor_tick(temp, fans), + None => (false, thermal::CpuAdvice::Release, false), + } + }; + // Frequency QoS updates and SMC accesses may sleep: outside the device lock. + this.actuators.lock().apply(advice, want_fans, temp, now_ns()); + if again { + this.queue_governor(); + } + } +} impl_has_work! { impl HasWork for Shared { self.grow } } @@ -1185,6 +1267,23 @@ impl Shared { .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); } + fn queue_governor(self: &Arc) { + if !self.governor_armed.swap(true, Ordering::AcqRel) { + let _ = workqueue::system_unbound().enqueue_delayed::, GOVERNOR_WORK>( + self.clone(), msecs_to_jiffies(thermal::PERIOD_MS)); + } + } + + /// M5: work was published. Arms the thermal governor while it is requested. Called at + /// actual publication, by publishers with or without the device mutex; callable from any + /// context. + pub(super) fn note_gpu_activity(self: &Arc) { + if self.governs && thermal::requested() { + self.governor_activity_ns.store(now_ns().max(1), Ordering::Release); + self.queue_governor(); + } + } + fn queue_idle(self: &Arc, delay_ns: u64) { // Effort::idle bounds this by the 16 ms idle interval. Round up so a // timer never changes the firmware effort word before that deadline. @@ -1374,11 +1473,16 @@ impl Gpu { housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), + governs: m5, + governor <- new_delayed_work!("g17::Shared::governor"), + governor_armed: AtomicBool::new(false), + governor_activity_ns: AtomicU64::new(0), grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), + actuators <- new_mutex!(thermal::Actuators::new(), "g17::Shared::actuators"), changed <- new_condvar!("g17::Shared::changed"), port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), @@ -1590,6 +1694,7 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), + governor: thermal::Governor::new(), registration_power_release_pending: false, variant: shared.variant, queues <- runtime::Registry::with_waiters( @@ -1749,6 +1854,11 @@ impl Gpu { self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); self.shared.disable_timer(&self.shared.idle); + self.shared.disable_timer(&self.shared.governor); + if let Some(firmware) = (*self.shared.state.lock()).as_deref_mut() { + firmware.governor_restore(); + } + self.shared.actuators.lock().release(); self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4a2a116f83bc80..ff88b43b1d0d6a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1380,6 +1380,7 @@ impl DeferredBatch { } struct ComputeHost<'a> { + shared: &'a Arc, init: &'a super::initdata::InitData, regs: &'a crate::regs::G17Resources, port: &'a super::port::Port, @@ -1439,6 +1440,7 @@ impl compute::Host for ComputeHost<'_> { self.port.notify(message) } fn note_submission(&mut self) -> Result { + self.shared.note_gpu_activity(); self.effort.submit(); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) } @@ -1607,6 +1609,7 @@ impl super::Firmware { return Err(EAGAIN); } let mut host = ComputeHost { + shared: &self.primary.state.shared, init: &self.init, regs: &self.bridge.regs, port: &self.port, @@ -1863,6 +1866,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { Ok(_) => {} Err(error) if error == EBUSY => return None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index a0626544002178..a19ff9ea7cab77 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -297,6 +297,7 @@ impl Backend { if !port.note_submission() { return None; } + self.shared.note_gpu_activity(); match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Render) { Ok(_) => {} Err(error) if error == EBUSY => return None, diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs new file mode 100644 index 00000000000000..bea0b9708352d4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -0,0 +1,843 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU power budget governor for M5 (J714). +//! +//! The firmware limits the GPU to the host power budget word in Region C within about a second +//! (the calibrated maximum is written there at initialization). Left at that maximum under +//! sustained load, the SoC hotspot temperature computed by the SMC (about 6 C above the hottest +//! GPU die sensor while the GPU dominates) spends time above roughly 97 C, and the firmware's +//! emergency path then pins the GPU to its base performance state for 20-40 s, repeatedly. This +//! governor reads the hotspot temperature through the thermal framework once per second while the +//! GPU is busy and, once it reaches the engage temperature, lowers the budget with a PI controller +//! (plus a feed-forward on the temperature's rise) so it settles at the target, below that region. +//! +//! The controller works below the GPU's actual draw (the calibrated maximum is far above it). It +//! keeps the integral it converged to across releases, so after a release or a firmware clamp it +//! resumes from that budget; after a long period without control the memory decays to the +//! cold-start budget, never to the ceiling. It holds while the firmware's emergency clamp is +//! active and never releases then. The default target keeps the hotspot below the level at which +//! the firmware clamps even with a heat-soaked chassis. As a safety net, each emergency clamp the +//! governor sees lowers its effective target by 1 C (at most 6 C, not below 84 C), and every 10 +//! minutes without one raises it again by 1 C. It publishes the budget like the effort input, only when it changes, and +//! restores the calibrated maximum on release, when the GPU goes idle, when no sensor can be read +//! and at shutdown. Below the engage temperature nothing is written. + +use core::ffi::{c_char, c_int, c_void, CStr}; +use kernel::{bindings, prelude::*}; + +extern "C" { + fn thermal_zone_get_zone_by_name(name: *const c_char) -> *mut c_void; + fn thermal_zone_get_temp(zone: *mut c_void, temp: *mut c_int) -> c_int; + fn apple_smc_read(smc: *mut c_void, key: u32, buf: *mut c_void, size: usize) -> c_int; + fn apple_smc_write(smc: *mut c_void, key: u32, buf: *const c_void, size: usize) -> c_int; +} + +/// SoC hotspot temperature computed by the SMC (registered as a thermal zone by the SMC driver): +/// the input of the firmware's emergency clamp. +const J714_HOTSPOT_ZONE: &CStr = c"smc-TCMb"; +/// The hottest of the 42 SMC GPU die sensors of J714 near the limit, and their offset to the +/// hotspot while the GPU dominates: the fallback input when the hotspot cannot be read. +const J714_GPU_DIE_ZONES: [&CStr; 8] = [ + c"smc-Tg08", c"smc-Tg12", c"smc-Tg0O", c"smc-Tg0X", c"smc-Tg29", c"smc-Tg2P", c"smc-Tg1U", + c"smc-Tg43", +]; +const GPU_DIE_TO_HOTSPOT_MC: i32 = 6000; + +/// Region C word holding the host GPU power budget (mW). +pub(crate) const REGION_C_POWER_BUDGET: usize = 0x88; +/// Shared power-policy word holding the GPU's current power draw (mW), written by the firmware. +pub(crate) const POLICY_GPU_POWER: usize = 0xad4; +/// Shared power-policy words of the firmware's primary performance limiter: the mask of active +/// limit reasons and the limit. The emergency reason with the limit at its floor is the clamp. +pub(crate) const POLICY_LIMITER_MASK: usize = 0x44; +pub(crate) const POLICY_LIMITER_LIMIT: usize = 0x48; +const LIMITER_EMERGENCY: u32 = 0x4; +const LIMITER_FLOOR: u32 = 100; +/// Control period while the GPU is busy. +pub(crate) const PERIOD_MS: u32 = 1000; +/// Restore the maximum after this long without a submission. +const IDLE_RESTORE_NS: u64 = 2_000_000_000; +/// Engage when the hotspot reaches this; release below it minus the hysteresis once the +/// controller has returned to its ceiling. +const ENGAGE_BELOW_TARGET_MC: i32 = 4000; +const RELEASE_HYSTERESIS_MC: i32 = 3000; +/// Lowest budget the governor sets (mW). +const FLOOR_MW: i64 = 10_000; +/// Lowest ceiling of the controller (mW): the GPU's draw at engage, but at least this. +const CEILING_MIN_MW: i64 = 40_000; +/// Proportional gain (mW per C) and integral gain (mW per C per second). Power follows the +/// budget within a second; the die settles over 30-60 s at ~1.2-1.5 C per W. +const KP_MW_PER_C: i64 = 1500; +const KI_MW_PER_C_S: i64 = 50; +/// Feed-forward on the hotspot's rise (mW per C/s), applied only while it rises, to curb the +/// overshoot when the GPU restarts from cool. +const KF_MW_PER_C_PER_S: i64 = 2000; +/// Without a converged integral, start at this fraction of the ceiling: the budget that holds +/// the target is about half the GPU's unconstrained draw. +const INITIAL_NUM: i64 = 3; +const INITIAL_DEN: i64 = 5; +/// The converged integral is kept for this long after the controller was last active, then +/// decays linearly to the cold-start integral over the decay time. +const MEMORY_HOLD_NS: u64 = 60_000_000_000; +const MEMORY_DECAY_NS: u64 = 300_000_000_000; +/// Effective target reduction per emergency clamp seen while governing, its limit, the lowest +/// effective target it leads to, and the time without a clamp after which it is reduced by one +/// step again. +const CLAMP_TARGET_STEP_MC: i32 = 1000; +const CLAMP_TARGET_MAX_MC: i32 = 6000; +const CLAMP_TARGET_FLOOR_MC: i32 = 84_000; +const CLAMP_TARGET_RECOVER_NS: u64 = 600_000_000_000; +/// Budget changes smaller than this are not published. +const STEP_MW: i64 = 250; + +/// CPU ceiling: one step lowers every cpufreq policy's maximum by this share of its hardware +/// maximum, down to the floor share. +const CPU_STEP_PERCENT: u32 = 4; +const CPU_FLOOR_PERCENT: u32 = 60; +/// Minimum time between CPU ceiling steps; the hotspot and the GPU draw follow within it. +const CPU_DWELL_NS: u64 = 2_000_000_000; +/// The GPU uses its budget when it draws at least this share of it, and leaves headroom below +/// the lower share (it then waits for the CPU). +const GPU_BOUND_PERCENT: i64 = 90; +const GPU_SLACK_PERCENT: i64 = 70; +/// The CPU's own heat: the hotspot this far above the target while the GPU budget is within the +/// slack of its floor. +const CPU_HEAT_MARGIN_MC: i32 = 2000; +const CPU_HEAT_FLOOR_SLACK_MW: i64 = 1000; + +fn enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 1 != 0 +} +fn cpu_balance_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 4 != 0 +} +fn fan_boost_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 8 != 0 +} +/// The proportional fan curve is on unless bit 4 (16) of m5_thermal_governor opts out. +fn fan_curve_enabled() -> bool { + crate::module_parameters::m5_thermal_governor.value() & 16 == 0 +} +pub(super) fn requested() -> bool { + enabled() +} +fn target_mc() -> i32 { + let parameter = crate::module_parameters::m5_thermal_target_mc.value(); + i32::try_from(parameter).unwrap_or(90_000).clamp(70_000, 104_000) +} + +/// Whether the firmware's primary limiter words describe the emergency clamp. +pub(crate) fn is_emergency_clamp(mask: u32, limit: u32) -> bool { + mask & LIMITER_EMERGENCY != 0 && limit <= LIMITER_FLOOR +} + +/// Pins the sensor provider and serializes reads with its probe and removal. +/// Its thermal zones are managed by the hwmon device registered during probe; +/// removal releases that device's resources while holding the provider lock. +struct SensorProvider(*mut bindings::device); + +impl SensorProvider { + fn acquire() -> Option { + // SAFETY: the bus and name have static lifetimes. The bus iterator + // returns an owned device reference on success. + let device = unsafe { + bindings::bus_find_device( + &bindings::platform_bus_type, + core::ptr::null_mut(), + c"macsmc-hwmon".as_ptr().cast(), + Some(bindings::device_match_name), + ) + }; + if device.is_null() { + return None; + } + // SAFETY: the owned reference keeps the device and mutex alive. No + // GPU driver lock is held during acquisition or sensor reads. + unsafe { bindings::mutex_lock(&raw mut (*device).mutex) }; + let provider = Self(device); + // SAFETY: the provider's lock protects binding and unbinding. + if unsafe { (*device).driver.is_null() } { + return None; + } + Some(provider) + } +} + +impl Drop for SensorProvider { + fn drop(&mut self) { + // SAFETY: acquire obtained this reference and locked this mutex on + // the current thread; the guard never leaves the sensor-read call. + unsafe { + bindings::mutex_unlock(&raw mut (*self.0).mutex); + bindings::put_device(self.0); + } + } +} + +/// The provider guard prevents its managed zones from disappearing during +/// lookup and read. Do not retain a zone pointer after dropping the guard. +fn zone_temperature_mc(_provider: &SensorProvider, name: &CStr) -> Option { + // SAFETY: name is a NUL-terminated string that outlives the call. + let zone = unsafe { thermal_zone_get_zone_by_name(name.as_ptr()) }; + if zone.is_null() || (zone as usize) >= usize::MAX - 4095 { + return None; + } + let mut temp: c_int = 0; + // SAFETY: the provider guard protects the zone's lifetime, and temp is a + // valid destination for this call. + (unsafe { thermal_zone_get_temp(zone, &mut temp) } == 0) + .then_some(temp) + .filter(|temp| (0..=125_000).contains(temp)) +} + +/// SoC hotspot temperature in milli-C (the hottest GPU die sensor plus its offset when the hotspot +/// cannot be read), or None when no sensor can be read. Called without driver locks held: zone +/// lookups and sensor reads may sleep. +pub(crate) fn hotspot_temperature_mc() -> Option { + let provider = SensorProvider::acquire()?; + zone_temperature_mc(&provider, J714_HOTSPOT_ZONE).or_else(|| { + J714_GPU_DIE_ZONES.iter().filter_map(|name| zone_temperature_mc(&provider, name)).max() + .map(|temp| temp + GPU_DIE_TO_HOTSPOT_MC) + }) +} + +/// Keeps the fans boosted at least this long once boosted, so a short release or idle period +/// (a loading screen) does not cycle them. +const FAN_HOLD_NS: u64 = 30_000_000_000; + +/// Proportional fan curve on the hotspot while the GPU is busy: the SMC's automatic mode keeps +/// the fans near their minimum under sustained GPU load on Linux. From FAN_CURVE_START_MC to +/// the governor target the fans ramp from the minimum to two thirds of the range, and on to +/// FAN_CURVE_TOP_PERMILLE at FAN_CURVE_TOP_MC. +const FAN_CURVE_START_MC: i32 = 84_000; +const FAN_CURVE_TOP_MC: i32 = 98_000; +const FAN_CURVE_TARGET_PERMILLE: i32 = 660; +const FAN_CURVE_TOP_PERMILLE: i32 = 800; +/// Below this the curve releases the fans to automatic mode (after FAN_HOLD_NS). +const FAN_CURVE_RELEASE_MC: i32 = 82_000; +/// A new target is written only when it moves at least this far. +const FAN_CURVE_STEP_RPM: u32 = 150; + +/// The permille of the fan range the curve asks for at hotspot `temp_mc`. +fn fan_curve_permille(temp_mc: i32, target_mc: i32) -> i32 { + let target_mc = target_mc.max(FAN_CURVE_START_MC + 1000); + if temp_mc <= FAN_CURVE_START_MC { + 0 + } else if temp_mc <= target_mc { + (temp_mc - FAN_CURVE_START_MC) * FAN_CURVE_TARGET_PERMILLE / (target_mc - FAN_CURVE_START_MC) + } else { + let top = FAN_CURVE_TOP_MC.max(target_mc + 1000); + (FAN_CURVE_TARGET_PERMILLE + (temp_mc.min(top) - target_mc) + * (FAN_CURVE_TOP_PERMILLE - FAN_CURVE_TARGET_PERMILLE) / (top - target_mc)) + .min(FAN_CURVE_TOP_PERMILLE) + } +} + +/// The integer value of a non-negative SMC float (truncated), without FPU use. +fn f32_bits_to_u32(bits: u32) -> u32 { + let exponent = ((bits >> 23) & 0xff) as i32 - 127; + if bits >> 31 != 0 || exponent < 0 { + return 0; + } + let mantissa = (bits & 0x7f_ffff) | 0x80_0000; + if exponent >= 23 { + mantissa.checked_shl((exponent - 23) as u32).unwrap_or(u32::MAX) + } else { + mantissa >> (23 - exponent) + } +} + +/// The SMC float encoding of `value` (exact below 2^24), without FPU use. +fn u32_to_f32_bits(value: u32) -> u32 { + if value == 0 { + return 0; + } + let exponent = 31 - value.leading_zeros(); + let mantissa = if exponent >= 23 { value >> (exponent - 23) } else { value << (23 - exponent) }; + ((exponent + 127) << 23) | (mantissa & 0x7f_ffff) +} + +fn smc_key(name: &[u8; 4]) -> u32 { + u32::from_be_bytes(*name) +} + +/// Runs `f` with the SMC core of the sensor provider, under the provider lock (which serializes +/// with its unbinding). +fn with_smc(f: impl FnOnce(*mut c_void) -> Option) -> Option { + let provider = SensorProvider::acquire()?; + // SAFETY: the provider lock keeps the device bound; the SMC core is its parent, whose driver + // data is the core's state for as long as a child is bound. + let smc = unsafe { + let parent = (*provider.0).parent; + if parent.is_null() { return None; } + (*parent).driver_data + }; + if smc.is_null() { None } else { f(smc) } +} + +/// Fan boost: while the governor limits the GPU, every fan runs at its hardware maximum instead of +/// the SMC's automatic curve, which settles near two thirds of the maximum at the governor's +/// target. +/// Released (automatic mode) when the governor releases or the GPU goes idle, after a hold. +pub(crate) struct FanBoost { + boosted: bool, + since_ns: u64, + /// The fan target the curve last wrote (rpm), 0 while the fans are automatic. + curve_rpm: u32, +} + +impl FanBoost { + const fn new() -> Self { + Self { boosted: false, since_ns: 0, curve_rpm: 0 } + } + + /// Puts every fan in manual mode at `permille` of its minimum-to-maximum range, or back in + /// automatic mode for `None`. Returns the first fan's target rpm, 0 for automatic. + fn set_curve(&mut self, permille: Option) -> Option { + with_smc(|smc| { + let mut count = 0u8; + // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. + if unsafe { apple_smc_read(smc, smc_key(b"FNum"), (&raw mut count).cast(), 1) } < 0 { + return None; + } + let mut first = 0; + for fan in 0..count.min(4) { + let digit = b'0' + fan; + let mode = u8::from(permille.is_some()); + let (mut minimum, mut maximum) = ([0u8; 4], [0u8; 4]); + // SAFETY: as above. + unsafe { + if let Some(permille) = permille { + if apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'n']), minimum.as_mut_ptr().cast(), 4) < 0 + || apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'x']), maximum.as_mut_ptr().cast(), 4) < 0 + { + return None; + } + let low = f32_bits_to_u32(u32::from_le_bytes(minimum)); + let high = f32_bits_to_u32(u32::from_le_bytes(maximum)).max(low); + let rpm = low + (high - low) * permille.clamp(0, 1000) as u32 / 1000; + if fan == 0 { + first = rpm; + } + let target = u32_to_f32_bits(rpm).to_le_bytes(); + if apple_smc_write(smc, smc_key(&[b'F', digit, b'T', b'g']), target.as_ptr().cast(), 4) < 0 { + return None; + } + } + if apple_smc_write(smc, smc_key(&[b'F', digit, b'm', b'd']), (&raw const mode).cast(), 1) < 0 { + return None; + } + } + } + Some(first) + }) + } + + /// One governor step of the proportional curve: `busy` while the governor runs a busy GPU. + fn apply_curve(&mut self, busy: bool, temp_mc: Option, target_mc: i32, now_ns: u64) { + let permille = temp_mc.filter(|_| busy).map(|t| fan_curve_permille(t, target_mc)).unwrap_or(0); + if permille > 0 { + self.since_ns = now_ns; + // The target rpm moves with the range only when it moved enough. + if let Some(rpm) = self.set_curve_if_moved(permille) { + self.curve_rpm = rpm; + } + } else if self.curve_rpm != 0 + && (!busy || temp_mc.is_none_or(|t| t < FAN_CURVE_RELEASE_MC)) + && now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS + && self.set_curve(None).is_some() + { + self.curve_rpm = 0; + } + } + + fn set_curve_if_moved(&mut self, permille: i32) -> Option { + // The range is read on every write; skip the write while the target barely moves. + let estimate = with_smc(|smc| { + let mut bytes = [[0u8; 4]; 2]; + // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. + unsafe { + if apple_smc_read(smc, smc_key(b"F0Mn"), bytes[0].as_mut_ptr().cast(), 4) < 0 + || apple_smc_read(smc, smc_key(b"F0Mx"), bytes[1].as_mut_ptr().cast(), 4) < 0 + { + return None; + } + } + let low = f32_bits_to_u32(u32::from_le_bytes(bytes[0])); + let high = f32_bits_to_u32(u32::from_le_bytes(bytes[1])).max(low); + Some(low + (high - low) * permille.clamp(0, 1000) as u32 / 1000) + })?; + if self.curve_rpm != 0 && estimate.abs_diff(self.curve_rpm) < FAN_CURVE_STEP_RPM { + return None; + } + self.set_curve(Some(permille)) + } + + fn set(&mut self, boost: bool) -> bool { + with_smc(|smc| { + let mut count = 0u8; + // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. + if unsafe { apple_smc_read(smc, smc_key(b"FNum"), (&raw mut count).cast(), 1) } < 0 { + return None; + } + let mut ok = count != 0; + for fan in 0..count.min(4) { + let digit = b'0' + fan; + let mode = u8::from(boost); + let mut maximum = [0u8; 4]; + // SAFETY: as above. + unsafe { + if boost + && apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'x']), maximum.as_mut_ptr().cast(), 4) < 0 + { + ok = false; + continue; + } + ok &= apple_smc_write(smc, smc_key(&[b'F', digit, b'm', b'd']), (&raw const mode).cast(), 1) >= 0; + if boost { + ok &= apple_smc_write(smc, smc_key(&[b'F', digit, b'T', b'g']), maximum.as_ptr().cast(), 4) >= 0; + } + } + } + Some(ok) + }) + .unwrap_or(false) + } + + /// Applies one governor step's request. Called from the governor work without driver locks: + /// SMC accesses sleep. + fn apply(&mut self, want: bool, temp_mc: Option, now_ns: u64) { + if !fan_boost_enabled() { + if fan_curve_enabled() { + self.apply_curve(want, temp_mc, target_mc(), now_ns); + } else if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + let want = want && fan_boost_enabled(); + if want { + self.since_ns = now_ns; + } + if want && !self.boosted { + self.boosted = self.set(true); + } else if !want && self.boosted + && (!fan_boost_enabled() || now_ns.saturating_sub(self.since_ns) >= FAN_HOLD_NS) + { + if self.set(false) { + self.boosted = false; + } + } + } + + pub(crate) fn release(&mut self) { + if self.boosted && self.set(false) { + self.boosted = false; + } + if self.curve_rpm != 0 && self.set_curve(None).is_some() { + self.curve_rpm = 0; + } + } + + pub(crate) fn boosted(&self) -> bool { + self.boosted || self.curve_rpm != 0 + } +} + +impl Drop for FanBoost { + fn drop(&mut self) { + self.release(); + } +} + +/// The host's thermal actuators besides the GPU budget, applied outside the device lock. +pub(crate) struct Actuators { + pub(crate) cpu: CpuCeiling, + pub(crate) fans: FanBoost, +} + +impl Actuators { + pub(crate) const fn new() -> Self { + Self { cpu: CpuCeiling::new(), fans: FanBoost::new() } + } + + pub(crate) fn apply(&mut self, advice: CpuAdvice, fans: bool, temp_mc: Option, now_ns: u64) { + self.cpu.apply(advice, now_ns); + self.fans.apply(fans, temp_mc, now_ns); + } + + pub(crate) fn release(&mut self) { + self.cpu.release(); + self.fans.release(); + } +} + +/// What the CPU ceiling should do after a governor step. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum CpuAdvice { + /// Keep the current ceiling. + Hold, + /// The GPU draws its whole thermally limited budget: give it the CPU's share of the heat. + Lower, + /// The GPU is not limited by its budget (or waits for the CPU): give the CPU a step back. + Raise, + /// Not governing: drop the ceiling. + Release, +} + +/// One cpufreq policy's maximum-frequency request. +struct CpuRequest { + policy: *mut bindings::cpufreq_policy, + request: KBox, + max_khz: u32, +} + +// SAFETY: the policy reference and the request are only used from the governor work, under the +// ceiling's lock. +unsafe impl Send for CpuRequest {} + +/// Shares the SoC's heat between the CPU and the GPU while the governor limits the GPU: when the +/// GPU is budget-bound it lowers the maximum +/// frequency of every CPU cluster one step at a time, and gives the steps back once the GPU no +/// longer uses its budget (it then waits for the CPU) or is no longer limited. The requests are +/// ordinary frequency QoS maxima, combined with any other limit on the policies. +pub(crate) struct CpuCeiling { + requests: KVec, + level: u32, + last_change_ns: u64, +} + +impl CpuCeiling { + pub(crate) const fn new() -> Self { + Self { requests: KVec::new(), level: 0, last_change_ns: 0 } + } + + fn max_level() -> u32 { + (100 - CPU_FLOOR_PERCENT) / CPU_STEP_PERCENT + } + + fn limit_khz(max_khz: u32, level: u32) -> i32 { + let share = 100 - level.min(Self::max_level()) * CPU_STEP_PERCENT; + (u64::from(max_khz) * u64::from(share) / 100).min(i32::MAX as u64) as i32 + } + + /// Takes a request on every policy (each policy once, through its first CPU). + fn acquire(&mut self) -> bool { + if !self.requests.is_empty() { + return true; + } + // SAFETY: nr_cpu_ids is set once during early boot. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: cpufreq_cpu_get accepts any CPU number and returns a counted reference or + // null. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + // SAFETY: the reference keeps the policy alive. + let (first, max_khz) = unsafe { ((*policy).cpu, (*policy).cpuinfo.max_freq) }; + let request = if first == cpu && max_khz != 0 { + // The zeroed (default) request is the unused state freq_qos_add_request expects. + KBox::new(bindings::freq_qos_request::default(), GFP_KERNEL).ok() + } else { + None + }; + let Some(mut request) = request.filter(|_| self.requests.reserve(1, GFP_KERNEL).is_ok()) + else { + // SAFETY: drops the reference taken above. + unsafe { bindings::cpufreq_cpu_put(policy) }; + continue; + }; + // SAFETY: the policy outlives the request (the reference is held until removal) and + // the request is in its unused state, at a stable address in its box. + let added = unsafe { + bindings::freq_qos_add_request( + &raw mut (*policy).constraints, + &mut *request, + bindings::freq_qos_req_type_FREQ_QOS_MAX, + Self::limit_khz(max_khz, 0), + ) + }; + if added < 0 { + // SAFETY: drops the reference taken above. + unsafe { bindings::cpufreq_cpu_put(policy) }; + continue; + } + // Capacity was reserved above, so this cannot fail. + let _ = self.requests.push(CpuRequest { policy, request, max_khz }, GFP_KERNEL); + } + !self.requests.is_empty() + } + + /// Removes every request and the policy references. + pub(crate) fn release(&mut self) { + for mut entry in self.requests.drain_all() { + // SAFETY: the request was added in acquire and the policy reference is still held. + unsafe { + bindings::freq_qos_remove_request(&mut *entry.request); + bindings::cpufreq_cpu_put(entry.policy); + } + } + self.level = 0; + } + + fn publish(&mut self) { + for entry in self.requests.iter_mut() { + let limit = Self::limit_khz(entry.max_khz, self.level); + // SAFETY: the request was added in acquire and stays valid until release. + unsafe { bindings::freq_qos_update_request(&mut *entry.request, limit) }; + } + } + + /// Applies one governor step's advice. Called from the governor work without driver locks: + /// frequency QoS updates may sleep. + pub(crate) fn apply(&mut self, advice: CpuAdvice, now_ns: u64) { + if !cpu_balance_enabled() || advice == CpuAdvice::Release { + if !self.requests.is_empty() { + self.release(); + } + return; + } + if now_ns.saturating_sub(self.last_change_ns) < CPU_DWELL_NS { + return; + } + match advice { + CpuAdvice::Lower if self.level < Self::max_level() => { + if !self.acquire() { + return; + } + self.level += 1; + } + CpuAdvice::Raise if self.level > 0 => { + self.level -= 1; + if self.level == 0 { + self.release(); + self.last_change_ns = now_ns; + return; + } + } + _ => return, + } + self.last_change_ns = now_ns; + self.publish(); + } +} + +impl Drop for CpuCeiling { + fn drop(&mut self) { + self.release(); + } +} + +pub(crate) struct Governor { + maximum_mw: u32, + budget_mw: u32, + integral_mw: i64, + /// Upper limit of the controller while active: the GPU's draw at engage. + ceiling_mw: i64, + /// Integral the controller last converged to (output inside its range), kept across releases. + converged_mw: i64, + active: bool, + last_tick_ns: u64, + last_activity_ns: u64, + /// Last tick at which the controller was active, for the memory decay. + last_active_ns: u64, + last_temp_mc: Option, + /// Effective target reduction learnt from emergency clamps, and when it last changed. + target_offset_mc: i32, + target_offset_ns: u64, + was_clamped: bool, +} + +impl Governor { + pub(crate) const fn new() -> Self { + Self { maximum_mw: 0, budget_mw: 0, integral_mw: 0, ceiling_mw: 0, converged_mw: 0, active: false, + last_tick_ns: 0, last_activity_ns: 0, last_active_ns: 0, last_temp_mc: None, target_offset_mc: 0, + target_offset_ns: 0, was_clamped: false } + } + + /// The firmware accepted a submission; returns whether the control loop should run. + pub(crate) fn note_activity(&mut self, now_ns: u64) -> bool { + self.last_activity_ns = now_ns; + requested() || self.budget_mw != self.maximum_mw + } + + /// Learn the calibrated maximum from the budget word before the first change. + pub(crate) fn set_maximum(&mut self, current_mw: u32) { + if self.maximum_mw == 0 && i64::from(current_mw) >= FLOOR_MW { + self.maximum_mw = current_mw; + self.budget_mw = current_mw; + } + } + + /// The budget to publish while stopping, if it differs from the maximum. + pub(crate) fn restore(&mut self) -> Option { + self.active = false; + self.integral_mw = 0; + self.converged_mw = 0; + self.target_offset_mc = 0; + if self.maximum_mw == 0 || self.budget_mw == self.maximum_mw { + return None; + } + self.budget_mw = self.maximum_mw; + Some(self.budget_mw) + } + + /// Integral to engage with: the converged one while it is fresh, decaying to the cold-start + /// integral (a fraction of the ceiling) after a long period without control. + fn resume_integral(&self, now_ns: u64) -> i64 { + let initial = self.ceiling_mw * INITIAL_NUM / INITIAL_DEN; + if self.converged_mw == 0 { + return initial; + } + let kept = self.converged_mw.min(self.ceiling_mw); + let cool_ns = now_ns.saturating_sub(self.last_active_ns); + if cool_ns <= MEMORY_HOLD_NS { + return kept; + } + let decay_ns = (cool_ns - MEMORY_HOLD_NS).min(MEMORY_DECAY_NS); + kept + (initial - kept) * (decay_ns / 1_000_000) as i64 / (MEMORY_DECAY_NS / 1_000_000) as i64 + } + + /// Learn from the firmware's emergency clamp: lower the effective target by a step on each + /// clamp seen while governing, raise it again after a long period without one. + fn adapt_target(&mut self, clamped: bool, now_ns: u64) { + if clamped && !self.was_clamped && self.active { + let limit = (target_mc() - CLAMP_TARGET_FLOOR_MC).clamp(0, CLAMP_TARGET_MAX_MC); + self.target_offset_mc = (self.target_offset_mc + CLAMP_TARGET_STEP_MC).min(limit); + self.target_offset_ns = now_ns; + } else if !clamped && self.target_offset_mc > 0 + && now_ns.saturating_sub(self.target_offset_ns) >= CLAMP_TARGET_RECOVER_NS + { + self.target_offset_mc -= CLAMP_TARGET_STEP_MC; + self.target_offset_ns = now_ns; + } + self.was_clamped = clamped; + } + + /// How the CPU ceiling should follow the last step, from the GPU's draw (mW) against the + /// budget it was given. While governing, a GPU that draws its whole budget below its ceiling + /// is limited by heat the CPU shares; one that leaves headroom waits for the CPU. When the + /// hotspot stays above the target with the GPU budget at its floor, the heat is the CPU's + /// and only the CPU can keep the firmware from clamping the GPU. + pub(crate) fn cpu_advice(&self, power_mw: u32, now_ns: u64) -> CpuAdvice { + if self.maximum_mw == 0 || now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS { + return CpuAdvice::Release; + } + if !self.active { + // Released by the controller: hand the steps back one at a time, so the CPU's heat + // returns gradually and does not re-engage the governor at once. + return CpuAdvice::Raise; + } + let budget = i64::from(self.budget_mw); + let power = i64::from(power_mw); + let target = target_mc() - self.target_offset_mc; + let temp = self.last_temp_mc.unwrap_or(target); + if temp >= target + CPU_HEAT_MARGIN_MC + && budget <= FLOOR_MW.min(self.ceiling_mw) + CPU_HEAT_FLOOR_SLACK_MW + { + CpuAdvice::Lower + } else if budget < self.ceiling_mw && power * 100 >= budget * GPU_BOUND_PERCENT { + CpuAdvice::Lower + } else if budget >= self.ceiling_mw + || (power * 100 < budget * GPU_SLACK_PERCENT && temp < target) + { + // A GPU waiting for the CPU gets it back only below the target: above it, the + // controller is still taking heat out. + CpuAdvice::Raise + } else { + CpuAdvice::Hold + } + } + + /// Whether the fans should be boosted: while the controller governs a busy GPU. + pub(crate) fn fans_wanted(&self, now_ns: u64) -> bool { + self.maximum_mw != 0 && self.active + && now_ns.saturating_sub(self.last_activity_ns) < IDLE_RESTORE_NS + } + + /// One control step with the hotspot temperature, the GPU's current draw (mW) and whether the + /// firmware's emergency clamp is active. Returns the budget to publish (if it changed) and + /// whether to run again. + pub(crate) fn tick(&mut self, temp_mc: Option, power_mw: u32, clamped: bool, now_ns: u64) + -> (Option, bool) + { + let maximum = i64::from(self.maximum_mw); + let idle = now_ns.saturating_sub(self.last_activity_ns) >= IDLE_RESTORE_NS; + let dt_ns = now_ns.saturating_sub(self.last_tick_ns).min(2 * IDLE_RESTORE_NS); + self.last_tick_ns = now_ns; + let previous_mc = core::mem::replace(&mut self.last_temp_mc, temp_mc); + if enabled() { + self.adapt_target(clamped, now_ns); + } + let target = target_mc() - self.target_offset_mc; + let desired = match temp_mc { + _ if maximum == 0 => return (None, false), + None => { + self.active = false; + maximum + } + _ if idle || !enabled() => { + self.active = false; + maximum + } + Some(temp) => { + let error_mc = i64::from(target - temp); + let proportional = KP_MW_PER_C * error_mc / 1000; + let rise_mc_per_s = match previous_mc { + Some(previous) if temp > previous && dt_ns >= 1_000_000 => + i64::from(temp - previous) * 1000 / (dt_ns / 1_000_000) as i64, + _ => 0, + }; + let feed_forward = KF_MW_PER_C_PER_S * rise_mc_per_s / 1000; + if !self.active && temp >= target - ENGAGE_BELOW_TARGET_MC { + self.active = true; + self.ceiling_mw = i64::from(power_mw).max(CEILING_MIN_MW).clamp(FLOOR_MW, maximum); + self.integral_mw = self.resume_integral(now_ns); + } + if self.active && clamped { + // The firmware's emergency clamp: hold the budget and the integral until it + // ends, so the GPU resumes at the converged budget. + self.last_active_ns = now_ns; + i64::from(self.budget_mw) + } else if self.active { + self.last_active_ns = now_ns; + let floor = FLOOR_MW.min(self.ceiling_mw); + let candidate = self.integral_mw + KI_MW_PER_C_S * error_mc * (dt_ns / 1_000_000) as i64 / 1_000_000; + let output = proportional + candidate - feed_forward; + // Anti-windup: integrate only while the output is inside its range; at a + // limit, back-calculate the integral so the output sits on the limit. + let output = if output > self.ceiling_mw { + self.integral_mw = self.ceiling_mw - proportional + feed_forward; + self.ceiling_mw + } else if output < floor { + self.integral_mw = floor - proportional + feed_forward; + floor + } else { + self.integral_mw = candidate; + self.converged_mw = candidate; + output + }; + if output >= self.ceiling_mw + && temp < target - ENGAGE_BELOW_TARGET_MC - RELEASE_HYSTERESIS_MC + { + // Released: back to the calibrated maximum. + self.active = false; + maximum + } else { + output + } + } else { + maximum + } + } + }; + let desired = if desired >= maximum { maximum } else { desired - desired % STEP_MW }; + let publish = (desired != i64::from(self.budget_mw)).then(|| { + self.budget_mw = desired as u32; + self.budget_mw + }); + let again = (!idle && requested()) || self.budget_mw != self.maximum_mw; + (publish, again) + } +} From 368b72b9a4e7495e1b5045bcbd090c379d6cc57c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 08:15:17 -0400 Subject: [PATCH 072/171] drm/asahi: g17: signal a cancelled job's fence before drm_sched frees it drm_sched_fini cancels each job left on the pending list through cancel_job and frees it right after, while the job's completion callback is still registered on the scheduler fence; cancel_job must signal that fence. When the queue could not settle the packet in time (a busy or failed runtime, or a settle racing another queue's timeout handling), cancel returned with the fence unsignalled. A later settle then ran the freed job's callback: drm_sched_job_done on a job with no s_fence, or a NULL callback once the memory was reused, both seen as oopses in drm_sched_job_timedout under 12-process synchronization CTS. Fail such a completion while its queue still owns it (the VM stays pinned until the queue retires it) and wait briefly for a signaller that already claimed it. Signed-off-by: Diljit Singh Signed-off-by: Jeff Queisser (cherry picked from commit d00f1f4531df502420f96012e2e48777b51286c0 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 2b501e9c723985..f2c781c668c97c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -8,7 +8,7 @@ use crate::g17::{ job::{self, Engine, Packet}, }; use core::sync::atomic::Ordering; -use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; +use kernel::{dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { @@ -323,6 +323,26 @@ impl job::Backend for Backend { if result.is_err() { self.stop_after_error(); } + // drm_sched frees the job as soon as this returns while the job's + // completion callback is still on its scheduler fence, so the fence + // must be signalled by now even when the queue could not settle the + // packet (a busy or failed runtime, a settle racing another queue's + // timeout). A completion its physical queue still owns keeps the VM + // pinned until the queue retires it; that later completion is a no-op. + if !packet.completion.signalled() { + packet.completion.fail_while_owned(ECANCELED); + } + // Another thread may have claimed the completion and not yet signalled. + let fence = packet.completion.scheduler_fence(); + // SAFETY: the retained fence stays alive across this bounded wait, + // which holds no device lock. + unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(1000) as _, + ) + }; } fn long_job_beside(&self, _packet: &Arc) -> bool { From 763b7c47ff5bdbdfed4096d67844090cc991a3b7 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 11:01:23 -0400 Subject: [PATCH 073/171] drm/asahi: g17: describe the M5 Max (J716c) HW data The M5 Max shares the T6050 SoC and the gpu,t6050 binding with the M5 Pro, but its GPU has four clusters of ten cores instead of two, all eight AMCCs enabled instead of four, and its own voltage tables. Move the board-specific HW data behind M5Board { J714, J716c } and pass the board from the Config to the HW data and initdata builders: - Performance states carry a core and an SRAM voltage per table. HW perf lane n carries table n and lanes past the board's table count carry table 0, which reproduces the J714 data exactly. J716c has four tables (0 == 3, 1 == 2); they, the auxiliary-domain table and the relative-frequency curve come from its ADT. - Cluster count 2 / 4, core count 20 / 40 and core type 0x18 / 0x19 (the core type follows the ID variant). - The minimum parameter-buffer size: 2 MiB / 3 MiB. - The register map. J716c maps all eight AMCC windows in slot 3 (MCC reg[14..21]), and four instances, one per cluster, in slots 20 and 24, where J714 has two at the same stride. Slot 44 is a fixed pair on both. Gpu::new() checks the board's cluster and core counts against the GPU's. Only J714 is probed; the following commits complete the J716c data and the last one probes it. The J714 data is unchanged by construction; not retested on J714. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: adapted to integration's g17::Config (the board check in Gpu::new, the board field of the T6040 and T6041 configurations) and InitData::build_m5; the SPTM boot path's board argument removed; comments reworded for publication (code unchanged by it), and message lines that named register offsets; original fbbb15c50bee ] Signed-off-by: Jeff Queisser (cherry picked from commit c84e92cc2119d25bb42c1ef32da150107b21d139 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 239 +++++++++++++++----- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 8 +- drivers/gpu/drm/asahi/g17/mod.rs | 14 +- drivers/gpu/drm/asahi/hw/t6040.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 1 + drivers/gpu/drm/asahi/hw/t6050.rs | 2 + drivers/gpu/drm/asahi/hw/t8140.rs | 1 + 7 files changed, 207 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 1ad2413151823a..aaf7dfb3b6e71d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1480,31 +1480,42 @@ const J714_AUX_PERF_STATES: [(u32, u32, u32); 14] = [ (1470, 900, 900), (1578, 960, 960), (1620, 990, 990), ]; +// J716c ADT afr-perf-states (identical to cs-perf-states). SRAM is +// max(775, core), the rule the J714 column follows. +const J716C_AUX_PERF_STATES: [(u32, u32, u32); 14] = [ + (24, 125, 775), (338, 590, 775), (486, 625, 775), (636, 640, 775), + (796, 690, 775), (888, 700, 775), (988, 740, 775), (1084, 765, 775), + (1182, 780, 780), (1278, 830, 830), (1374, 875, 875), + (1470, 910, 910), (1578, 975, 975), (1620, 1020, 1020), +]; + // Evaluate the frequency-only curve at compile time; no kernel FPU use. // Conversion uses Hz before the single-precision multiply and divide. -const fn j714_relative_frequency() -> [u32; 16] { +const fn relative_frequency(states: &[M5PerfState; 14]) -> [u32; 16] { let mut values = [0; 16]; - let base = J714_GPU_PERF_STATES[1].0 as u64 * 1_000_000; - let maximum = J714_GPU_PERF_STATES[13].0 as u64 * 1_000_000; + let base = states[1].0 as u64 * 1_000_000; + let maximum = states[13].0 as u64 * 1_000_000; let denominator = (maximum - base) as f32; let mut state = 2; while state < 14 { - let frequency = J714_GPU_PERF_STATES[state].0 as u64 * 1_000_000; + let frequency = states[state].0 as u64 * 1_000_000; values[state] = (((frequency - base) as f32 * 100.0) / denominator).to_bits(); state += 1; } values } -const J714_RELATIVE_FREQUENCY: [u32; 16] = j714_relative_frequency(); +const J714_RELATIVE_FREQUENCY: [u32; 16] = relative_frequency(&J714_GPU_PERF_STATES); + +const J716C_RELATIVE_FREQUENCY: [u32; 16] = relative_frequency(&J716C_GPU_PERF_STATES); -fn encode_m5_aux_perf_states(out: &mut [u8]) { +fn encode_m5_aux_perf_states(board: M5Board, out: &mut [u8]) { let table = &mut out[M5_HW_AUX_PERF_STATE.. M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; table.fill(0); put_u32(table, 0, 13); put_u32(table, 4, 14); - for (state, &(frequency, core, sram)) in J714_AUX_PERF_STATES.iter().enumerate() { + for (state, &(frequency, core, sram)) in board.aux_perf_states().iter().enumerate() { put_u32(table, 8 + state * 4, frequency); put_u32(table, 0x48 + state * 8, core); put_u32(table, 0xc8 + state * 8, sram); @@ -1693,9 +1704,85 @@ pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; -pub(crate) const J714_CORE_REVISION: (u32, u32) = (0x18, 4); +/// Board whose HW data the M5 GPU is started with. Both are T6050. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5Board { + /// M5 Pro: 2 clusters of 10 cores. + J714, + /// M5 Max: 4 clusters of 10 cores. + J716c, +} + +impl M5Board { + pub(crate) const fn perf_states(self) -> &'static [M5PerfState; 14] { + match self { + Self::J714 => &J714_GPU_PERF_STATES, + Self::J716c => &J716C_GPU_PERF_STATES, + } + } + + /// Voltage tables in [`M5Board::perf_states`] (ADT perf-state-table-count). + const fn perf_tables(self) -> usize { + match self { + Self::J714 => 2, + Self::J716c => 4, + } + } + + const fn aux_perf_states(self) -> &'static [(u32, u32, u32); 14] { + match self { + Self::J714 => &J714_AUX_PERF_STATES, + Self::J716c => &J716C_AUX_PERF_STATES, + } + } -pub(crate) const J714_GPU_CORES: u32 = 20; + const fn relative_frequency(self) -> &'static [u32; 16] { + match self { + Self::J714 => &J714_RELATIVE_FREQUENCY, + Self::J716c => &J716C_RELATIVE_FREQUENCY, + } + } + + /// The core type, which follows the ID variant, and the firmware revision + /// the HW data names. + const fn core_revision(self) -> (u32, u32) { + match self { + Self::J714 => (0x18, 4), + Self::J716c => (0x19, 4), + } + } + + /// GPU clusters of the device. + pub(crate) const fn clusters(self) -> u32 { + match self { + Self::J714 => 2, + Self::J716c => 4, + } + } + + /// All cores of the device. + pub(crate) const fn gpu_cores(self) -> u32 { + match self { + Self::J714 => 20, + Self::J716c => 40, + } + } + + /// The minimum parameter-buffer size. + pub(crate) const fn min_parameter_buffer(self) -> u32 { + match self { + Self::J714 => 0x0020_0000, + Self::J716c => 0x0030_0000, + } + } + + pub(crate) const fn register_window_specs(self) -> &'static [M5RegisterWindowSpec; 29] { + match self { + Self::J714 => &J714_REGISTER_WINDOW_SPECS, + Self::J716c => &J716C_REGISTER_WINDOW_SPECS, + } + } +} /// Populate the enabled-USC count from the device's sampled core masks. /// This count covers the whole device, not one cluster. @@ -1725,20 +1812,44 @@ const J714_POWER_POLICY_MINIMUM_ENTRIES: usize = 16 * 32; pub(crate) const J714_GPU_NUM_PERF_STATES: u32 = 13; -// Frequency in MHz, two core-voltage tables, two SRAM-voltage tables (mV). -pub(crate) const J714_GPU_PERF_STATES: [(u32, u32, u32, u32, u32); 14] = [ - (0, 125, 125, 775, 775), (338, 605, 605, 775, 775), - (486, 620, 635, 775, 775), (636, 655, 655, 775, 775), - (796, 705, 705, 775, 775), (888, 715, 715, 775, 775), - (988, 745, 745, 775, 775), (1084, 770, 770, 775, 775), - (1182, 800, 800, 800, 800), (1278, 840, 840, 840, 840), - (1374, 880, 880, 880, 880), (1470, 925, 925, 925, 925), - (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), +/// A main-domain performance state: MHz, then core and SRAM mV for each +/// voltage table. HW perf lane n carries table n; lanes past the board's table +/// count carry table 0, as J714 does for lanes 0 and 2-15. +pub(crate) type M5PerfState = (u32, [u32; 4], [u32; 4]); + +// J714: two voltage tables. +pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = [ + (0, [125, 125, 0, 0], [775, 775, 0, 0]), (338, [605, 605, 0, 0], [775, 775, 0, 0]), + (486, [620, 635, 0, 0], [775, 775, 0, 0]), (636, [655, 655, 0, 0], [775, 775, 0, 0]), + (796, [705, 705, 0, 0], [775, 775, 0, 0]), (888, [715, 715, 0, 0], [775, 775, 0, 0]), + (988, [745, 745, 0, 0], [775, 775, 0, 0]), (1084, [770, 770, 0, 0], [775, 775, 0, 0]), + (1182, [800, 800, 0, 0], [800, 800, 0, 0]), (1278, [840, 840, 0, 0], [840, 840, 0, 0]), + (1374, [880, 880, 0, 0], [880, 880, 0, 0]), (1470, [925, 925, 0, 0], [925, 925, 0, 0]), + (1578, [1000, 1000, 0, 0], [1000, 1000, 0, 0]), + (1620, [1025, 1040, 0, 0], [1025, 1040, 0, 0]), +]; + +// J716c ADT perf-states / perf-states-sram: four tables (0 == 3, 1 == 2). +pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = [ + (0, [125, 125, 125, 125], [775, 775, 775, 775]), + (338, [635, 635, 635, 635], [775, 775, 775, 775]), + (486, [655, 640, 640, 655], [775, 775, 775, 775]), + (636, [680, 665, 665, 680], [775, 775, 775, 775]), + (796, [730, 715, 715, 730], [775, 775, 775, 775]), + (888, [740, 725, 725, 740], [775, 775, 775, 775]), + (988, [775, 760, 760, 775], [775, 775, 775, 775]), + (1084, [785, 785, 785, 785], [785, 785, 785, 785]), + (1182, [805, 810, 810, 805], [805, 810, 810, 805]), + (1278, [840, 840, 840, 840], [840, 840, 840, 840]), + (1374, [875, 865, 865, 875], [875, 865, 865, 875]), + (1470, [920, 905, 905, 920], [920, 905, 905, 920]), + (1578, [995, 980, 980, 995], [995, 980, 980, 995]), + (1620, [1015, 1015, 1015, 1015], [1015, 1015, 1015, 1015]), ]; pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; -pub(crate) struct J714RegisterWindowSpec { +pub(crate) struct M5RegisterWindowSpec { pub(crate) slot: usize, pub(crate) instances: &'static [u64], @@ -1749,13 +1860,13 @@ pub(crate) struct J714RegisterWindowSpec { } const fn jw(slot: usize, instances: &'static [u64], size: u32, unk_18: u64, flag: u32) - -> J714RegisterWindowSpec { - J714RegisterWindowSpec { slot, instances, size, unk_18, flag } + -> M5RegisterWindowSpec { + M5RegisterWindowSpec { slot, instances, size, unk_18, flag } } const G: u64 = J714_GPU_BASE; -pub(crate) const J714_REGISTER_WINDOW_SPECS: [J714RegisterWindowSpec; 29] = [ +pub(crate) const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ jw(0, &[0x2_8041_4000], 0x4000, 0, 2), jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000], 0x18000, 0, 2), jw(4, &[0x2_8046_8000], 0x4000, 0, 0), @@ -1787,11 +1898,28 @@ pub(crate) const J714_REGISTER_WINDOW_SPECS: [J714RegisterWindowSpec; 29] = [ jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), ]; -pub(crate) const J714_REGISTER_WINDOW_58: J714RegisterWindowSpec = +/// J714 with all eight AMCCs in slot 3 (J716c amcc-enable-mask 0xff, MCC +/// reg[14..21] + 0x104000) and four instances, one per cluster, in slots 20 and +/// 24, where J714 has two at the same stride. Slot 44 is a fixed pair on every M5 +/// part. +pub(crate) const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { + let mut specs = J714_REGISTER_WINDOW_SPECS; + assert!(specs[1].slot == 3 && specs[11].slot == 20 && specs[14].slot == 24); + specs[1] = jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], + 0x18000, 0, 2); + specs[11] = jw(20, &[0x23_0300_0000, 0x23_0320_0000, 0x23_0340_0000, 0x23_0360_0000], + 0x40000, 0, 2); + specs[14] = jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000], + 0x10000, 0, 2); + specs +}; + +pub(crate) const M5_REGISTER_WINDOW_58: M5RegisterWindowSpec = jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); #[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct J714RegisterWindow { +pub(crate) struct M5RegisterWindow { pub(crate) slot: usize, pub(crate) device_va: u64, pub(crate) count: usize, @@ -1800,22 +1928,24 @@ pub(crate) struct J714RegisterWindow { pub(crate) flag: u32, } -pub(crate) const J714_REGISTER_WINDOW_COUNT: usize = J714_REGISTER_WINDOW_SPECS.len() + 1; +pub(crate) const M5_REGISTER_WINDOW_COUNT: usize = J714_REGISTER_WINDOW_SPECS.len() + 1; -pub(crate) fn j714_register_window_spec(index: usize) -> &'static J714RegisterWindowSpec { - J714_REGISTER_WINDOW_SPECS.get(index).unwrap_or(&J714_REGISTER_WINDOW_58) +pub(crate) fn m5_register_window_spec(board: M5Board, index: usize) + -> &'static M5RegisterWindowSpec +{ + board.register_window_specs().get(index).unwrap_or(&M5_REGISTER_WINDOW_58) } -pub(crate) fn j714_register_windows() -> [J714RegisterWindow; J714_REGISTER_WINDOW_COUNT] { +pub(crate) fn m5_register_windows(board: M5Board) -> [M5RegisterWindow; M5_REGISTER_WINDOW_COUNT] { let mask = UAT_PAGE_SIZE as u64 - 1; - let mut out = [J714RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; - J714_REGISTER_WINDOW_COUNT]; + let mut out = [M5RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; + M5_REGISTER_WINDOW_COUNT]; let mut next: u64 = 0; for (index, window) in out.iter_mut().enumerate() { - let spec = j714_register_window_spec(index); + let spec = m5_register_window_spec(board, index); let va = next + (spec.instances[0] & mask); let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); - *window = J714RegisterWindow { + *window = M5RegisterWindow { slot: spec.slot, device_va: HW_REGISTER_APERTURE.start + va, count: spec.instances.len(), @@ -1829,8 +1959,9 @@ pub(crate) fn j714_register_windows() -> [J714RegisterWindow; J714_REGISTER_WIND out } -pub(crate) fn encode_j714_register_map( - windows: &[J714RegisterWindow], +pub(crate) fn encode_m5_register_map( + board: M5Board, + windows: &[M5RegisterWindow], out: &mut [u8], ) -> Result<(), InitdataError> { if out.len() != COMMON26_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { @@ -1838,7 +1969,7 @@ pub(crate) fn encode_j714_register_map( } out.fill(0); for (index, window) in windows.iter().enumerate() { - let spec = j714_register_window_spec(index); + let spec = m5_register_window_spec(board, index); if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS || !HW_REGISTER_APERTURE.contains(&window.device_va) { @@ -1856,9 +1987,10 @@ pub(crate) fn encode_j714_register_map( } pub(crate) fn encode_m5_hw_data( + board: M5Board, aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], qos_high: u64, - windows: &[J714RegisterWindow], + windows: &[M5RegisterWindow], out: &mut [u8], ) -> Result<(), InitdataError> { if out.len() != M5_HW_DATA_SIZE { @@ -1880,7 +2012,8 @@ pub(crate) fn encode_m5_hw_data( let g18 = &mut out[..COMMON26_HW_DATA_SIZE]; g18.fill(0); write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, COMMON26_HW_DATA_BANDS, g18)?; - encode_j714_register_map(windows, &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; + encode_m5_register_map(board, windows, + &mut g18[HW_REGISTER_MAP_OFFSET..COMMON26_HW_REGISTER_MAP_END])?; put_u32(g18, COMMON26_HW_CHIP_ID_OFFSET, 0x6050); put_u32(g18, COMMON26_HW_PERF_STATE_COUNT, J714_GPU_NUM_PERF_STATES); put_u32(g18, COMMON26_HW_FIXED_WORD_OFFSET, COMMON26_HW_FIXED_WORD_VALUE); @@ -1889,31 +2022,33 @@ pub(crate) fn encode_m5_hw_data( // J714's 26A428 HW data carries 1 here. The firmware copies this // header at initialization. put_u32(g18, 0x0fb8, 1); - put_u32(g18, 0x10d4, J714_CORE_REVISION.0); - put_u32(g18, 0x10d8, J714_CORE_REVISION.1); - put_u32(g18, 0x10dc, J714_GPU_CORES); + let (core_type, revision) = board.core_revision(); + put_u32(g18, 0x10d4, core_type); + put_u32(g18, 0x10d8, revision); + put_u32(g18, 0x10dc, board.gpu_cores()); put_u32(g18, 0x283a, 1); put_u32(g18, 0x283e, 1); - for (state, &(frequency, core0, core1, sram0, sram1)) in J714_GPU_PERF_STATES.iter().enumerate() { + let states = board.perf_states(); + for (state, &(frequency, core, sram)) in states.iter().enumerate() { put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); - let relative = frequency * 100 / J714_GPU_PERF_STATES[13].0; + let relative = frequency * 100 / states[13].0; put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); for lane in 0..COMMON26_HW_PERF_LANES { let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; - put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, if lane == 1 { core1 } else { core0 }); - put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, if lane == 1 { sram1 } else { sram0 }); + let table = if lane < board.perf_tables() { lane } else { 0 }; + put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, core[table]); + put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, sram[table]); } } + let relative = board.relative_frequency(); for index in 0..COMMON26_HW_PERF_ROWS { put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); - put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, - J714_RELATIVE_FREQUENCY[index]); - put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, - J714_RELATIVE_FREQUENCY[index]); + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, relative[index]); + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, relative[index]); } - encode_m5_aux_perf_states(g18); + encode_m5_aux_perf_states(board, g18); let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; g18[table..table + completion.len()].copy_from_slice(&completion); put_u64(g18, COMMON26_QOS_POINTER_OFFSET, qos_high); @@ -1925,10 +2060,8 @@ pub(crate) fn encode_m5_hw_data( put_u32(out, 0x2660, 0); put_u32(out, 0x2728, 0); put_u32(out, 0x272c, 1); - // J714's 26A428 HW data carries this tiling-allocator word. The - // firmware copies it at initialization and the tiling allocator shifts - // it by 19. The inherited J813 word selected three groups instead of four. - put_u32(out, 0x26b0, 0x0020_0000); + // The inherited J813 minimum parameter-buffer size is replaced per board. + put_u32(out, 0x26b0, board.min_parameter_buffer()); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 6103ea35f8e65a..4643519b614b58 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -66,7 +66,7 @@ impl InitData { } pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, - id: &crate::regs::G17Id) -> Result { + id: &crate::regs::G17Id, board: m::M5Board) -> Result { use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, PROT_GPU_SHARED_RW as GPU}; self.m5 = true; @@ -119,9 +119,9 @@ impl InitData { m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; m::encode_m5_private_cluster(PRIVATE, FWCTL, &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; - let windows = m::j714_register_windows(); + let windows = m::m5_register_windows(board); for (index, window) in windows.iter().enumerate() { - let spec = m::j714_register_window_spec(index); + let spec = m::m5_register_window_spec(board, index); let stride = (u64::from(window.size) + mmu::UAT_PGMSK as u64) & !(mmu::UAT_PGMSK as u64); for (instance, physical) in spec.instances.iter().copied().enumerate() { let offset = physical & mmu::UAT_PGMSK as u64; @@ -141,7 +141,7 @@ impl InitData { let views = m::PrimaryRegionViews { sentinel_high: BASE + 0xc07e_0000, alias_high: [PB, UMA], }; - m::encode_m5_hw_data(completions, QOS, &windows, + m::encode_m5_hw_data(board, completions, QOS, &windows, &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; m::set_m5_kick_shared( &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index c90bf8c8b13409..e00e8a142c00e1 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -215,6 +215,8 @@ pub(crate) struct Config { /// Parameter buffer page metrics shared with the firmware; `None` where the firmware graph /// places its own (M5). pub(crate) pm_metrics: Option, + /// M5 board whose HW data the GPU is started with. + pub(crate) m5_board: Option, /// Register windows the firmware maps through the hardware data. pub(crate) windows: &'static [fw::initdata::RegisterWindow], /// Register-map slots that are reserved but have no window. @@ -1387,14 +1389,22 @@ impl Gpu { ); return Err(ENODEV); } + let m5 = cfg.profile.is_m5(); + if m5 { + let board = cfg.m5_board.ok_or(EINVAL)?; + if id.num_clusters != board.clusters() || id.num_clusters * id.num_cores != board.gpu_cores() { + dev_err!(dev.as_ref(), "M5 {:?} expects {} clusters and {} cores, the GPU reports {} x {}\n", + board, board.clusters(), board.gpu_cores(), id.num_clusters, id.num_cores); + return Err(ENODEV); + } + } let geometry = mmu::UatGeometry { ias: cfg.uat_ias, oas: cfg.uat_oas, }; - let m5 = cfg.profile.is_m5(); let max_frequency_khz = if m5 { - fw::m5::initdata::J714_GPU_PERF_STATES[13].0 * 1000 + cfg.m5_board.ok_or(EINVAL)?.perf_states()[13].0 * 1000 } else { Self::max_frequency_khz(dev)? }; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 29aff6ef4b5f54..c3486d2888f0b8 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -38,6 +38,7 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), + m5_board: None, windows: ®ISTER_WINDOWS, reserved_slots: &[], perf: Some(&PERF), diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 7938ee0248c4bd..298aaf3f2d6014 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -29,6 +29,7 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), + m5_board: None, windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_SLOTS, perf: Some(&PERF), diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index dde03d20fcdac6..807e497be23236 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -3,6 +3,7 @@ //! //! The M5 firmware builds its HW data, performance tables and region C in its own graph //! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. +use crate::g17::fw::m5::initdata::M5Board; use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; @@ -29,6 +30,7 @@ pub(crate) const CONFIG: Config = Config { ]), profile: FirmwareProfile::M5NoBareMetalTables, pm_metrics: None, + m5_board: Some(M5Board::J714), windows: &[], reserved_slots: &[], perf: None, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 6c2c2a173630e0..a369ff9a3dd0d7 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -44,6 +44,7 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), + m5_board: None, recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, windows: ®ISTER_WINDOWS, From d6c1f88d61f8fa5a66f79af667533a72ca6a32d1 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 11:01:23 -0400 Subject: [PATCH 074/171] drm/asahi: g17: J716c: encode its own AON PTD records The three AON PTD records (RT0g, RT1g, RT0r) carry device-specific table indices. Encode them per board; the M5 Max rows are 0x58, 0xa3 and 0xb8. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora; original c280d3b1777e ] Signed-off-by: Jeff Queisser (cherry picked from commit 9972221cfc863e8915163cb445567c3c84e72a42 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 18 ++++++++++++++---- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index aaf7dfb3b6e71d..0169f10ead18fd 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1578,18 +1578,28 @@ pub(crate) fn encode_common26_region_c(out: &mut [u8]) -> Result<(), InitdataErr Ok(()) } -pub(crate) fn encode_common26_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { +pub(crate) fn encode_common26_aon_ptd(board: M5Board, out: &mut [u8]) -> Result<(), InitdataError> { if out.len() != M5_AON_PTD_SIZE { return Err(InitdataError::BufferSize); } - // J714 descriptors carry device-specific AON table indices. - const RECORDS: [[u32; 3]; 3] = [ + // The ADT aon-ptd table rows tagged RT0g, RT1g and RT0r; their indices are + // device-specific. + const J714_RECORDS: [[u32; 3]; 3] = [ [0x6730_5452, 0x0079_0001, 0x0000_1401], [0x6731_5452, 0x00bc_0001, 0x0000_1401], [0x7230_5452, 0x00ce_0001, 0x0000_1401], ]; + const J716C_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0058_0001, 0x0000_1401], + [0x6731_5452, 0x00a3_0001, 0x0000_1401], + [0x7230_5452, 0x00b8_0001, 0x0000_1401], + ]; + let records = match board { + M5Board::J714 => &J714_RECORDS, + M5Board::J716c => &J716C_RECORDS, + }; out.fill(0); - for (index, record) in RECORDS.iter().enumerate() { + for (index, record) in records.iter().enumerate() { for (word, value) in record.iter().enumerate() { put_u32(out, index * 12 + word * 4, *value); } diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 4643519b614b58..4c35e0e73f6bc8 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -114,7 +114,7 @@ impl InitData { self.completion_rings[index] = object; self.alias(object, uat.kernel_lower_vm(), pair.low, GPU)?; } - m::encode_common26_aon_ptd(&mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) + m::encode_common26_aon_ptd(board, &mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) .map_err(|_| EINVAL)?; m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; m::encode_m5_private_cluster(PRIVATE, FWCTL, From acad0f4cafdaa37405ac220253fe9cc7e0baea63 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 11:01:24 -0400 Subject: [PATCH 075/171] drm/asahi: g17: probe the M5 Max (J716c) Both boards bind gpu,t6050 and probe used the J714 configuration unconditionally, so a Max was rejected at the identity check (ID_VERSION variant 4 instead of 3). Add the J716c Config (variant 4, GpuVariant::C, M5Board::J716c) with its crash-log buffers at the ADT gfx-data-shared-ro-base and gfx1-data-shared-ro-base, and select the configuration from the ID registers after powering AFR, with the same Resources the GPU then boots with. probe_g17() now takes those Resources from its caller; on the other G17 path they are created before the DMA mask is set instead of after. On J714 the only change is that AFR is powered and the ID read once more during probe; both are idempotent. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: adapted to integration's probe_m5, probe_g17 and Config; the SPTM profile and the SPTM probe with its AFR power-up removed, so J716c, like J714, is refused at probe until it has bare-metal GPU tables; original 11a9c30bacae ] Signed-off-by: Jeff Queisser [Eduard Moldovan: integration/aurora's M3 T8122 probe arm is kept; the ProbeConfig::M5 arm from a2797fa0626e (DjDeveloperr) is restored here] (cherry picked from commit b2c862db522a7b8e0b52407ea1b9e1d5aa3a4cd9 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 17 +++++-- drivers/gpu/drm/asahi/hw/t6050.rs | 73 +++++++++++++++++++++++++++---- 2 files changed, 78 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index fb0058fae4007e..33a198194e5164 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -266,7 +266,10 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, - ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + ProbeConfig::G17(cfg) => { + return AsahiPlatformDriver::probe_g17(pdev, regs::G17Resources::new(pdev)?, cfg) + } + ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev), #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), @@ -436,16 +439,23 @@ impl AsahiPlatformDriver { fn probe_m5<'bound>( pdev: &'bound platform::Device>, ) -> Result> { - let cfg = &hw::t6050::CONFIG; + // Both boards bind gpu,t6050. Let the ID registers choose the configuration. + let res = regs::G17Resources::new(pdev)?; + let id = res.get_g17_id()?; + let cfg = hw::t6050::config(&id).ok_or_else(|| { + dev_err!(pdev.as_ref(), "No M5 configuration for GPU variant {:#x}\n", id.variant); + ENODEV + })?; if let g17::FirmwareProfile::M5NoBareMetalTables = cfg.profile { dev_err!(pdev.as_ref(), "M5 Pro/Max bare-metal GPU tables not yet available\n"); return Err(ENODEV); } - Self::probe_g17(pdev, cfg) + Self::probe_g17(pdev, res, cfg) } fn probe_g17<'bound>( pdev: &'bound platform::Device>, + res: regs::G17Resources, cfg: &'static g17::Config, ) -> Result> { // The GPU memory limits and RSS attribution apply only to G17 for now. @@ -453,7 +463,6 @@ impl AsahiPlatformDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; - let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; // SAFETY: Every adapter calls close_binding in unbind before devres; // probe errors drop this local owner while the bound callback is live. diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 807e497be23236..a3cbfb198d6d26 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -1,11 +1,20 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! M5 Pro GPU firmware and address-space configuration. +//! M5 Pro and M5 Max GPU address-space configuration. //! //! The M5 firmware builds its HW data, performance tables and region C in its own graph //! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. use crate::g17::fw::m5::initdata::M5Board; use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; +use crate::regs::G17Id; + +// The coprocessors request a crash log buffer at these addresses (primary first) when they start and hand +// its contents back when they crash; outside the canonical range of dynamic objects. They are the +// ADT sgx gfx-data-shared-ro-base and gfx1-data-shared-ro-base (each 0x8000), so they differ per board. +const J714_CRASH_BUFFERS: [FixedBuffer; 2] = [ + FixedBuffer { va: 0x100_025b_4000, size: 0x8000 }, + FixedBuffer { va: 0x100_0267_0000, size: 0x8000 }, +]; pub(crate) const CONFIG: Config = Config { chip_id: 0x6050, @@ -21,13 +30,7 @@ pub(crate) const CONFIG: Config = Config { uat_oas: 42, // Unknown until the board has bare-metal page tables. secondary_root_offset: None, - // The coprocessors request a crash log buffer at these addresses (primary first) when they - // start and hand its contents back when they crash; outside the canonical range of dynamic - // objects. - crash_buffers: CrashBuffers::Fixed([ - FixedBuffer { va: 0x100_025b_4000, size: 0x8000 }, - FixedBuffer { va: 0x100_0267_0000, size: 0x8000 }, - ]), + crash_buffers: CrashBuffers::Fixed(J714_CRASH_BUFFERS), profile: FirmwareProfile::M5NoBareMetalTables, pm_metrics: None, m5_board: Some(M5Board::J714), @@ -53,3 +56,57 @@ pub(crate) const CONFIG: Config = Config { compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, }, }; + +// J716c ADT gfx-data-shared-ro-base and gfx1-data-shared-ro-base. +const J716C_CRASH_BUFFERS: [FixedBuffer; 2] = [ + FixedBuffer { va: 0x100_0283_c000, size: 0x8000 }, + FixedBuffer { va: 0x100_028f_8000, size: 0x8000 }, +]; + +/// M5 Max: ID variant 4, GPU variant C, HAL generation 300 like the M5 Pro. +pub(crate) const CONFIG_MAX: Config = Config { + chip_id: 0x6050, + identity: Identity { + family: 0xb, + variant: 4, + num_dies: 1, + gpu_variant: GpuVariant::C, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Unknown until the board has bare-metal page tables. + secondary_root_offset: None, + crash_buffers: CrashBuffers::Fixed(J716C_CRASH_BUFFERS), + profile: FirmwareProfile::M5NoBareMetalTables, + pm_metrics: None, + m5_board: Some(M5Board::J716c), + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: None, + chip: None, + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + // No loaded image is admitted: the board has no bare-metal page tables yet. + firmware: crate::g17::fw::variant::Firmware::Identified(&[]), + // Four clusters of ten cores. + core_layout: CoreLayout { + cores: Some((4, 40)), + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::FOUR_BY_TEN, + deflake: crate::g17::fw::render::Deflake::FOUR_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// Both boards bind `gpu,t6050`; the ID registers select the configuration. +pub(crate) fn config(id: &G17Id) -> Option<&'static Config> { + [&CONFIG, &CONFIG_MAX].into_iter().find(|cfg| id.family == cfg.identity.family + && id.variant == cfg.identity.variant && id.num_dies == cfg.identity.num_dies) +} From e4aa838f1a8fa3bab1806b6570f25ea97d66720d Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 13:19:23 -0400 Subject: [PATCH 076/171] drm/asahi: g17: J716c: use its 8 MiB render blocks The M5 Max uses 8 MiB render blocks and an 8 GiB private-memory address inventory, twice the M5 Pro's 4 MiB and 4 GiB. With the M5 Pro geometry, compute work on the Max ends in firmware recoveries within seconds. Select the render pool from the board at probe, before any pool is built, and size render and compute pool geometry and budgets with it. The M5 Pro values are unchanged. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: adapted to integration's compute_m5 and PoolGeometry; comments reworded for publication (code unchanged by it), and one word of the message; original 8d4eed54d540 ] Signed-off-by: Jeff Queisser (cherry picked from commit 98e0ea6955a15d4090202ffbc1876d7dce0eece9 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/freelist.rs | 3 +- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 8 +++++ drivers/gpu/drm/asahi/g17/fw/m5/pool.rs | 34 ++++++++++++++----- .../gpu/drm/asahi/g17/fw/m5/pool_budget.rs | 10 +++--- drivers/gpu/drm/asahi/g17/mod.rs | 1 + 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 32078e0daadbef..897407af5e609c 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -371,7 +371,8 @@ impl Pages { Self::new(alloc, blocks, RunLayout::Compute) } /// Fresh zeroed M5 compute backing, built without the device mutex. M5 keeps - /// its directory, block list and initial 32 MiB in one retained allocation. + /// its directory, block list and initial eight render blocks in one retained + /// allocation. fn compute_m5(alloc: &Allocator<'_>, required: Option) -> Result { use super::fw::m5::pool; let geometry = match required { Some(geometry) => geometry, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 0169f10ead18fd..373d5585e7c0de 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1778,6 +1778,14 @@ impl M5Board { } } + /// Render block size and private-memory address inventory. + pub(crate) const fn render_pool(self) -> (usize, u64) { + match self { + Self::J714 => (0x40_0000, 1 << 32), + Self::J716c => (0x80_0000, 1 << 33), + } + } + /// The minimum parameter-buffer size. pub(crate) const fn min_parameter_buffer(self) -> u32 { match self { diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs index abbde813cb5990..5fcd9ef4995587 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -2,11 +2,29 @@ //! M5 private-memory allocation encodings. #![allow(dead_code)] +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use super::initdata::M5Board; + pub(crate) const BLOCK_BYTES: usize = 0x20_0000; -pub(crate) const G17S_RENDER_BLOCK_BYTES: usize = 0x40_0000; -/// G17S private-memory address inventory. This sizes the directory, not the -/// physical allocation; each pool allocates only its requested initial blocks. -const G17S_POOL_BYTES: u64 = 1 << 32; +// Render block size and private-memory address inventory of the M5 +// part, selected at probe (`M5Board::render_pool()`). The inventory sizes the +// directory, not the physical allocation; each pool allocates only its +// requested initial blocks. +static RENDER_BLOCK_BYTES: AtomicUsize = AtomicUsize::new(M5Board::J714.render_pool().0); +static POOL_BYTES: AtomicU64 = AtomicU64::new(M5Board::J714.render_pool().1); + +/// Selects the board's render pool before any pool is built. +pub(crate) fn select_render_pool(board: M5Board) { + let (block, pool) = board.render_pool(); + RENDER_BLOCK_BYTES.store(block, Ordering::Relaxed); + POOL_BYTES.store(pool, Ordering::Relaxed); +} + +pub(crate) fn render_block_bytes() -> usize { + RENDER_BLOCK_BYTES.load(Ordering::Relaxed) +} + const RING_MAX_PAGES: u64 = 0x3ff800; pub(crate) const OPERAND_BYTES: usize = 0x2_0000; pub(crate) const STATE_BYTES: usize = 0x100; @@ -108,8 +126,8 @@ fn round_page(bytes: usize) -> Result { impl PoolGeometry { pub(crate) fn g17s(blocks: usize) -> Result { - let bytes = G17S_POOL_BYTES.min(RING_MAX_PAGES << 12) - & !(G17S_RENDER_BLOCK_BYTES as u64 - 1); + let bytes = POOL_BYTES.load(Ordering::Relaxed).min(RING_MAX_PAGES << 12) + & !(render_block_bytes() as u64 - 1); Self::g17s_render(blocks, (bytes >> 12) as u32) } pub(crate) fn new(blocks: usize, directory_entries: u32) -> Result { @@ -121,11 +139,11 @@ impl PoolGeometry { } pub(crate) fn block_bytes(self) -> usize { - if self.g17s_render_blocks { G17S_RENDER_BLOCK_BYTES } else { BLOCK_BYTES } + if self.g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES } } fn with_kind(blocks: usize, directory_entries: u32, g17s_render_blocks: bool) -> Result { - let block_bytes = if g17s_render_blocks { G17S_RENDER_BLOCK_BYTES } else { BLOCK_BYTES }; + let block_bytes = if g17s_render_blocks { render_block_bytes() } else { BLOCK_BYTES }; let pages = blocks.checked_mul(block_bytes / 0x1000) .and_then(|n| u32::try_from(n).ok()).ok_or(PoolError::Geometry)?; if blocks == 0 || pages > directory_entries { return Err(PoolError::Geometry); } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs index 80339f98a9026d..3e5584422364cf 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs @@ -2,7 +2,7 @@ //! G17S shared render private-memory demand. Counts describe hardware and //! simultaneous work; byte requirements come from the submitted programs. -use super::pool::{PoolError, PoolGeometry, G17S_RENDER_BLOCK_BYTES}; +use super::pool::{render_block_bytes, PoolError, PoolGeometry}; #[derive(Clone, Copy)] pub(crate) struct RenderBudget { @@ -44,8 +44,8 @@ fn mul(a: u64, b: u64) -> Result { a.checked_mul(b).ok_or(PoolError::Geometry) } fn aligned(bytes: u64) -> Result { - Ok(add(bytes, G17S_RENDER_BLOCK_BYTES as u64 - 1)? - & !(G17S_RENDER_BLOCK_BYTES as u64 - 1)) + Ok(add(bytes, render_block_bytes() as u64 - 1)? + & !(render_block_bytes() as u64 - 1)) } impl RenderBudget { @@ -143,9 +143,9 @@ impl Sizes { /// physical blocks. The hardware directory limit is checked by geometry. pub(crate) fn geometry(self) -> Result { if self.minimum == 0 || self.minimum > self.ideal { return Err(PoolError::Geometry); } - let blocks = usize::try_from(self.ideal / G17S_RENDER_BLOCK_BYTES as u64) + let blocks = usize::try_from(self.ideal / render_block_bytes() as u64) .map_err(|_| PoolError::Geometry)?; - if self.ideal % G17S_RENDER_BLOCK_BYTES as u64 != 0 { return Err(PoolError::Geometry); } + if self.ideal % render_block_bytes() as u64 != 0 { return Err(PoolError::Geometry); } PoolGeometry::g17s(blocks) } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index e00e8a142c00e1..f37a33f9f699cc 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1397,6 +1397,7 @@ impl Gpu { board, board.clusters(), board.gpu_cores(), id.num_clusters, id.num_cores); return Err(ENODEV); } + fw::m5::pool::select_render_pool(board); } let geometry = mmu::UatGeometry { From cea200b58d911f6346ad12c096d14bf3a554e59e Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 11:01:23 -0400 Subject: [PATCH 077/171] drm/asahi: g17: carry the minimum parameter-buffer size in renders The last segment-control word of an M5 render is the part's minimum parameter-buffer size in 32 KiB pages. It was passed the segment count (64), which is right only on the M5 Pro. Derive it from the physical cluster count instead, for every render including single-group ones: 64 pages on the M5 Pro (unchanged) and 96 on the M5 Max. A compile-time check ties the page count to each board's HW data. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: adapted to integration's queue/render/m5_render_submission.rs; the m5_render_resources.rs hunk dropped, as integration has no such file; comments reworded for publication (code unchanged by it), and message lines that named register offsets; original d290b4a2ceab ] Signed-off-by: Jeff Queisser (cherry picked from commit 24b3a2075c185e4587a4218e6613d28829cbe72a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/buffer/target.rs | 7 +++++-- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 11 ++++++++++- drivers/gpu/drm/asahi/g17/fw/m5/target.rs | 5 +++++ .../asahi/g17/queue/render/m5_render_submission.rs | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index b1ef1a16ed00bf..8561053a336f2f 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -148,12 +148,15 @@ impl Layout { m5: None, }) } - pub(crate) fn with_m5_slices(mut self, pass: &RenderPass, groups: u32, + pub(crate) fn with_m5_slices(mut self, pass: &RenderPass, clusters: u32, groups: u32, segments: u32, parameter_bytes: usize) -> Result { let slices = crate::g17::fw::m5::target::TargetSlices::new( self.tilemap_size, self.tpc_size, pass.layers, groups, segments, parameter_bytes) .ok_or(EINVAL)?; - let control = crate::g17::fw::m5::target::cluster_control(groups, u64::from(segments)) + // The last word is the part's minimum parameter-buffer size, whatever + // the group count. + let control = crate::g17::fw::m5::target::cluster_control(groups, + crate::g17::fw::m5::target::min_parameter_buffer_pages(clusters)) .ok_or(EINVAL)?; self.m5 = Some((slices, control)); Ok(self) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 373d5585e7c0de..0bf8c85ea5ed79 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1786,7 +1786,8 @@ impl M5Board { } } - /// The minimum parameter-buffer size. + /// The minimum parameter-buffer size. Renders carry it in + /// pages too (`target::min_parameter_buffer_pages()`). pub(crate) const fn min_parameter_buffer(self) -> u32 { match self { Self::J714 => 0x0020_0000, @@ -1933,6 +1934,14 @@ pub(crate) const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { specs }; +const _: () = { + use super::target::min_parameter_buffer_pages; + assert!(M5Board::J714.min_parameter_buffer() as u64 + == min_parameter_buffer_pages(M5Board::J714.clusters()) << 15); + assert!(M5Board::J716c.min_parameter_buffer() as u64 + == min_parameter_buffer_pages(M5Board::J716c.clusters()) << 15); +}; + pub(crate) const M5_REGISTER_WINDOW_58: M5RegisterWindowSpec = jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs index 2dc50a4eaabecf..a21fb72e2b7e34 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs @@ -2,6 +2,11 @@ //! M5 render-target suballocation geometry. +/// The part's minimum parameter-buffer size in 32 KiB pages. +pub(crate) const fn min_parameter_buffer_pages(clusters: u32) -> u64 { + 32 + 16 * clusters as u64 +} + pub(crate) fn cluster_control(clusters: u32, pb_pages: u64) -> Option<[u64; 3]> { let difference = match clusters { 1 => return Some([1, 0, pb_pages]), diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs index 443b8d0d0d5b32..9f40046215d11c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -138,7 +138,7 @@ pub(crate) fn layout(command: &Validated, clusters: u32, programs: RegisterProgr // future growth, rather than just the initially resident physical blocks. let bytes = geometry.capacity.checked_mul(geometry.block_bytes).ok_or(EOVERFLOW)?; let groups = if pass.few_primitives { 1 } else { clusters }; - layout.with_m5_slices(pass, groups, 64, bytes) + layout.with_m5_slices(pass, clusters, groups, 64, bytes) } pub(crate) fn controls(pass: &RenderPass) -> Result { From 2a3f88facdc1007d7a2eb72612694c8a97e3a299 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sun, 1 Nov 2026 15:34:45 -0500 Subject: [PATCH 078/171] drm/asahi: g17: M5: reinstall a compute queue the firmware killed work on After a compute timeout, the driver asks the firmware to kill the closed context's remaining commands and, once the kill is acknowledged and every command settled, returns the queue to service with fresh backing. On M5 the next owner's first command on that queue then never runs: it retires with a zero-duration record and nothing written, and that client sees a lost device. Commands after it run normally. A killed QID stays killed in the firmware until a queue configuration with its QoS update set binds it to a scheduler page other than the one the firmware has for it. The new owner's first publication carries such a configuration, but its command reaches the still-killed queue first. On M5, reset a killed queue's install state, so the next owner's first publication sends the configuration alone and waits until the firmware consumed it before any command is published. The install record now goes at the ring's current producer: a reinstalled queue restarts at the firmware's consumer, not at zero, where the append was refused. Other GPUs retire a killed queue instead: it keeps its QID and installed graph but takes no owner again, and its backing and exited binding are still freed. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: M5 half only: the retirement of killed queues on other GPUs and the reinstall and retirement logs removed; adapted to integration's recycle_killed; original 457ef7786341 ] Signed-off-by: Jeff Queisser (cherry picked from commit 8b2c99c2509965012f64d73583b3daa38d66acff in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2d99e4a2505590..d894e402462a0c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1782,9 +1782,16 @@ impl Queue { self.needs_fresh_backing } /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) - /// and every killed command was settled: return the QID to service. The item - /// ring is reset to empty at the firmware's consumer cursor; the USC pool and - /// descriptor ring are vacated and re-armed fresh before the next owner binds. + /// and every killed command was settled. The QID stays killed in the firmware + /// until a queue configuration with its QoS update set binds it to a scheduler + /// page other than the one the firmware has for it. A new owner's first + /// publication carries such a configuration, but its command is retired + /// unexecuted on the still-killed queue, with a zero-duration record. + /// + /// On M5 the queue is installed again instead: the next owner's first + /// publication sends the configuration alone and waits until the firmware + /// consumed it (`install_m5`), so the kill is cleared before any command. The + /// USC pool and descriptor ring are vacated and re-armed fresh as before. pub(crate) fn recycle_killed(&mut self) -> Result { if self.configuration.is_some() || !self.installed || !self.exit.as_ref().is_some_and(|exit| exit.released()) @@ -1795,6 +1802,15 @@ impl Queue { return Ok(false); } self.recycle_settled()?; + if self.graph.m5 { + // Start over as a fresh install: no published configuration and kick + // count zero, so the next publication installs the queue first. Kick + // timestamps keep counting, so its stamps stay ordered. + self.m5_install = None; + self.published_config = None; + self.outer_started = 0; + self.ordinal = 0; + } Ok(true) } /// A quarantined queue whose owner closed and whose every retained command was @@ -2752,11 +2768,14 @@ impl Queue { qos_update: true, owner_pid: context.owner_pid(), context_update: false, })?; if self.configuration.is_some() { return Err(EBUSY); } - self.configuration = Some(configuration::Configuration::new(context, 1, epoch)?); + // A queue reinstalled after a kill restarts at the firmware's consumer. + let start = self.graph.cursors()?.producer; + self.configuration = Some(configuration::Configuration::new( + context, (start + 1) % ITEM_RING_ENTRIES, epoch)?); let result = (|| { let va = self.graph.record(None, CONFIG, 0xc0, config)?; let configuration = self.configuration.as_mut().ok_or(EIO)?; - let producer = self.graph.append(0, va, || configuration.mark_visible())?; + let producer = self.graph.append(start, va, || configuration.mark_visible())?; self.m5_install = Some(producer); self.installed = true; host.publish_qid(self.kick.id())?; From 42fdf077e7185870dc14c0fbe7784e91f2fb166e Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Sat, 31 Oct 2026 20:35:50 -0400 Subject: [PATCH 079/171] drm/asahi: g17: M5: keep the firmware's blame for a GPU MMU fault On M5 the recovery path reads no firmware fault reason, so a blamed compute command whose completion record shows zero duration is taken to have never executed: its blame is withdrawn and the command is replayed. But firmware writes exactly that record when it cancels a command that was running, which is what happens when that command faulted. The replay then faults again on the next kick. Read the GPU MMU fault status at every M5 recovery. When it holds a valid fault, report the recovery as a page fault, as the other G17 profile does from the firmware's reason word, so attribution keeps the firmware's blame and the command is not withdrawn. Also log the decoded fault: access, reason, address, page-table level, requestor and VM slot. Developed with Claude Code. Signed-off-by: Erik Bourget [ J775D: backport onto integration/aurora: adapted to integration's Backend::fault_sources, where InitData::is_m5 already exists; the classify hunk dropped, as integration does not withdraw a blamed compute command; the recovery log prefix changed to the series' "asahi:"; original 6b85752ad673 ] Signed-off-by: Jeff Queisser (cherry picked from commit 2a4980db7c0dac11213f60b2ad56ca621c2a89df in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 2 +- .../gpu/drm/asahi/g17/recovery/attribution.rs | 3 ++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 25 ++++++++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 81e2d1d81d6bb4..dcd584f28c0caf 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -11,7 +11,7 @@ mod attribution; mod slot; mod safety; pub(crate) use safety::{note_retired, since_retired}; -pub(crate) use attribution::Sources; +pub(crate) use attribution::{Sources, PAGE_FAULT}; use attribution::Attribution; pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index fdf0cd62cbd075..9c7f42c8398816 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -4,7 +4,8 @@ pub(super) const SLOTS: usize = 16; const STAMP_MASK: u64 = (1 << 40) - 1; -const PAGE_FAULT: u32 = 3; +/// Firmware recovery reason for a GPU MMU fault. +pub(crate) const PAGE_FAULT: u32 = 3; #[derive(Clone, Copy, Default)] pub(super) struct Source { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 96a3de5776013c..3a16ed7733ea88 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -137,7 +137,30 @@ impl recovery::Memory for Service<'_> { self.firmware.init.host_recovery() } fn fault_sources(&self) -> Result { - self.firmware.init.fault_sources() + let mut mmu_fault = false; + if self.firmware.init.is_m5() { + let registers = self.firmware.bridge.regs.m5_fault_registers(); + pr_err!("asahi: recovery fault registers: {:x?}\n", registers); + // MMU fault status: valid (bit 0), reason (1..3, 0 = unmapped), read + // (4), level (7..8), requestor (9..16), VM slot (17..22); the address + // word holds the faulting address shifted right by 6. + if let Ok([_, status, address, ..]) = registers { + if status & 1 != 0 { + mmu_fault = true; + let access = if status & 0x10 != 0 { "read" } else { "write" }; + pr_err!("asahi: recovery: GPU MMU fault: {} reason {} at {:#x}, level {}, requestor {:#x}, VM slot {}\n", + access, (status >> 1) & 7, + (address & ((1 << 42) - 1)) << 6, (status >> 7) & 3, (status >> 9) & 0xff, + (status >> 17) & 0x3f); + } + } + } + let mut sources = self.firmware.init.fault_sources()?; + // M5 reads no firmware reason word; a latched MMU fault is reason 3. + if mmu_fault && sources.reason.is_none() { + sources.reason = Some(recovery::PAGE_FAULT); + } + Ok(sources) } fn transition(&self, expected: u32, next: u32) -> Result { From 55f29e21567c59465192150903de36d19d030817 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 17:11:19 -0800 Subject: [PATCH 080/171] drm/asahi: fwid: Prepare firmware identity for two-role GPU images No change for any GPU that runs today: the T6040, T6041 and J713 builds keep their digests. It lets a later commit identify the M5 Ultra's (J775D) GPU firmware, whose loaded image holds two roles plus ranges the boot loader rewrites on every boot. fwid admits a GPU firmware build by the SHA-256 of its loaded TEXT with the 8 entropy bytes of its two entropy records zeroed. That assumes one firmware role per TEXT region and nothing else the boot loader changes. Neither holds for every SoC: the boot loader can load two GPU roles' TEXT back to back into one region, each with its own pair of entropy records, and fill other ranges of it on every boot (two RTKit sections it patches and two load fields of each image header). A build now lists its entropy records per role (at most two roles) and the (offset, length) ranges the boot loader fills. Both are zeroed in a private copy before hashing, so a loaded image and the firmware file it came from hash alike. A build with no boot-filled ranges is hashed as before, in place, with only its entropy zeroed: every existing build keeps its digest. A record counts only if it is whole (its tag, length and 8 entropy bytes all inside TEXT), in both paths and when an unknown image's records are located, so a tag in the last 8 bytes of an unknown image is refused like any other unknown image instead of indexing past the end. measure() copies the loaded TEXT twice, before anything is written to the GPU, and keeps the copy only if both agree; Measured::find() looks a build's images up in that one measurement, and Measured::identity() states an unknown image in the form a pinned build would take (sizes, each role's records, the number of boot-filled ranges zeroed, the digest). identify() is built on them. Testing: the fwid unit tests cover two roles, boot-filled ranges, malformed records and ranges (each leaving the image unchanged) and a record without its entropy bytes, in a pinned build's lookup and in an unknown image's report. They pass on the host: fwid.rs built by rustc --test against a stand-in for the kernel crate (KUnit itself was not run). Without the whole-record check the new test panics on the slice. The existing builds (T6040, T6041 and J713) name their one pair of records and no boot-filled ranges, and keep their digests. Signed-off-by: Jeff Queisser [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of (no-g16:g16/hw.rs)] (cherry picked from commit af6a4a1876a094a437d0138e6553b74fcd44aee3 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/fwid.rs | 384 +++++++++++++++++++++++------- drivers/gpu/drm/asahi/hw/t6040.rs | 9 +- drivers/gpu/drm/asahi/hw/t6041.rs | 3 +- 3 files changed, 311 insertions(+), 85 deletions(-) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index 34c65a0d8a0979..c5ee5c1bec2bc6 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -2,10 +2,15 @@ //! Exact GPU firmware identity. //! -//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its two entropy records +//! A build is the SHA-256 of the loaded TEXT with the 8 entropy bytes of its entropy records //! (`GKTS` and `ECAP`, each a tag, the length 8 and the bytes) zeroed, plus the TEXT and DATA -//! sizes and the records' offsets. An unknown build is refused with its -//! measured identity, so nothing is written to a firmware the driver was not written for. +//! sizes and the records' offsets. One TEXT region can hold several firmware roles back to back +//! (two GPU roles' TEXT loaded one after the other), each with its own pair of records. A build +//! can also name other ranges the boot loader fills (two sections it patches with the boot +//! arguments and the image header's load fields, which can change from boot to boot); they are +//! zeroed too, so the digest is that of the firmware's TEXT with the same ranges zeroed. An +//! unknown build is refused with its measured identity, so nothing is written to a firmware the +//! driver was not written for. use core::ops::Range; use kernel::{bindings, device, fmt, prelude::*}; @@ -15,9 +20,11 @@ pub(crate) struct FirmwareBuild { pub(crate) name: &'static str, pub(crate) text_size: u64, pub(crate) data_size: u64, - /// Offsets of the GKTS and ECAP entropy records in TEXT; their 8 entropy bytes are zeroed - /// before hashing. - pub(crate) entropy: [u32; 2], + /// Offsets of each role's GKTS and ECAP entropy records in TEXT, in TEXT order; their 8 + /// entropy bytes are zeroed before hashing. + pub(crate) entropy: &'static [[u32; 2]], + /// Other (offset, length) ranges of TEXT the boot loader fills, zeroed before hashing. + pub(crate) boot_filled: &'static [(u32, u32)], pub(crate) text_sha256: [u8; 32], } @@ -65,42 +72,111 @@ pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { Ok(copy?) } -/// Entropy record tags, in TEXT order. +/// Entropy record tags, in TEXT order within a role. const ENTROPY_TAGS: [[u8; 4]; 2] = [*b"GKTS", *b"ECAP"]; -/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` zeroed, or `None` if a -/// record (tag, then the length 8) is not there. `image` is left as it was. -fn digest_at(image: &mut [u8], at: [u32; 2]) -> Option<[u8; 32]> { - let mut saved = [[0u8; 8]; 2]; - for ((off, tag), save) in at.into_iter().zip(ENTROPY_TAGS).zip(saved.iter_mut()) { - let off = off as usize; - if image.get(off..off + 4) != Some(tag.as_slice()) - || image.get(off + 4..off + 8) != Some(8u32.to_le_bytes().as_slice()) - { +/// Firmware roles one TEXT region can hold. +const MAX_ROLES: usize = 2; + +/// The record (tag, then the length 8) that holds a role's entropy of `tag`. +fn record(tag: [u8; 4]) -> [u8; 8] { + let mut record = [0u8; 8]; + record[..4].copy_from_slice(&tag); + record[4..].copy_from_slice(&8u32.to_le_bytes()); + record +} + +/// Whether `image` holds a whole entropy record of `tag` at `off`: the tag, the length 8 and the +/// 8 entropy bytes, all inside `image`. +fn record_at(image: &[u8], off: usize, tag: [u8; 4]) -> bool { + off.checked_add(16) + .and_then(|end| image.get(off..end)) + .is_some_and(|whole| whole[..8] == record(tag)) +} + +/// The SHA-256 of `image` with the 8 entropy bytes of the records at `at` (one GKTS/ECAP pair +/// per role) and the `filled` ranges zeroed, or `None` if a record or range is not there. +/// `image` is left as it was. +fn digest_at(image: &mut [u8], at: &[[u32; 2]], filled: &[(u32, u32)]) -> Option<[u8; 32]> { + if at.is_empty() || at.len() > MAX_ROLES { + return None; + } + if !filled.is_empty() { + for (off, tag) in at.iter().flat_map(|pair| pair.iter().zip(ENTROPY_TAGS)) { + if !record_at(image, *off as usize, tag) { + return None; + } + } + let mut copy = KVVec::from_elem(0u8, image.len(), GFP_KERNEL).ok()?; + copy.copy_from_slice(image); + for off in at.iter().flatten() { + copy[*off as usize + 8..*off as usize + 16].fill(0); + } + for &(off, len) in filled { + let off = off as usize; + copy.get_mut(off..off.checked_add(len as usize)?)?.fill(0); + } + return Some(sha256(©)); + } + let records = || { + at.iter() + .flat_map(|pair| pair.iter().map(|&off| off as usize).zip(ENTROPY_TAGS)) + }; + let mut saved = [[0u8; 8]; 2 * MAX_ROLES]; + for ((off, tag), save) in records().zip(saved.iter_mut()) { + if !record_at(image, off, tag) { return None; } - save.copy_from_slice(image.get(off + 8..off + 16)?); + save.copy_from_slice(&image[off + 8..off + 16]); } - for off in at { - image[off as usize + 8..off as usize + 16].fill(0); + for (off, _) in records() { + image[off + 8..off + 16].fill(0); } let digest = sha256(image); - for (off, save) in at.into_iter().zip(saved) { - image[off as usize + 8..off as usize + 16].copy_from_slice(&save); + for ((off, _), save) in records().zip(saved) { + image[off + 8..off + 16].copy_from_slice(&save); } Some(digest) } -/// The first record of each tag in `image`, to report an unknown build. -fn find_entropy(image: &[u8]) -> Option<[u32; 2]> { - let mut found = [0u32; 2]; - for (slot, tag) in found.iter_mut().zip(ENTROPY_TAGS) { - let mut record = [0u8; 8]; - record[..4].copy_from_slice(&tag); - record[4..].copy_from_slice(&8u32.to_le_bytes()); - *slot = u32::try_from(image.windows(8).position(|w| w == record)?).ok()?; +/// The entropy records of each role found in `image`, to report an unknown build: each GKTS +/// record with the first ECAP record after it, at most [`MAX_ROLES`] pairs. +struct Records { + at: [[u32; 2]; MAX_ROLES], + roles: usize, +} + +impl Records { + fn find(image: &[u8]) -> Option { + let [gkts, ecap] = ENTROPY_TAGS.map(record); + let mut found = Records { + at: [[0; 2]; MAX_ROLES], + roles: 0, + }; + let mut from = 0; + while found.roles < MAX_ROLES { + let Some(g) = image.get(from..)?.windows(8).position(|w| w == gkts) else { + break; + }; + let g = from + g; + let Some(e) = image.get(g + 8..)?.windows(8).position(|w| w == ecap) else { + break; + }; + let e = g + 8 + e; + // A record whose entropy bytes run past TEXT is no record ([`record_at`]). + if !record_at(image, g, ENTROPY_TAGS[0]) || !record_at(image, e, ENTROPY_TAGS[1]) { + break; + } + found.at[found.roles] = [u32::try_from(g).ok()?, u32::try_from(e).ok()?]; + found.roles += 1; + from = e + 8; + } + (found.roles > 0).then_some(found) + } + + fn as_slice(&self) -> &[[u32; 2]] { + &self.at[..self.roles] } - Some(found) } /// Finds an exact identity, measuring each candidate at its own entropy offsets. @@ -108,54 +184,128 @@ fn lookup( builds: &'static [FirmwareBuild], text_size: u64, data_size: u64, - mut digest: impl FnMut([u32; 2]) -> Option<[u8; 32]>, + mut digest: impl FnMut(&FirmwareBuild) -> Option<[u8; 32]>, ) -> Option<&'static FirmwareBuild> { builds.iter().find(|build| { build.text_size == text_size && build.data_size == data_size - && digest(build.entropy) == Some(build.text_sha256) + && digest(build) == Some(build.text_sha256) }) } +/// A private, stable copy of a loaded TEXT, with the size of its DATA. +pub(crate) struct Measured { + text: KVVec, + data_size: u64, +} + +/// Copies the TEXT loaded at `text` (DATA `data_size` bytes) twice, before any firmware-visible +/// write, and keeps the copy if both agree (`EAGAIN` if the TEXT changed meanwhile). +pub(crate) fn measure(text: Range, data_size: u64) -> Result { + let size = usize::try_from(text.end.checked_sub(text.start).ok_or(EINVAL)?)?; + let copy = copy_physical(text.start, size)?; + let again = copy_physical(text.start, size)?; + if copy.as_slice() != again.as_slice() { + return Err(EAGAIN); + } + Ok(Measured { + text: copy, + data_size, + }) +} + +impl Measured { + /// The build among `builds` this TEXT is exactly, if any. + pub(crate) fn find( + &mut self, + builds: &'static [FirmwareBuild], + ) -> Option<&'static FirmwareBuild> { + let text_size = self.text.len() as u64; + let text = &mut self.text; + lookup(builds, text_size, self.data_size, |build| { + digest_at(&mut text[..], build.entropy, build.boot_filled) + }) + } + + /// The measured identity, to report a build that is not admitted: normalized as the first + /// of `builds` with this TEXT size would be, else with only its entropy records zeroed. + pub(crate) fn identity( + &mut self, + builds: impl IntoIterator, + ) -> Identity { + let text_size = self.text.len() as u64; + let filled = builds + .into_iter() + .find(|build| build.text_size == text_size && !build.boot_filled.is_empty()) + .map_or(&[][..], |build| build.boot_filled); + let records = Records::find(&self.text); + let digest = records + .as_ref() + .and_then(|r| digest_at(&mut self.text, r.as_slice(), filled)); + Identity { + text_size, + data_size: self.data_size, + filled: filled.len(), + measured: records.zip(digest), + } + } +} + +/// A measured identity, as a pinned [`FirmwareBuild`] would state it. +pub(crate) struct Identity { + text_size: u64, + data_size: u64, + /// How many boot-filled ranges were zeroed besides the entropy. + filled: usize, + measured: Option<(Records, [u8; 32])>, +} + +impl fmt::Display for Identity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TEXT {:#x} DATA {:#x}", self.text_size, self.data_size)?; + let Some((records, digest)) = &self.measured else { + return write!(f, " has no entropy records"); + }; + write!(f, " entropy")?; + for (i, [gkts, ecap]) in records.as_slice().iter().enumerate() { + write!( + f, + "{}{:#x}/{:#x}", + if i == 0 { " " } else { "," }, + gkts, + ecap + )?; + } + if self.filled != 0 { + write!(f, " ({} boot-filled ranges zeroed)", self.filled)?; + } + write!(f, " sha256 ")?; + fmt::Display::fmt(&Hex(digest), f) + } +} + /// Identifies the firmware build loaded at `text` (DATA `data_size` bytes) among `builds`, /// before any firmware-visible write. +#[cfg_attr(not(CONFIG_DRM_ASAHI_G16), allow(dead_code))] pub(crate) fn identify( dev: &device::Device, text: Range, data_size: u64, builds: &'static [FirmwareBuild], ) -> Result<&'static FirmwareBuild> { - let size = usize::try_from(text.end - text.start)?; - let mut copy = copy_physical(text.start, size)?; - let again = copy_physical(text.start, size)?; - if copy.as_slice() != again.as_slice() { - dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); - return Err(EAGAIN); - } - if let Some(build) = lookup(builds, size as u64, data_size, |at| { - digest_at(&mut copy, at) - }) { + let mut measured = measure(text, data_size).inspect_err(|e| { + if *e == EAGAIN { + dev_err!(dev, "GPU firmware TEXT changed while it was measured\n"); + } + })?; + if let Some(build) = measured.find(builds) { dev_info!(dev, "GPU firmware build: {}\n", build.name); return Ok(build); } - let Some((entropy, digest)) = - find_entropy(©).and_then(|at| Some((at, digest_at(&mut copy, at)?))) - else { - dev_err!( - dev, - "unsupported GPU firmware build: TEXT {:#x} has no entropy records\n", - size - ); - return Err(EINVAL); - }; dev_err!( dev, - "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} entropy {:#x}/{:#x} sha256 {}\n", - size, - data_size, - entropy[0], - entropy[1], - Hex(&digest) + "unsupported GPU firmware build: {}\n", + measured.identity(builds) ); for build in builds { dev_err!(dev, "supported: {}\n", build.name); @@ -167,48 +317,120 @@ pub(crate) fn identify( mod tests { use super::*; - fn image() -> [u8; 64] { - let mut image = [0x5a; 64]; - for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { - image[off..off + 4].copy_from_slice(&tag); - image[off + 4..off + 8].copy_from_slice(&8u32.to_le_bytes()); + /// `roles` GKTS/ECAP pairs, at 0/32 within each 64 bytes. + fn image() -> [u8; N] { + let mut image = [0x5a; N]; + for role in 0..N / 64 { + for (off, tag) in [0usize, 32].into_iter().zip(ENTROPY_TAGS) { + image[role * 64 + off..role * 64 + off + 8].copy_from_slice(&record(tag)); + } } image } #[test] fn entropy_is_normalized_without_changing_the_image() { - let mut first = image(); + let mut first = image::<64>(); let original = first; - let digest = digest_at(&mut first, [0, 32]).unwrap(); + let digest = digest_at(&mut first, &[[0, 32]], &[]).unwrap(); assert_eq!(first, original); let mut second = first; second[8..16].fill(0xff); second[40..48].fill(0x01); - assert_eq!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_eq!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); second[63] ^= 1; - assert_ne!(digest_at(&mut second, [0, 32]), Some(digest)); + assert_ne!(digest_at(&mut second, &[[0, 32]], &[]), Some(digest)); + } + + #[test] + fn every_role_is_normalized() { + let mut first = image::<128>(); + let original = first; + let both = &[[0, 32], [64, 96]]; + let digest = digest_at(&mut first, both, &[]).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[72..80].fill(0xff); + second[104..112].fill(0x01); + assert_eq!(digest_at(&mut second, both, &[]), Some(digest)); + // The second role's entropy is part of the digest unless its records are named. + assert_ne!( + digest_at(&mut second, &[[0, 32]], &[]), + digest_at(&mut first, &[[0, 32]], &[]) + ); + second[127] ^= 1; + assert_ne!(digest_at(&mut second, both, &[]), Some(digest)); + let found = Records::find(&first).unwrap(); + assert_eq!(found.as_slice(), both); + assert_eq!( + digest_at(&mut first, &[[0, 32], [64, 96], [0, 32]], &[]), + None + ); + } + + #[test] + fn boot_filled_ranges_are_normalized() { + let mut first = image::<128>(); + let original = first; + let filled = &[(16, 8), (100, 28)]; + let digest = digest_at(&mut first, &[[0, 32]], filled).unwrap(); + assert_eq!(first, original); + let mut second = first; + second[16..24].fill(0x11); + second[100..128].fill(0x22); + assert_eq!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + second[99] ^= 1; + assert_ne!(digest_at(&mut second, &[[0, 32]], filled), Some(digest)); + assert_eq!(digest_at(&mut first, &[[0, 32]], &[(120, 9)]), None); + } + + #[test] + fn a_record_without_its_entropy_bytes_is_refused() { + // The role's ECAP record is in the last 8 bytes, so it has no entropy bytes. + let mut tail = image::<64>(); + tail[32..40].fill(0x5a); + tail[56..64].copy_from_slice(&record(ENTROPY_TAGS[1])); + let original = tail; + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[(16, 8)]), None); + assert_eq!(digest_at(&mut tail, &[[0, 56]], &[]), None); + assert_eq!(tail, original); + assert!(Records::find(&tail).is_none()); + // An unknown build with a pinned build's TEXT size is reported, not a panic. + static PINNED: [FirmwareBuild; 1] = [FirmwareBuild { + name: "pinned", + text_size: 64, + data_size: 0, + entropy: &[[0, 32]], + boot_filled: &[(16, 8)], + text_sha256: [0; 32], + }]; + let mut text = KVVec::from_elem(0u8, tail.len(), GFP_KERNEL).unwrap(); + text.copy_from_slice(&tail); + let mut measured = Measured { text, data_size: 0 }; + assert!(measured.find(&PINNED).is_none()); + assert!(measured.identity(&PINNED).measured.is_none()); } #[test] fn malformed_records_are_refused_without_changing_the_image() { - let mut image = image(); - assert_eq!(digest_at(&mut image, [0, 56]), None); + let mut image = image::<64>(); + assert_eq!(digest_at(&mut image, &[[0, 56]], &[]), None); + assert_eq!(digest_at(&mut image, &[], &[]), None); image[36] = 7; let original = image; - assert_eq!(digest_at(&mut image, [0, 32]), None); + assert_eq!(digest_at(&mut image, &[[0, 32]], &[]), None); assert_eq!(image, original); - assert_eq!(find_entropy(&image), None); + assert!(Records::find(&image).is_none()); } fn measured( builds: &'static [FirmwareBuild], data_size: u64, - entropy: [u32; 2], + entropy: &[[u32; 2]], digest: [u8; 32], ) -> Option<&'static FirmwareBuild> { - lookup(builds, 0x60000, data_size, |at| { - (at == entropy).then_some(digest) + lookup(builds, 0x60000, data_size, |build| { + (build.entropy == entropy).then_some(digest) }) } @@ -238,13 +460,13 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(build.images, 0x138000, [0x59839, 0x5991c], digest).unwrap(); + let found = measured(build.images, 0x138000, &[[0x59839, 0x5991c]], digest).unwrap(); assert!(core::ptr::eq(found, &build.images[i])); - assert!(measured(build.images, 0x13c000, [0x59839, 0x5991c], digest).is_none()); - assert!(measured(build.images, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(build.images, 0x13c000, &[[0x59839, 0x5991c]], digest).is_none()); + assert!(measured(build.images, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); assert!(lookup(build.images, 0x60001, 0x138000, |_| Some(digest)).is_none()); } - assert!(measured(build.images, 0x138000, [0x59839, 0x5991c], [0; 32]).is_none()); + assert!(measured(build.images, 0x138000, &[[0x59839, 0x5991c]], [0; 32]).is_none()); } #[cfg(CONFIG_DRM_ASAHI_G16)] @@ -264,10 +486,10 @@ mod tests { ], ]; for (i, digest) in digests.into_iter().enumerate() { - let found = measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], digest).unwrap(); + let found = measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], digest).unwrap(); assert!(core::ptr::eq(found, &builds[i])); - assert!(measured(builds, 0x138000, [0x5c8df, 0x5c9c2], digest).is_none()); + assert!(measured(builds, 0x138000, &[[0x5c8df, 0x5c9c2]], digest).is_none()); } - assert!(measured(builds, 0x13c000, [0x5c8df, 0x5c9c2], [0; 32]).is_none()); + assert!(measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], [0; 32]).is_none()); } } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index c3486d2888f0b8..725b8817ede922 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -71,7 +71,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, @@ -82,7 +83,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, @@ -93,7 +95,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x59839, 0x5991c], + entropy: &[[0x59839, 0x5991c]], + boot_filled: &[], text_sha256: [ 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 298aaf3f2d6014..2dff2c8ec91fd2 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -55,7 +55,8 @@ pub(crate) const CONFIG: Config = Config { name: "Apple GFX 25F84 for T6041 (normalized loaded image)", text_size: 0x60000, data_size: 0x138000, - entropy: [0x5983a, 0x5991d], + entropy: &[[0x5983a, 0x5991d]], + boot_filled: &[], text_sha256: [ 0x6d, 0xa4, 0xd4, 0x42, 0x35, 0x35, 0x14, 0x44, 0xf6, 0xec, 0x00, 0x33, 0xf1, 0x9c, 0xbf, 0x98, 0xd3, 0x43, 0x91, 0x28, 0x83, 0xe4, 0xe2, 0x94, 0xc1, 0x8b, 0xb8, 0xb8, From ce1a38baa0d386c0e34ab61f3d1f35a05ae74d71 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:05:40 -0800 Subject: [PATCH 081/171] drm/asahi: g17: M5: Prepare the M5 graph's board values for a third board No change in what is written for J714 or J716c. The per-board values move into a table so the M5 Ultra (J775D), which runs the same firmware build with other values, can be added as a third board. M5Board picks each per-board value of the M5 graph by a match on the board: performance and auxiliary-domain states, core count and revision, AON records, register windows, render pool and minimum parameter buffer. The policy controller configuration and the leakage calibration it was made for stay outside it. A third T6050 board runs the same firmware build with other values of each, more voltage tables than a performance state holds and a second auxiliary lane. Collect the values in fw::m5::initdata::Board, a table per board (J714 and J716C, with M5Board's values), and name a configuration's board in Config::m5 in place of Config::m5_board. The builder, the HW data and AON encoders, the render pool selection, the board check and the maximum frequency read the table, and the table carries the controller configuration and its calibration. A performance state holds up to eight voltage tables and an auxiliary state two lanes; a board names how many tables it uses, and lanes past them carry table 0, as before. J716C keeps J714's controller configuration and calibration, so, as before, the configuration applies only to a part with J714's calibration. No change in what is written for J714 or J716c. Signed-off-by: Jeff Queisser (cherry picked from commit a3d419ac6c20972e32693a26eb8545e42b0969fd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 392 +++++++++++--------- drivers/gpu/drm/asahi/g17/fw/m5/pool.rs | 12 +- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 26 +- drivers/gpu/drm/asahi/g17/mod.rs | 14 +- drivers/gpu/drm/asahi/hw/t6040.rs | 2 +- drivers/gpu/drm/asahi/hw/t6041.rs | 2 +- drivers/gpu/drm/asahi/hw/t6050.rs | 5 +- drivers/gpu/drm/asahi/hw/t8140.rs | 2 +- 8 files changed, 262 insertions(+), 193 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 0bf8c85ea5ed79..91dcd33ddfb6c9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1473,21 +1473,37 @@ const M5_HW_AUX_PERF_STATE: usize = 0x1cac; const M5_HW_AUX_PERF_STATE_SIZE: usize = 0x148; // Clock-domain 2: MHz, core mV, SRAM mV. The second voltage lane is unused. -const J714_AUX_PERF_STATES: [(u32, u32, u32); 14] = [ +const J714_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ (24, 125, 775), (338, 590, 775), (486, 620, 775), (636, 645, 775), (796, 685, 775), (888, 700, 775), (988, 745, 775), (1084, 770, 775), (1182, 795, 795), (1278, 835, 835), (1374, 860, 860), (1470, 900, 900), (1578, 960, 960), (1620, 990, 990), -]; +]); // J716c ADT afr-perf-states (identical to cs-perf-states). SRAM is // max(775, core), the rule the J714 column follows. -const J716C_AUX_PERF_STATES: [(u32, u32, u32); 14] = [ +const J716C_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ (24, 125, 775), (338, 590, 775), (486, 625, 775), (636, 640, 775), (796, 690, 775), (888, 700, 775), (988, 740, 775), (1084, 765, 775), (1182, 780, 780), (1278, 830, 830), (1374, 875, 875), (1470, 910, 910), (1578, 975, 975), (1620, 1020, 1020), -]; +]); + +/// An auxiliary-domain performance state: MHz, then core and SRAM mV of each of its two voltage +/// lanes. A one-lane board leaves the second lane 0. +pub(crate) type M5AuxPerfState = (u32, [u32; 2], [u32; 2]); + +/// One-lane auxiliary tables. +const fn one_lane(states: [(u32, u32, u32); 14]) -> [M5AuxPerfState; 14] { + let mut out = [(0, [0; 2], [0; 2]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state] = (frequency, [core, 0], [sram, 0]); + state += 1; + } + out +} // Evaluate the frequency-only curve at compile time; no kernel FPU use. // Conversion uses Hz before the single-precision multiply and divide. @@ -1505,20 +1521,18 @@ const fn relative_frequency(states: &[M5PerfState; 14]) -> [u32; 16] { values } -const J714_RELATIVE_FREQUENCY: [u32; 16] = relative_frequency(&J714_GPU_PERF_STATES); - -const J716C_RELATIVE_FREQUENCY: [u32; 16] = relative_frequency(&J716C_GPU_PERF_STATES); - -fn encode_m5_aux_perf_states(board: M5Board, out: &mut [u8]) { +fn encode_m5_aux_perf_states(board: &Board, out: &mut [u8]) { let table = &mut out[M5_HW_AUX_PERF_STATE.. M5_HW_AUX_PERF_STATE + M5_HW_AUX_PERF_STATE_SIZE]; table.fill(0); put_u32(table, 0, 13); put_u32(table, 4, 14); - for (state, &(frequency, core, sram)) in board.aux_perf_states().iter().enumerate() { + for (state, &(frequency, core, sram)) in board.aux_perf_states.iter().enumerate() { put_u32(table, 8 + state * 4, frequency); - put_u32(table, 0x48 + state * 8, core); - put_u32(table, 0xc8 + state * 8, sram); + for lane in 0..2 { + put_u32(table, 0x48 + state * 8 + lane * 4, core[lane]); + put_u32(table, 0xc8 + state * 8 + lane * 4, sram[lane]); + } } } @@ -1578,28 +1592,26 @@ pub(crate) fn encode_common26_region_c(out: &mut [u8]) -> Result<(), InitdataErr Ok(()) } -pub(crate) fn encode_common26_aon_ptd(board: M5Board, out: &mut [u8]) -> Result<(), InitdataError> { - if out.len() != M5_AON_PTD_SIZE { +// J714 descriptors carry device-specific AON table indices. +const J714_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0079_0001, 0x0000_1401], + [0x6731_5452, 0x00bc_0001, 0x0000_1401], + [0x7230_5452, 0x00ce_0001, 0x0000_1401], +]; + +// J716c's rows tagged RT0g, RT1g and RT0r. +const J716C_AON_RECORDS: [[u32; 3]; 3] = [ + [0x6730_5452, 0x0058_0001, 0x0000_1401], + [0x6731_5452, 0x00a3_0001, 0x0000_1401], + [0x7230_5452, 0x00b8_0001, 0x0000_1401], +]; + +pub(crate) fn encode_common26_aon_ptd(board: &Board, out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_AON_PTD_SIZE || board.aon_records.len() * 12 > out.len() { return Err(InitdataError::BufferSize); } - // The ADT aon-ptd table rows tagged RT0g, RT1g and RT0r; their indices are - // device-specific. - const J714_RECORDS: [[u32; 3]; 3] = [ - [0x6730_5452, 0x0079_0001, 0x0000_1401], - [0x6731_5452, 0x00bc_0001, 0x0000_1401], - [0x7230_5452, 0x00ce_0001, 0x0000_1401], - ]; - const J716C_RECORDS: [[u32; 3]; 3] = [ - [0x6730_5452, 0x0058_0001, 0x0000_1401], - [0x6731_5452, 0x00a3_0001, 0x0000_1401], - [0x7230_5452, 0x00b8_0001, 0x0000_1401], - ]; - let records = match board { - M5Board::J714 => &J714_RECORDS, - M5Board::J716c => &J716C_RECORDS, - }; out.fill(0); - for (index, record) in records.iter().enumerate() { + for (index, record) in board.aon_records.iter().enumerate() { for (word, value) in record.iter().enumerate() { put_u32(out, index * 12 + word * 4, *value); } @@ -1714,94 +1726,80 @@ pub(crate) const M5_HW_DATA_INSERT: usize = 0x267c; pub(crate) const M5_HW_DATA_INSERT_LEN: usize = 8; -/// Board whose HW data the M5 GPU is started with. Both are T6050. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum M5Board { - /// M5 Pro: 2 clusters of 10 cores. - J714, - /// M5 Max: 4 clusters of 10 cores. - J716c, -} - -impl M5Board { - pub(crate) const fn perf_states(self) -> &'static [M5PerfState; 14] { - match self { - Self::J714 => &J714_GPU_PERF_STATES, - Self::J716c => &J716C_GPU_PERF_STATES, - } - } - - /// Voltage tables in [`M5Board::perf_states`] (ADT perf-state-table-count). - const fn perf_tables(self) -> usize { - match self { - Self::J714 => 2, - Self::J716c => 4, - } - } - - const fn aux_perf_states(self) -> &'static [(u32, u32, u32); 14] { - match self { - Self::J714 => &J714_AUX_PERF_STATES, - Self::J716c => &J716C_AUX_PERF_STATES, - } - } - - const fn relative_frequency(self) -> &'static [u32; 16] { - match self { - Self::J714 => &J714_RELATIVE_FREQUENCY, - Self::J716c => &J716C_RELATIVE_FREQUENCY, - } - } - - /// The core type, which follows the ID variant, and the firmware revision - /// the HW data names. - const fn core_revision(self) -> (u32, u32) { - match self { - Self::J714 => (0x18, 4), - Self::J716c => (0x19, 4), - } - } - - /// GPU clusters of the device. - pub(crate) const fn clusters(self) -> u32 { - match self { - Self::J714 => 2, - Self::J716c => 4, - } - } - - /// All cores of the device. - pub(crate) const fn gpu_cores(self) -> u32 { - match self { - Self::J714 => 20, - Self::J716c => 40, - } - } - - /// Render block size and private-memory address inventory. - pub(crate) const fn render_pool(self) -> (usize, u64) { - match self { - Self::J714 => (0x40_0000, 1 << 32), - Self::J716c => (0x80_0000, 1 << 33), - } - } +/// What the M5 graph's HW data and power policy depend on per board. Every board is a T6050 +/// running the same firmware build; [`crate::g17::Config::m5`] names the board's. +pub(crate) struct Board { + /// GPU clusters. + pub(crate) clusters: u32, + /// Cores of all clusters. + pub(crate) gpu_cores: u32, + /// The core type and firmware revision the HW data names. + core_revision: (u32, u32), + /// Main-domain performance states. + pub(crate) perf_states: &'static [M5PerfState; 14], + /// Voltage tables in [`Board::perf_states`]. + perf_tables: usize, + /// Auxiliary-domain performance states. + aux_perf_states: &'static [M5AuxPerfState; 14], + /// The relative-frequency curve of [`Board::perf_states`]. + relative_frequency: [u32; 16], + /// Rows of the board's AON table the firmware reads. + aon_records: &'static [[u32; 3]], + /// Register windows the firmware maps, in device-VA order. + register_windows: &'static [M5RegisterWindowSpec], + /// The smallest parameter buffer, bytes. Renders carry it in pages too + /// (`target::min_parameter_buffer_pages()`). + min_parameter_buffer: u32, + /// Render block size and private-memory address inventory of the board's + /// render pools, bytes. + pub(crate) render_pool: (usize, u64), + /// The leakage calibration words [`Board::controller_defaults`] are valid for, or `None` + /// where they hold for every part of the board. + pub(crate) calibration: Option<[u32; 3]>, + /// The policy controllers' configuration. + pub(crate) controller_defaults: fn(&mut [u8]) -> Result<(), InitdataError>, +} + +/// M5 Pro: two clusters of ten cores. +pub(crate) const J714: Board = Board { + clusters: 2, + gpu_cores: 20, + core_revision: (0x18, 4), + perf_states: &J714_GPU_PERF_STATES, + perf_tables: 2, + aux_perf_states: &J714_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J714_GPU_PERF_STATES), + aon_records: &J714_AON_RECORDS, + register_windows: &J714_REGISTER_WINDOWS, + min_parameter_buffer: 0x0020_0000, + render_pool: (0x40_0000, 1 << 32), + calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), + controller_defaults: encode_m5_policy_controller_defaults, +}; - /// The minimum parameter-buffer size. Renders carry it in - /// pages too (`target::min_parameter_buffer_pages()`). - pub(crate) const fn min_parameter_buffer(self) -> u32 { - match self { - Self::J714 => 0x0020_0000, - Self::J716c => 0x0030_0000, - } - } +/// M5 Max: four clusters of ten cores. +pub(crate) const J716C: Board = Board { + clusters: 4, + gpu_cores: 40, + core_revision: (0x19, 4), + perf_states: &J716C_GPU_PERF_STATES, + perf_tables: 4, + aux_perf_states: &J716C_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES), + aon_records: &J716C_AON_RECORDS, + register_windows: &J716C_REGISTER_WINDOWS, + min_parameter_buffer: 0x0030_0000, + render_pool: (0x80_0000, 1 << 33), + // J714's controller configuration, for a part with J714's calibration only. + calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), + controller_defaults: encode_m5_policy_controller_defaults, +}; - pub(crate) const fn register_window_specs(self) -> &'static [M5RegisterWindowSpec; 29] { - match self { - Self::J714 => &J714_REGISTER_WINDOW_SPECS, - Self::J716c => &J716C_REGISTER_WINDOW_SPECS, - } - } -} +const _: () = { + use super::target::min_parameter_buffer_pages; + assert!(J714.min_parameter_buffer as u64 == min_parameter_buffer_pages(J714.clusters) << 15); + assert!(J716C.min_parameter_buffer as u64 == min_parameter_buffer_pages(J716C.clusters) << 15); +}; /// Populate the enabled-USC count from the device's sampled core masks. /// This count covers the whole device, not one cluster. @@ -1831,25 +1829,60 @@ const J714_POWER_POLICY_MINIMUM_ENTRIES: usize = 16 * 32; pub(crate) const J714_GPU_NUM_PERF_STATES: u32 = 13; -/// A main-domain performance state: MHz, then core and SRAM mV for each -/// voltage table. HW perf lane n carries table n; lanes past the board's table -/// count carry table 0, as J714 does for lanes 0 and 2-15. -pub(crate) type M5PerfState = (u32, [u32; 4], [u32; 4]); - -// J714: two voltage tables. -pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = [ - (0, [125, 125, 0, 0], [775, 775, 0, 0]), (338, [605, 605, 0, 0], [775, 775, 0, 0]), - (486, [620, 635, 0, 0], [775, 775, 0, 0]), (636, [655, 655, 0, 0], [775, 775, 0, 0]), - (796, [705, 705, 0, 0], [775, 775, 0, 0]), (888, [715, 715, 0, 0], [775, 775, 0, 0]), - (988, [745, 745, 0, 0], [775, 775, 0, 0]), (1084, [770, 770, 0, 0], [775, 775, 0, 0]), - (1182, [800, 800, 0, 0], [800, 800, 0, 0]), (1278, [840, 840, 0, 0], [840, 840, 0, 0]), - (1374, [880, 880, 0, 0], [880, 880, 0, 0]), (1470, [925, 925, 0, 0], [925, 925, 0, 0]), - (1578, [1000, 1000, 0, 0], [1000, 1000, 0, 0]), - (1620, [1025, 1040, 0, 0], [1025, 1040, 0, 0]), -]; +/// A main-domain performance state: MHz, then core and SRAM mV for each voltage table. HW perf +/// lane n carries table n; lanes past the board's table count carry table 0. +pub(crate) type M5PerfState = (u32, [u32; M5_PERF_LANES], [u32; M5_PERF_LANES]); + +/// Voltage tables an [`M5PerfState`] can hold. +pub(crate) const M5_PERF_LANES: usize = 8; + +/// Two-table states (MHz, two core-voltage tables, two SRAM-voltage tables), the rest 0. +const fn two_tables(states: [(u32, u32, u32, u32, u32); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core0, core1, sram0, sram1) = states[state]; + out[state].0 = frequency; + out[state].1[0] = core0; + out[state].1[1] = core1; + out[state].2[0] = sram0; + out[state].2[1] = sram1; + state += 1; + } + out +} + +// Frequency in MHz, two core-voltage tables, two SRAM-voltage tables (mV). +pub(crate) const J714_GPU_PERF_STATES: [M5PerfState; 14] = two_tables([ + (0, 125, 125, 775, 775), (338, 605, 605, 775, 775), + (486, 620, 635, 775, 775), (636, 655, 655, 775, 775), + (796, 705, 705, 775, 775), (888, 715, 715, 775, 775), + (988, 745, 745, 775, 775), (1084, 770, 770, 775, 775), + (1182, 800, 800, 800, 800), (1278, 840, 840, 840, 840), + (1374, 880, 880, 880, 880), (1470, 925, 925, 925, 925), + (1578, 1000, 1000, 1000, 1000), (1620, 1025, 1040, 1025, 1040), +]); + +/// Four-table states (MHz, four core-voltage tables, four SRAM-voltage tables), the rest 0. +const fn four_tables(states: [(u32, [u32; 4], [u32; 4]); 14]) -> [M5PerfState; 14] { + let mut out = [(0, [0; M5_PERF_LANES], [0; M5_PERF_LANES]); 14]; + let mut state = 0; + while state < 14 { + let (frequency, core, sram) = states[state]; + out[state].0 = frequency; + let mut table = 0; + while table < 4 { + out[state].1[table] = core[table]; + out[state].2[table] = sram[table]; + table += 1; + } + state += 1; + } + out +} // J716c ADT perf-states / perf-states-sram: four tables (0 == 3, 1 == 2). -pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = [ +pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = four_tables([ (0, [125, 125, 125, 125], [775, 775, 775, 775]), (338, [635, 635, 635, 635], [775, 775, 775, 775]), (486, [655, 640, 640, 655], [775, 775, 775, 775]), @@ -1864,10 +1897,11 @@ pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = [ (1470, [920, 905, 905, 920], [920, 905, 905, 920]), (1578, [995, 980, 980, 995], [995, 980, 980, 995]), (1620, [1015, 1015, 1015, 1015], [1015, 1015, 1015, 1015]), -]; +]); pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; +#[derive(Copy, Clone)] pub(crate) struct M5RegisterWindowSpec { pub(crate) slot: usize, @@ -1885,7 +1919,7 @@ const fn jw(slot: usize, instances: &'static [u64], size: u32, unk_18: u64, flag const G: u64 = J714_GPU_BASE; -pub(crate) const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ +const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ jw(0, &[0x2_8041_4000], 0x4000, 0, 2), jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000], 0x18000, 0, 2), jw(4, &[0x2_8046_8000], 0x4000, 0, 0), @@ -1917,11 +1951,25 @@ pub(crate) const J714_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = [ jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), ]; +const J714_REGISTER_WINDOW_58: M5RegisterWindowSpec = + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); + +/// J714's windows, ending with slot 58. +const J714_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J714_REGISTER_WINDOW_SPECS.len() { + windows[index] = J714_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows +}; + /// J714 with all eight AMCCs in slot 3 (J716c amcc-enable-mask 0xff, MCC /// reg[14..21] + 0x104000) and four instances, one per cluster, in slots 20 and /// 24, where J714 has two at the same stride. Slot 44 is a fixed pair on every M5 /// part. -pub(crate) const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { +const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { let mut specs = J714_REGISTER_WINDOW_SPECS; assert!(specs[1].slot == 3 && specs[11].slot == 20 && specs[14].slot == 24); specs[1] = jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, @@ -1934,17 +1982,17 @@ pub(crate) const J716C_REGISTER_WINDOW_SPECS: [M5RegisterWindowSpec; 29] = { specs }; -const _: () = { - use super::target::min_parameter_buffer_pages; - assert!(M5Board::J714.min_parameter_buffer() as u64 - == min_parameter_buffer_pages(M5Board::J714.clusters()) << 15); - assert!(M5Board::J716c.min_parameter_buffer() as u64 - == min_parameter_buffer_pages(M5Board::J716c.clusters()) << 15); +/// J716c's windows, ending with slot 58. +const J716C_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { + let mut windows = [J714_REGISTER_WINDOW_58; 30]; + let mut index = 0; + while index < J716C_REGISTER_WINDOW_SPECS.len() { + windows[index] = J716C_REGISTER_WINDOW_SPECS[index]; + index += 1; + } + windows }; -pub(crate) const M5_REGISTER_WINDOW_58: M5RegisterWindowSpec = - jw(58, &[G + 0xe7_3800], 0x4000, 0, 2); - #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct M5RegisterWindow { pub(crate) slot: usize, @@ -1955,21 +2003,34 @@ pub(crate) struct M5RegisterWindow { pub(crate) flag: u32, } -pub(crate) const M5_REGISTER_WINDOW_COUNT: usize = J714_REGISTER_WINDOW_SPECS.len() + 1; +/// The most register windows of any board. +pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J714_REGISTER_WINDOWS.len(); -pub(crate) fn m5_register_window_spec(board: M5Board, index: usize) - -> &'static M5RegisterWindowSpec -{ - board.register_window_specs().get(index).unwrap_or(&M5_REGISTER_WINDOW_58) +impl Board { + /// The board's `index`th register window. + pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { + self.register_windows.get(index) + } + + /// The board's register windows: the first [`Board::register_window_count`] entries. + pub(crate) fn register_windows(&self) -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] { + m5_register_windows(self.register_windows) + } + + /// Register windows of the board. + pub(crate) fn register_window_count(&self) -> usize { + self.register_windows.len() + } } -pub(crate) fn m5_register_windows(board: M5Board) -> [M5RegisterWindow; M5_REGISTER_WINDOW_COUNT] { +fn m5_register_windows(specs: &'static [M5RegisterWindowSpec]) + -> [M5RegisterWindow; M5_MAX_REGISTER_WINDOWS] +{ let mask = UAT_PAGE_SIZE as u64 - 1; let mut out = [M5RegisterWindow { slot: 0, device_va: 0, count: 0, size: 0, unk_18: 0, flag: 0 }; - M5_REGISTER_WINDOW_COUNT]; + M5_MAX_REGISTER_WINDOWS]; let mut next: u64 = 0; - for (index, window) in out.iter_mut().enumerate() { - let spec = m5_register_window_spec(board, index); + for (window, spec) in out.iter_mut().zip(specs) { let va = next + (spec.instances[0] & mask); let span = spec.instances.len() as u64 * ((spec.size as u64 + mask) & !mask); *window = M5RegisterWindow { @@ -1987,7 +2048,7 @@ pub(crate) fn m5_register_windows(board: M5Board) -> [M5RegisterWindow; M5_REGIS } pub(crate) fn encode_m5_register_map( - board: M5Board, + board: &Board, windows: &[M5RegisterWindow], out: &mut [u8], ) -> Result<(), InitdataError> { @@ -1996,7 +2057,7 @@ pub(crate) fn encode_m5_register_map( } out.fill(0); for (index, window) in windows.iter().enumerate() { - let spec = m5_register_window_spec(board, index); + let spec = board.register_windows.get(index).ok_or(InitdataError::RegisterWindow)?; if window.slot >= COMMON26_HW_REGISTER_MAP_SLOTS || !HW_REGISTER_APERTURE.contains(&window.device_va) { @@ -2014,7 +2075,7 @@ pub(crate) fn encode_m5_register_map( } pub(crate) fn encode_m5_hw_data( - board: M5Board, + board: &Board, aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], qos_high: u64, windows: &[M5RegisterWindow], @@ -2049,13 +2110,13 @@ pub(crate) fn encode_m5_hw_data( // J714's 26A428 HW data carries 1 here. The firmware copies this // header at initialization. put_u32(g18, 0x0fb8, 1); - let (core_type, revision) = board.core_revision(); + let (core_type, revision) = board.core_revision; put_u32(g18, 0x10d4, core_type); put_u32(g18, 0x10d8, revision); - put_u32(g18, 0x10dc, board.gpu_cores()); + put_u32(g18, 0x10dc, board.gpu_cores); put_u32(g18, 0x283a, 1); put_u32(g18, 0x283e, 1); - let states = board.perf_states(); + let states = board.perf_states; for (state, &(frequency, core, sram)) in states.iter().enumerate() { put_u32(g18, COMMON26_HW_PERF_FREQUENCY + state * 4, frequency); put_u32(g18, COMMON26_HW_PERF_SCALE + state * 4, COMMON26_PERF_SCALE_VALUE); @@ -2064,16 +2125,15 @@ pub(crate) fn encode_m5_hw_data( put_u32(g18, COMMON26_HW_PERF_RELATIVE_A + state * 4, relative); for lane in 0..COMMON26_HW_PERF_LANES { let row = state * COMMON26_HW_PERF_ROW_STRIDE + lane * 4; - let table = if lane < board.perf_tables() { lane } else { 0 }; + let table = if lane < board.perf_tables { lane } else { 0 }; put_u32(g18, COMMON26_HW_PERF_CORE_VOLTAGE + row, core[table]); put_u32(g18, COMMON26_HW_PERF_SRAM_VOLTAGE + row, sram[table]); } } - let relative = board.relative_frequency(); for index in 0..COMMON26_HW_PERF_ROWS { put_u32(g18, COMMON26_HW_PERF_INDEX_MAP + index * 4, index as u32); - put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, relative[index]); - put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, relative[index]); + put_u32(g18, COMMON26_HW_PERF_RELATIVE_B + index * 4, board.relative_frequency[index]); + put_u32(g18, M5_HW_AUX_RELATIVE_FREQUENCY + index * 4, board.relative_frequency[index]); } encode_m5_aux_perf_states(board, g18); let table = COMMON26_KSM_COMPLETION_TABLE_OFFSET; @@ -2088,7 +2148,7 @@ pub(crate) fn encode_m5_hw_data( put_u32(out, 0x2728, 0); put_u32(out, 0x272c, 1); // The inherited J813 minimum parameter-buffer size is replaced per board. - put_u32(out, 0x26b0, board.min_parameter_buffer()); + put_u32(out, 0x26b0, board.min_parameter_buffer); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs index 5fcd9ef4995587..b8c52bc634c5f8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool.rs @@ -4,19 +4,19 @@ use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use super::initdata::M5Board; +use super::initdata::{Board, J714}; pub(crate) const BLOCK_BYTES: usize = 0x20_0000; // Render block size and private-memory address inventory of the M5 -// part, selected at probe (`M5Board::render_pool()`). The inventory sizes the +// part, selected at probe (`Board::render_pool`). The inventory sizes the // directory, not the physical allocation; each pool allocates only its // requested initial blocks. -static RENDER_BLOCK_BYTES: AtomicUsize = AtomicUsize::new(M5Board::J714.render_pool().0); -static POOL_BYTES: AtomicU64 = AtomicU64::new(M5Board::J714.render_pool().1); +static RENDER_BLOCK_BYTES: AtomicUsize = AtomicUsize::new(J714.render_pool.0); +static POOL_BYTES: AtomicU64 = AtomicU64::new(J714.render_pool.1); /// Selects the board's render pool before any pool is built. -pub(crate) fn select_render_pool(board: M5Board) { - let (block, pool) = board.render_pool(); +pub(crate) fn select_render_pool(board: &Board) { + let (block, pool) = board.render_pool; RENDER_BLOCK_BYTES.store(block, Ordering::Relaxed); POOL_BYTES.store(pool, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 4c35e0e73f6bc8..37ba520b222fdb 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -66,7 +66,7 @@ impl InitData { } pub(crate) fn build_m5(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, - id: &crate::regs::G17Id, board: m::M5Board) -> Result { + id: &crate::regs::G17Id, board: &m::Board) -> Result { use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, PROT_GPU_SHARED_RW as GPU}; self.m5 = true; @@ -119,9 +119,10 @@ impl InitData { m::encode_m5_region_c(self.object_mut(self.region_c)?.bytes_mut()).map_err(|_| EINVAL)?; m::encode_m5_private_cluster(PRIVATE, FWCTL, &mut self.object_mut(self.cluster)?.bytes_mut()[..private.total]).map_err(|_| EINVAL)?; - let windows = m::m5_register_windows(board); + let windows = board.register_windows(); + let windows = &windows[..board.register_window_count()]; for (index, window) in windows.iter().enumerate() { - let spec = m::m5_register_window_spec(board, index); + let spec = board.register_window_spec(index).ok_or(EINVAL)?; let stride = (u64::from(window.size) + mmu::UAT_PGMSK as u64) & !(mmu::UAT_PGMSK as u64); for (instance, physical) in spec.instances.iter().copied().enumerate() { let offset = physical & mmu::UAT_PGMSK as u64; @@ -141,7 +142,7 @@ impl InitData { let views = m::PrimaryRegionViews { sentinel_high: BASE + 0xc07e_0000, alias_high: [PB, UMA], }; - m::encode_m5_hw_data(board, completions, QOS, &windows, + m::encode_m5_hw_data(board, completions, QOS, windows, &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE]).map_err(|_| EINVAL)?; m::set_m5_kick_shared( &mut self.object_mut(self.bundle)?.bytes_mut()[..m::M5_HW_DATA_SIZE], KICK_SHARED) @@ -155,10 +156,19 @@ impl InitData { m::encode_m5_bundle_policy(m::M5PolicyBindings { state: PRIVATE + private.policy_state as u64, aon_ptd: AON, boot_timestamp: 0, }, self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; - let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let calibration: Result> = node.get_property(c_str!("apple,leakage-calibration-words")); - if calibration.as_ref().is_ok_and(|words| words.as_slice() == [0xd2035701, 0xac3a1280, 0x0a281d81]) { - m::encode_m5_policy_controller_defaults(self.object_mut(self.bundle)?.bytes_mut()) + // The controller configuration applies where the part's leakage calibration matches the + // one it was made for, or on a board whose configuration holds for every part. + let calibrated = match board.calibration { + None => true, + Some(expected) => { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let calibration: Result> = + node.get_property(c_str!("apple,leakage-calibration-words")); + calibration.as_ref().is_ok_and(|words| words.as_slice() == expected) + } + }; + if calibrated { + (board.controller_defaults)(self.object_mut(self.bundle)?.bytes_mut()) .map_err(|_| EINVAL)?; let maximum = self.object(self.bundle)?.word(shared.policy + m::M5_POLICY_MAXIMUM_POWER)? .load(Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f37a33f9f699cc..4815888a25a2c4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -215,8 +215,6 @@ pub(crate) struct Config { /// Parameter buffer page metrics shared with the firmware; `None` where the firmware graph /// places its own (M5). pub(crate) pm_metrics: Option, - /// M5 board whose HW data the GPU is started with. - pub(crate) m5_board: Option, /// Register windows the firmware maps through the hardware data. pub(crate) windows: &'static [fw::initdata::RegisterWindow], /// Register-map slots that are reserved but have no window. @@ -238,6 +236,8 @@ pub(crate) struct Config { /// The SoC's values of region C; `None` where the firmware graph has its own (M5). pub(crate) region_c: Option<&'static fw::initdata::RegionCConfig>, pub(crate) chip: Option<&'static fw::initdata::v25g83::Chip>, + /// The M5 board whose HW data and power policy the M5 graph is built with; `None` off M5. + pub(crate) m5: Option<&'static fw::m5::initdata::Board>, /// Physical address of the firmware's per-slot recovery keys, read while the firmware is /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the /// firmware's own blame). @@ -1391,10 +1391,10 @@ impl Gpu { } let m5 = cfg.profile.is_m5(); if m5 { - let board = cfg.m5_board.ok_or(EINVAL)?; - if id.num_clusters != board.clusters() || id.num_clusters * id.num_cores != board.gpu_cores() { - dev_err!(dev.as_ref(), "M5 {:?} expects {} clusters and {} cores, the GPU reports {} x {}\n", - board, board.clusters(), board.gpu_cores(), id.num_clusters, id.num_cores); + let board = cfg.m5.ok_or(EINVAL)?; + if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores { + dev_err!(dev.as_ref(), "M5 board expects {} clusters and {} cores, the GPU reports {} x {}\n", + board.clusters, board.gpu_cores, id.num_clusters, id.num_cores); return Err(ENODEV); } fw::m5::pool::select_render_pool(board); @@ -1405,7 +1405,7 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = if m5 { - cfg.m5_board.ok_or(EINVAL)?.perf_states()[13].0 * 1000 + cfg.m5.ok_or(EINVAL)?.perf_states[13].0 * 1000 } else { Self::max_frequency_khz(dev)? }; diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 725b8817ede922..9674a4bd466654 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -38,7 +38,6 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), - m5_board: None, windows: ®ISTER_WINDOWS, reserved_slots: &[], perf: Some(&PERF), @@ -55,6 +54,7 @@ pub(crate) const CONFIG: Config = Config { }, region_c: Some(®ION_C), chip: None, + m5: None, adopt_root_entries: false, fabric_bridge: false, // The firmware's per-slot recovery keys are not known for this build; recovery relies on diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 2dff2c8ec91fd2..439751c3c5dd5e 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -29,7 +29,6 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), - m5_board: None, windows: ®ISTER_WINDOWS, reserved_slots: &RESERVED_SLOTS, perf: Some(&PERF), @@ -44,6 +43,7 @@ pub(crate) const CONFIG: Config = Config { }, region_c: Some(®ION_C), chip: Some(&CHIP), + m5: None, adopt_root_entries: false, fabric_bridge: false, recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index a3cbfb198d6d26..8f53ff3df4e056 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -3,7 +3,6 @@ //! //! The M5 firmware builds its HW data, performance tables and region C in its own graph //! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. -use crate::g17::fw::m5::initdata::M5Board; use crate::g17::{Config, CoreLayout, CrashBuffers, FirmwareProfile, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; use crate::regs::G17Id; @@ -33,7 +32,6 @@ pub(crate) const CONFIG: Config = Config { crash_buffers: CrashBuffers::Fixed(J714_CRASH_BUFFERS), profile: FirmwareProfile::M5NoBareMetalTables, pm_metrics: None, - m5_board: Some(M5Board::J714), windows: &[], reserved_slots: &[], perf: None, @@ -41,6 +39,7 @@ pub(crate) const CONFIG: Config = Config { uat_quirks: crate::mmu::UatQuirks::DEFAULT, region_c: None, chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J714), adopt_root_entries: true, fabric_bridge: false, recovery_slot_keys: None, @@ -81,7 +80,6 @@ pub(crate) const CONFIG_MAX: Config = Config { crash_buffers: CrashBuffers::Fixed(J716C_CRASH_BUFFERS), profile: FirmwareProfile::M5NoBareMetalTables, pm_metrics: None, - m5_board: Some(M5Board::J716c), windows: &[], reserved_slots: &[], perf: None, @@ -89,6 +87,7 @@ pub(crate) const CONFIG_MAX: Config = Config { uat_quirks: crate::mmu::UatQuirks::DEFAULT, region_c: None, chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J716C), adopt_root_entries: true, fabric_bridge: false, recovery_slot_keys: None, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index a369ff9a3dd0d7..15de4fd988b0c7 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -44,7 +44,6 @@ pub(crate) const CONFIG: Config = Config { va: 0x10_0008_0000, size: 0x8000, }), - m5_board: None, recovery_slot_keys: Some(recovery::SLOT_KEYS), gpu_generation: GpuGen::G17, windows: ®ISTER_WINDOWS, @@ -54,6 +53,7 @@ pub(crate) const CONFIG: Config = Config { uat_quirks: crate::mmu::UatQuirks::DEFAULT, region_c: Some(&crate::g17::fw::initdata::REGION_C_26A428), chip: None, + m5: None, adopt_root_entries: true, fabric_bridge: true, firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), From a8deaa48d893c05a24a62baf3ee28f87be0563c6 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:11:33 -0800 Subject: [PATCH 082/171] drm/asahi: t6050: Support the GPU on the bare-metal M5 Ultra (J775D) On the M5 Ultra Mac Studio (J775D) the GPU driver found no configuration for the chip ("No M5 configuration for GPU variant") and the GPU stayed off. It now boots both GPU firmware roles on bare metal and brings up the M5 firmware graph; jobs are accepted from the later submission commit on. J775D (T6050, G17D, two dies) runs the M5 firmware's protocol with the UAT roots both roles publish (FirmwareProfile::M5PublishedRoots). - The J775D board: both dies' register windows with their explicit instances (32), one voltage table per cluster in the main performance table, the two-lane auxiliary table, the six Ultra AON records, core type 0x1a revision 2, 8 clusters of 10 cores and the minimum parameter buffer for eight clusters. No policy controller configuration. Values from the J775D ADT. - hw/t6050.rs CONFIG_ULTRA, one of the configurations the ID registers select at probe (variant 6, two dies), with the UAT policy this board has run with: tables in no-map reservations, every table change written back, page-by-page TLB invalidation, the handoff's no-slot value without the lock, and the handoff byte after the flush records 1. Crash logs in fw-data. Its core layout is eight clusters of ten cores, with the eight-cluster operand-state layout (OperandState::EIGHT_BY_TEN). - M5 GPUs are matched on the board's cluster and core-slot counts, the values the M5 layouts are sized from, not on enabled cores: the J775D has 64 of its 80 cores enabled. The T6050 core layouts drop their own copy of the counts. - The M5 firmware build (V26A428_M5): the 26A428 variant with the M5 command set and no host power hold around render-pair registration (Variant::registration_power_hold). The bare-metal M5 firmware powers the cores for work by its own policy: under a hold, J775D's core-power witness reads unpowered until the hold is released, so taking one only times out. - Gpu::boot_m5: link the kernel lower root's tables the firmware reaches (entries 1, 7 and the canonical arena's 8) before either role starts, boot both roles, adopt the published roots, build the M5 graph in the shared kernel window and bind its metrics object, attach the host command port to the graph's views, then the M5 preparation, InitData to the primary and the secondary, POWER_ON_DONE, and wait for the opening records. A failed bootstrap keeps the session. The graph builder, the M5 preparation and the metrics binding are the M5 tree's. Signed-off-by: Jeff Queisser (cherry picked from commit a39f215ec8c7a61c71d41e0ed1bf4309fb57f0a7 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 11 ++ drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 117 +++++++++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 14 ++ drivers/gpu/drm/asahi/g17/initdata.rs | 2 - drivers/gpu/drm/asahi/g17/initdata/m5.rs | 21 +- drivers/gpu/drm/asahi/g17/mod.rs | 201 ++++++++++++++++++-- drivers/gpu/drm/asahi/hw/t6040.rs | 1 + drivers/gpu/drm/asahi/hw/t6041.rs | 1 + drivers/gpu/drm/asahi/hw/t6050.rs | 73 ++++++- drivers/gpu/drm/asahi/mmu.rs | 25 ++- 11 files changed, 435 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 33a198194e5164..458ac0933e4294 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -439,11 +439,12 @@ impl AsahiPlatformDriver { fn probe_m5<'bound>( pdev: &'bound platform::Device>, ) -> Result> { - // Both boards bind gpu,t6050. Let the ID registers choose the configuration. + // Every T6050 board binds gpu,t6050. Let the ID registers choose the configuration. let res = regs::G17Resources::new(pdev)?; let id = res.get_g17_id()?; let cfg = hw::t6050::config(&id).ok_or_else(|| { - dev_err!(pdev.as_ref(), "No M5 configuration for GPU variant {:#x}\n", id.variant); + dev_err!(pdev.as_ref(), "No M5 configuration for GPU variant {:#x} with {} dies\n", + id.variant, id.num_dies); ENODEV })?; if let g17::FirmwareProfile::M5NoBareMetalTables = cfg.profile { diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 91b3a2f58334c3..819427d8af58f4 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -209,6 +209,17 @@ impl OperandState { r0a5c9: 0x3c000, size: 0x80000, }; + /// Eight clusters of ten cores (M5 Ultra): bank stride 0x78400, tails 0x4000. The size is + /// the M5 compute queues' operand buffer, the tails' end rounded up to 16 KiB. + pub(crate) const EIGHT_BY_TEN: OperandState = OperandState { + r10229: 0xf0800, + r140a8: 0xf4800, + r10099: 0x78405, + r10091: 0xf0400, + r0a5c1: 0x0005, + r0a5c9: 0x78000, + size: 0xf8000, + }; } /// Checkpoints into the main register array, as entry counts: after the diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 91dcd33ddfb6c9..f2766b0fe5c3c9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1493,6 +1493,17 @@ const J716C_AUX_PERF_STATES: [M5AuxPerfState; 14] = one_lane([ /// lanes. A one-lane board leaves the second lane 0. pub(crate) type M5AuxPerfState = (u32, [u32; 2], [u32; 2]); +// J775D ADT afr-perf-states / cs-perf-states: one lane per die, mV; SRAM floor 775 mV. +const J775D_AUX_PERF_STATES: [M5AuxPerfState; 14] = [ + (24, [125, 125], [775, 775]), (338, [600, 600], [775, 775]), + (486, [660, 655], [775, 775]), (636, [670, 665], [775, 775]), + (796, [710, 705], [775, 775]), (888, [720, 720], [775, 775]), + (988, [755, 765], [775, 775]), (1084, [800, 795], [800, 795]), + (1182, [810, 820], [810, 820]), (1278, [850, 860], [850, 860]), + (1374, [905, 915], [905, 915]), (1470, [945, 955], [945, 955]), + (1578, [1030, 1040], [1030, 1040]), (1620, [1075, 1085], [1075, 1085]), +]; + /// One-lane auxiliary tables. const fn one_lane(states: [(u32, u32, u32); 14]) -> [M5AuxPerfState; 14] { let mut out = [(0, [0; 2], [0; 2]); 14]; @@ -1606,6 +1617,16 @@ const J716C_AON_RECORDS: [[u32; 3]; 3] = [ [0x7230_5452, 0x00b8_0001, 0x0000_1401], ]; +// J775D: both dies' records, g0TR-g3TR and r0TR-r1TR. +const J775D_AON_RECORDS: [[u32; 3]; 6] = [ + [0x6730_5452, 0x019f_0001, 0x0000_1401], + [0x6731_5452, 0x01bb_0001, 0x0000_1401], + [0x6732_5452, 0x00a7_0001, 0x0000_1401], + [0x6733_5452, 0x00c3_0001, 0x0000_1401], + [0x7230_5452, 0x0113_0001, 0x0000_1401], + [0x7231_5452, 0x001b_0001, 0x0000_1401], +]; + pub(crate) fn encode_common26_aon_ptd(board: &Board, out: &mut [u8]) -> Result<(), InitdataError> { if out.len() != M5_AON_PTD_SIZE || board.aon_records.len() * 12 > out.len() { return Err(InitdataError::BufferSize); @@ -1756,8 +1777,8 @@ pub(crate) struct Board { /// The leakage calibration words [`Board::controller_defaults`] are valid for, or `None` /// where they hold for every part of the board. pub(crate) calibration: Option<[u32; 3]>, - /// The policy controllers' configuration. - pub(crate) controller_defaults: fn(&mut [u8]) -> Result<(), InitdataError>, + /// The policy controllers' configuration, where the board has one. + pub(crate) controller_defaults: Option Result<(), InitdataError>>, } /// M5 Pro: two clusters of ten cores. @@ -1774,7 +1795,7 @@ pub(crate) const J714: Board = Board { min_parameter_buffer: 0x0020_0000, render_pool: (0x40_0000, 1 << 32), calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), - controller_defaults: encode_m5_policy_controller_defaults, + controller_defaults: Some(encode_m5_policy_controller_defaults), }; /// M5 Max: four clusters of ten cores. @@ -1792,7 +1813,28 @@ pub(crate) const J716C: Board = Board { render_pool: (0x80_0000, 1 << 33), // J714's controller configuration, for a part with J714's calibration only. calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), - controller_defaults: encode_m5_policy_controller_defaults, + controller_defaults: Some(encode_m5_policy_controller_defaults), +}; + +/// M5 Ultra: two dies of four clusters of ten cores (G17D). Values from the J775D ADT. +pub(crate) const J775D: Board = Board { + clusters: 8, + gpu_cores: 80, + // Firmware revision 2 for raw revision 0x11. + core_revision: (0x1a, 2), + perf_states: &J775D_GPU_PERF_STATES, + // One voltage table per cluster. + perf_tables: 8, + aux_perf_states: &J775D_AUX_PERF_STATES, + relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES), + aon_records: &J775D_AON_RECORDS, + register_windows: &J775D_REGISTER_WINDOWS, + // The minimum parameter buffer for eight clusters. + min_parameter_buffer: 0x0050_0000, + render_pool: (0x40_0000, 1 << 32), + calibration: None, + // No controller configuration. + controller_defaults: None, }; const _: () = { @@ -1899,6 +1941,26 @@ pub(crate) const J716C_GPU_PERF_STATES: [M5PerfState; 14] = four_tables([ (1620, [1015, 1015, 1015, 1015], [1015, 1015, 1015, 1015]), ]); +// J775D ADT sgx perf-states (one table per cluster) and perf-states-sram, mV. +pub(crate) const J775D_GPU_PERF_STATES: [M5PerfState; 14] = [ + (0, [125, 125, 125, 125, 125, 125, 125, 125], [775, 775, 775, 775, 775, 775, 775, 775]), + (338, [615, 615, 615, 615, 615, 610, 610, 615], [775, 775, 775, 775, 775, 775, 775, 775]), + (486, [670, 655, 655, 670, 650, 665, 665, 650], [775, 775, 775, 775, 775, 775, 775, 775]), + (636, [695, 690, 690, 695, 690, 690, 690, 690], [775, 775, 775, 775, 775, 775, 775, 775]), + (796, [760, 745, 745, 760, 745, 740, 740, 745], [775, 775, 775, 775, 775, 775, 775, 775]), + (888, [770, 755, 755, 770, 750, 750, 750, 750], [775, 775, 775, 775, 775, 775, 775, 775]), + (988, [795, 780, 780, 795, 775, 775, 775, 775], [795, 780, 780, 795, 775, 775, 775, 775]), + (1084, [805, 805, 805, 805, 815, 800, 800, 815], [805, 805, 805, 805, 815, 800, 800, 815]), + (1182, [815, 810, 810, 815, 825, 820, 820, 825], [815, 810, 810, 815, 825, 820, 820, 825]), + (1278, [860, 845, 845, 860, 870, 855, 855, 870], [860, 845, 845, 860, 870, 855, 855, 870]), + (1374, [885, 885, 885, 885, 895, 895, 895, 895], [885, 885, 885, 885, 895, 895, 895, 895]), + (1470, [930, 930, 930, 930, 940, 935, 935, 940], [930, 930, 930, 930, 940, 935, 935, 940]), + (1578, [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030], + [1020, 1005, 1005, 1020, 1030, 1015, 1015, 1030]), + (1620, [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065], + [1045, 1040, 1040, 1045, 1065, 1050, 1050, 1065]), +]; + pub(crate) const J714_GPU_BASE: u64 = 0x23_0000_0000; #[derive(Copy, Clone)] @@ -1993,6 +2055,51 @@ const J716C_REGISTER_WINDOWS: [M5RegisterWindowSpec; 30] = { windows }; +/// Second die of J775D. +const D: u64 = 0x40_0000_0000; + +/// J775D: the explicit instances of both dies (ADT), in device-VA order. +const J775D_REGISTER_WINDOWS: [M5RegisterWindowSpec; 32] = [ + jw(0, &[0x2_8041_4000], 0x4000, 0, 2), + // One per active memory cache controller. + jw(3, &[0x22_2010_4000, 0x22_2210_4000, 0x22_2410_4000, 0x22_2610_4000, + 0x22_2810_4000, 0x22_2a10_4000, 0x22_2c10_4000, 0x22_2e10_4000], 0x18000, 0, 2), + jw(4, &[0x2_8046_8000], 0x4000, 0, 0), + jw(9, &[0x2_8425_0000], 0x4000, 0, 2), + jw(10, &[0x2_8424_0000], 0x4000, 0, 0), + jw(11, &[0x2_8425_8000], 0x4000, 0, 2), + jw(12, &[0x5_0965_c000], 0x4000, 0, 2), + jw(13, &[0x45_0965_c000], 0x4000, 0, 2), + jw(14, &[0x23_8030_0000], 0x8000, 0, 0), + jw(15, &[0x2_8cdc_0000], 0x24000, 0, 2), + jw(17, &[G], 0x21500, 0, 2), + jw(18, &[G + 0xeb_4000, G + 0xeb_4000 + D], 0x4000, 0, 2), + jw(20, &[G + 0x300_0000, G + 0x320_0000, G + 0x340_0000, G + 0x360_0000, + G + 0x380_0000, G + 0x3a0_0000, G + 0x3c0_0000, G + 0x3e0_0000, + 0x23_035a_8000, 0x23_037a_8000, 0x23_03da_8000, 0x23_03fa_8000], 0x40000, 0, 2), + jw(22, &[G + 0x100_0000], 0x8000, 0, 2), + jw(23, &[G + 0xf0_0000, G + 0xf0_0000 + D], 0x10000, 0, 2), + jw(24, &[G + 0xec_0000, G + 0xed_0000, G + 0xee_0000, G + 0xef_0000, + G + 0xec_0000 + D, G + 0xed_0000 + D, G + 0xee_0000 + D, G + 0xef_0000 + D], + 0x10000, 0, 2), + jw(31, &[G + 0xd0_4000], 0x8000, 0xd0_4000, 2), + jw(33, &[G + 0xd5_0000], 0x10000, 0xd5_0000, 2), + jw(34, &[G + 0xd1_0000], 0x4000, 0xd1_0000, 2), + jw(36, &[G + 0xd4_0000], 0x4000, 0xd4_0000, 2), + jw(37, &[G + 0xd6_0000], 0x20000, 0xd6_0000, 2), + jw(38, &[G + 0xd4_4000], 0x4000, 0xd4_4000, 2), + jw(39, &[G + 0xd4_c000], 0x200, 0xd4_c000, 2), + jw(40, &[G + 0xe0_0000], 0x4000, 0xe0_0000, 2), + jw(42, &[G + 0xe4_0000], 0x4000, 0xe4_0000, 2), + jw(44, &[G + 0xe0_8000, G + 0xe1_0000], 0x8000, 0, 2), + jw(46, &[G + 0xe3_3800], 0x4000, 0, 2), + jw(48, &[G + 0xe6_0000], 0x58, 0xe6_0000, 2), + jw(52, &[G + 0xe4_8000], 0x8000, 0, 2), + jw(53, &[G + 0x2_3d00], 0x200, 0x2_3d00, 2), + jw(58, &[G + 0xe7_3800], 0x4000, 0, 2), + jw(59, &[0x23_0358_0000, 0x23_0378_0000, 0x23_03d8_0000, 0x23_03f8_0000], 0x60000, 0, 2), +]; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct M5RegisterWindow { pub(crate) slot: usize, @@ -2004,7 +2111,7 @@ pub(crate) struct M5RegisterWindow { } /// The most register windows of any board. -pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J714_REGISTER_WINDOWS.len(); +pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J775D_REGISTER_WINDOWS.len(); impl Board { /// The board's `index`th register window. diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 3119fb2a155ba4..83fd6515a1bfca 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -80,6 +80,10 @@ pub(crate) struct Variant { /// Command validation applies the M5 command set: resource commands, the M5 render and /// compute flags, 8x MSAA and the M5 tile-buffer limit. pub(crate) m5_commands: bool, + /// The host holds the cores powered around a render pair's registration and its first + /// work. The bare-metal M5 firmware powers the cores for work by its own policy: under a + /// hold, the M5 Ultra's core-power witness reads unpowered until the hold is released. + pub(crate) registration_power_hold: bool, } /// A firmware build a SoC supports, identified by its loaded TEXT. @@ -125,6 +129,7 @@ pub(crate) const V26A428: Variant = Variant { opening_record: true, split_firmware_area: false, m5_commands: false, + registration_power_hold: true, }; /// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). @@ -157,6 +162,15 @@ pub(crate) const V25G83: Variant = Variant { opening_record: true, split_firmware_area: false, m5_commands: false, + registration_power_hold: true, +}; + +/// The 26A428 firmware of the M5 GPUs, which runs the M5 protocol (the M5 graph, queues and +/// command set). +pub(crate) const V26A428_M5: Variant = Variant { + m5_commands: true, + registration_power_hold: false, + ..V26A428 }; pub(crate) const V25F84: Variant = Variant { diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6894277a3d416f..ad56900418b5ea 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -17,8 +17,6 @@ use kernel::{c_str, prelude::*, sync::Arc}; use t8140::{kernel_window as high, low}; mod v25g83; -// The M5 graph is built only once an M5 board has a page-table boot path. -#[allow(dead_code)] mod m5; mod work_pair; pub(crate) mod outer_credit; diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 37ba520b222fdb..e97cc01947b154 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -21,7 +21,23 @@ const COMPLETION_STRIDE: u64 = 0x2_8000; const PB: u64 = BASE + 0x15e_0000; const UMA: u64 = BASE + 0x15e_8000; +/// Lower-root entry 1 of the M5 layout (Mesa's driver window), linked before the firmware starts +/// as the bare-metal J775D boot has always run. +const BOOT_LINK: u64 = 0x10_0008_0000; + impl InitData { + /// The kernel lower-root ranges a bare-metal M5 firmware reaches, whose top-level links must + /// exist before it starts: entry 1, the descriptor tables and completion rings (entry 7) and + /// the first page of the canonical arena (`canonical`, entry 8), where queues go later. + pub(in crate::g17) fn m5_lower_links(canonical: core::ops::Range) + -> [core::ops::Range; 8] + { + let page = |address: u64| address..address + mmu::UAT_PGSZ as u64; + let ring = |index: u64| page(COMPLETION_LOW + index * COMPLETION_STRIDE); + [page(BOOT_LINK), page(m::REGION_VIEW_LOW_ADDRS[0]), page(m::REGION_VIEW_LOW_ADDRS[1]), + ring(0), ring(1), ring(2), ring(3), page(canonical.start)] + } + /// Diagnostic-only read of an aligned word within the existing policy allocation. /// Callers hold the device state mutex, retaining this mapping for the entire sample. pub(crate) fn m5_policy_telemetry_word(&self, offset: usize) -> Result { @@ -167,9 +183,8 @@ impl InitData { calibration.as_ref().is_ok_and(|words| words.as_slice() == expected) } }; - if calibrated { - (board.controller_defaults)(self.object_mut(self.bundle)?.bytes_mut()) - .map_err(|_| EINVAL)?; + if let Some(defaults) = board.controller_defaults.filter(|_| calibrated) { + defaults(self.object_mut(self.bundle)?.bytes_mut()).map_err(|_| EINVAL)?; let maximum = self.object(self.bundle)?.word(shared.policy + m::M5_POLICY_MAXIMUM_POWER)? .load(Ordering::Relaxed); m::set_m5_region_maximum_power(self.object_mut(self.region_c)?.bytes_mut(), maximum) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 4815888a25a2c4..caca2dfc29b207 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -169,6 +169,9 @@ pub(crate) enum FirmwareProfile { PublishedRoots, /// An M5 board without bare-metal GPU tables yet: probe refuses it. M5NoBareMetalTables, + /// The M5 firmware on bare metal: both roles publish their kernel roots + /// ([`mmu::UatFirmware::PublishedRoots`]) and the driver owns the page tables. + M5PublishedRoots, } impl Config { @@ -191,7 +194,7 @@ impl Config { impl FirmwareProfile { /// The M5 firmware's protocol (26A428 InitData graph, queues and submission). pub(crate) const fn is_m5(self) -> bool { - matches!(self, Self::M5NoBareMetalTables) + matches!(self, Self::M5NoBareMetalTables | Self::M5PublishedRoots) } } @@ -743,10 +746,16 @@ impl Firmware { /// Holds idle descent while a fresh physical render pair is registered. fn acquire_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } power::acquire(self) } fn release_registration_power(&mut self) -> Result { + if !self.variant.registration_power_hold { + return Ok(()); + } // An idle hold keeps the cores powered past registration; it releases them itself. if self.hold.active() { self.registration_power_release_pending = false; @@ -1392,9 +1401,18 @@ impl Gpu { let m5 = cfg.profile.is_m5(); if m5 { let board = cfg.m5.ok_or(EINVAL)?; - if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores { - dev_err!(dev.as_ref(), "M5 board expects {} clusters and {} cores, the GPU reports {} x {}\n", - board.clusters, board.gpu_cores, id.num_clusters, id.num_cores); + // The M5 layouts are sized for every core slot of the board, fused-off cores + // included, so the GPU is matched on clusters and slots, not on enabled cores. + if id.num_clusters != board.clusters || id.num_clusters * id.num_cores != board.gpu_cores + { + dev_err!( + dev.as_ref(), + "M5 board expects {} clusters of {} core slots, the GPU reports {} x {}\n", + board.clusters, + board.gpu_cores / board.clusters, + id.num_clusters, + id.num_cores + ); return Err(ENODEV); } fw::m5::pool::select_render_pool(board); @@ -1535,7 +1553,7 @@ impl Gpu { }, GFP_KERNEL, )?; - let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.id, &gpu.shared)?; gpu.shared .port .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); @@ -1616,17 +1634,169 @@ impl Gpu { Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) } - /// Starts the M5 firmware. It runs only on page tables the driver owns, which no M5 board - /// has yet. + /// Starts the M5 firmware on bare metal: both roles boot and publish their kernel roots, + /// the M5 graph is built in the kernel window they share, then the M5 preparation, + /// InitData to the primary and then the secondary, POWER_ON_DONE to both, and both opening + /// records retired. fn boot_m5( - _pdev: &platform::Device>, - _dev: &AsahiDrmDevice, - _cfg: &'static Config, - _regs: regs::G17Resources, - _geometry: mmu::UatGeometry, - _shared: &Arc, + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + cfg: &'static Config, + regs: regs::G17Resources, + geometry: mmu::UatGeometry, + id: ®s::G17Id, + shared: &Arc, ) -> Result> { - Err(ENODEV) + if cfg.profile != FirmwareProfile::M5PublishedRoots { + return Err(ENODEV); + } + let board = cfg.m5.ok_or(EINVAL)?; + // Names the step a failed bootstrap stopped at. + let at = |step: &'static str| { + move |e: &Error| { + dev_err!(dev.as_ref(), "M5 bare-metal bootstrap failed at {}: {:?}\n", step, e) + } + }; + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + cfg.uat_quirks, + ) + .inspect_err(at("UAT"))? + .with_m5(), + GFP_KERNEL, + )?; + // The firmware samples the kernel lower root's top-level entries only when it starts: + // link the tables of every lower range the M5 graph and its queues use first. + uat.prepare_lower_tables(&initdata::InitData::m5_lower_links( + uat.kernel_lower_dynamic_range(), + )) + .inspect_err(at("lower links"))?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?; + let render_global = uat.new_vm(0, geometry.user_range()).inspect_err(at("render VM"))?; + let metrics_ids = Arc::new( + buffer::MetricsIds::with_stride(64).inspect_err(at("metrics IDs"))?, + GFP_KERNEL, + )?; + let metrics = Arc::new( + KernelObject::new( + dev, + &render_global, + Placement::In(buffer::BlockGeometry::M5_COMPACT, mmu::UAT_PGSZ as u64), + metrics_ids.storage_bytes()?, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + .inspect_err(at("metrics"))?, + GFP_KERNEL, + )?; + let session_metrics = metrics.clone(); + let firmware = KBox::try_init( + kernel::try_init!(Firmware { + secondary: Some(secondary), + primary, + bridge: Bridge::unused(regs), + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, + seen_submissions: 0, + rings: channel::Rings::new(shared.variant.graph)?, + recovery: recovery::State::new(), + effort: power::Effort::new(), + hold: power::Hold::new(), + governor: thermal::Governor::new(), + registration_power_release_pending: false, + variant: shared.variant, + queues <- runtime::Registry::with_waiters( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), + ), + pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, + render_ids: freelist::RenderIds::new()?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: session_metrics, + uat, + }), + GFP_KERNEL, + ) + .inspect_err(at("session"))?; + // Once a role runs, a stopped CPU is not proof that no engine still reads what it was + // given: a failed bootstrap keeps the session, nothing is freed or unmapped. + let mut retained = core::mem::ManuallyDrop::new(firmware); + let session = &mut **retained; + // M5 renders take 64-byte metrics records. + session.queues.metrics = metrics_ids; + session.primary.boot().inspect_err(at("primary boot"))?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.boot().inspect_err(at("secondary boot"))?; + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = session + .uat + .adopt_firmware_root_entries() + .inspect_err(at("root adoption"))?; + session + .init + .build_m5(dev, &session.uat, id, board) + .inspect_err(at("M5 graph"))?; + session + .init + .bind_m5_metrics(&session.uat, &metrics) + .inspect_err(at("metrics binding"))?; + // Publishers reach the mailbox, QoS table and recovery word through the port, as on + // G17P; M5's rings are the shared ones its InitData views name. + let mailbox: *const Mailbox = &*session.primary.rtkit; + session + .port + .attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + shared.variant.registration, + ) + .inspect_err(at("port"))?; + session + .uat + .confirm_firmware_root_entries(root_entries) + .inspect_err(at("root confirmation"))?; + session.uat.mirror_secondary_root().inspect_err(at("secondary root"))?; + session + .bridge + .regs + .prepare_m5_firmware() + .inspect_err(at("M5 preparation"))?; + let ack = shared.variant.init_ack; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack) + .inspect_err(at("primary InitData"))?; + dev_info!(dev.as_ref(), "M5 primary initdata acknowledged\n"); + let root_va = session.init.root_va(Role::Secondary)?; + let secondary = session.secondary.as_mut().ok_or(EINVAL)?; + secondary.init(root_va, ack).inspect_err(at("secondary InitData"))?; + dev_info!(dev.as_ref(), "M5 secondary initdata acknowledged\n"); + session.primary.notify(MSG_POWER_ON_DONE)?; + secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref()) + .inspect_err(at("opening records"))?; + dev_info!(dev.as_ref(), "M5 firmware opening records retired\n"); + Ok(core::mem::ManuallyDrop::into_inner(retained)) } /// Starts both coprocessors and initializes their firmware. @@ -1636,10 +1806,11 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, geometry: mmu::UatGeometry, + id: ®s::G17Id, shared: &Arc, ) -> Result> { if cfg.profile.is_m5() { - return Self::boot_m5(pdev, dev, cfg, regs, geometry, shared); + return Self::boot_m5(pdev, dev, cfg, regs, geometry, id, shared); } let uat = Arc::new( mmu::Uat::new( diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 9674a4bd466654..337adb07ddf840 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -50,6 +50,7 @@ pub(crate) const CONFIG: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, region_c: Some(®ION_C), diff --git a/drivers/gpu/drm/asahi/hw/t6041.rs b/drivers/gpu/drm/asahi/hw/t6041.rs index 439751c3c5dd5e..0c949ff11caee2 100644 --- a/drivers/gpu/drm/asahi/hw/t6041.rs +++ b/drivers/gpu/drm/asahi/hw/t6041.rs @@ -39,6 +39,7 @@ pub(crate) const CONFIG: Config = Config { page_tlbi: true, handoff_lock: false, handoff_slot: 0xffff_ffff, + handoff_unk2: 0, native_gpu_permissions: true, }, region_c: Some(®ION_C), diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 8f53ff3df4e056..b3257a7f18437f 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! M5 Pro and M5 Max GPU address-space configuration. +//! M5 Pro, M5 Max and M5 Ultra GPU firmware and address-space configuration. //! //! The M5 firmware builds its HW data, performance tables and region C in its own graph //! ([`crate::g17::fw::m5::initdata`]), so the 26A428/25G83 graph tables of [`Config`] are empty. @@ -46,9 +46,10 @@ pub(crate) const CONFIG: Config = Config { gpu_generation: GpuGen::G17, // No loaded image is admitted: the board has no bare-metal page tables yet. firmware: crate::g17::fw::variant::Firmware::Identified(&[]), - // Two clusters of ten cores. + // Two clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. core_layout: CoreLayout { - cores: Some((2, 20)), + cores: None, alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::TWO_BY_TEN, deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, @@ -94,9 +95,10 @@ pub(crate) const CONFIG_MAX: Config = Config { gpu_generation: GpuGen::G17, // No loaded image is admitted: the board has no bare-metal page tables yet. firmware: crate::g17::fw::variant::Firmware::Identified(&[]), - // Four clusters of ten cores. + // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the + // board's cluster and slot counts admit the GPU, whichever cores are fused off. core_layout: CoreLayout { - cores: Some((4, 40)), + cores: None, alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::FOUR_BY_TEN, deflake: crate::g17::fw::render::Deflake::FOUR_CLUSTERS, @@ -104,8 +106,65 @@ pub(crate) const CONFIG_MAX: Config = Config { }, }; -/// Both boards bind `gpu,t6050`; the ID registers select the configuration. +/// M5 Ultra (J775D, two dies, G17D) on bare metal: both roles publish their kernel roots, the +/// secondary's 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. The +/// UAT tables are no-map reservations outside /memory; the rest of the UAT policy is the one +/// this board has run with: every table change written back, page-by-page TLB invalidation, and +/// the handoff values its firmware expects (current slot 0xffff, the byte after the flush records +/// 1, the lock never taken). +pub(crate) const CONFIG_ULTRA: Config = Config { + chip_id: 0x6050, + // Family 0xb (G17), variant 6 (G17D), revision 0x11. + identity: Identity { + family: 0xb, + variant: 6, + num_dies: 2, + gpu_variant: GpuVariant::D, + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, + pm_metrics: None, + windows: &[], + reserved_slots: &[], + perf: None, + opps: 0, + uat_quirks: crate::mmu::UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff, + handoff_unk2: 1, + native_gpu_permissions: false, + }, + region_c: None, + chip: None, + m5: Some(&crate::g17::fw::m5::initdata::J775D), + adopt_root_entries: true, + fabric_bridge: false, + recovery_slot_keys: None, + gpu_generation: GpuGen::G17, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), + // Eight clusters of ten cores. M5 renders lay out their tiler context store themselves + // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the + // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: + // parts with fused-off cores (64 enabled) take the same layout. + core_layout: CoreLayout { + cores: None, + alternate_core_masks: None, + operand_state: crate::g17::fw::compute::OperandState::EIGHT_BY_TEN, + deflake: crate::g17::fw::render::Deflake::TWO_CLUSTERS, + compute_blocks: crate::g17::fw::freelist::COMPUTE_BLOCKS, + }, +}; + +/// Every T6050 board binds the same compatible; the ID registers select the configuration. pub(crate) fn config(id: &G17Id) -> Option<&'static Config> { - [&CONFIG, &CONFIG_MAX].into_iter().find(|cfg| id.family == cfg.identity.family + [&CONFIG, &CONFIG_MAX, &CONFIG_ULTRA].into_iter().find(|cfg| id.family == cfg.identity.family && id.variant == cfg.identity.variant && id.num_dies == cfg.identity.num_dies) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index a2820df7d314cc..1949a0d88c7613 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -194,6 +194,9 @@ pub(crate) struct UatQuirks { pub(crate) handoff_lock: bool, /// Handoff "current slot" written at initialization. pub(crate) handoff_slot: u32, + /// Handoff byte after the flush records written at initialization. It is 1 on T6050, where + /// it also turns the legacy flush protocol off. + pub(crate) handoff_unk2: u8, /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. pub(crate) native_gpu_permissions: bool, @@ -207,6 +210,7 @@ impl UatQuirks { page_tlbi: false, handoff_lock: true, handoff_slot: 0, + handoff_unk2: 0, native_gpu_permissions: false, }; } @@ -2041,9 +2045,10 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32, unk2: u8) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(slot, Ordering::Relaxed); + self.unk2.store(unk2, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -3348,6 +3353,16 @@ impl Uat { &self.kernel_lower_vm } + /// Links the kernel lower root's tables for each range, with no mappings. A published-roots + /// firmware samples that root's top-level entries only when it starts. + pub(crate) fn prepare_lower_tables(&self, ranges: &[Range]) -> Result { + let mut inner = lock_vm(&self.kernel_lower_vm.gpuvm, None, false)?; + for range in ranges { + inner.page_table.alloc_pages(range.clone())?; + } + Ok(()) + } + /// Installed descriptor and bind contexts rooted in this user VM. A graph /// constructor checks this before changing a retained pool's mappings. pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { @@ -3587,6 +3602,12 @@ impl Uat { self.m5 } + /// Marks a published-roots UAT as an M5 GPU's ([`Uat::is_m5`]). + pub(crate) fn with_m5(mut self) -> Self { + self.m5 = true; + self + } + /// Keep dynamic objects outside the M5 boot graph, its unmapped sentinel, /// and the firmware register aperture. Client mappings have their own VA. pub(crate) fn kernel_dynamic_range(&self) -> Range { @@ -3877,7 +3898,7 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; + inner.handoff().init(firmware, quirks.handoff_slot, quirks.handoff_unk2)?; if quirks.clean_tables { pgtable::clean_poc( core::ptr::from_ref(inner.handoff()).cast::(), From 874cc5b1510b771f51236d5a832764506cc7e717 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:12:10 -0800 Subject: [PATCH 083/171] drm/asahi: t6050: Refuse unknown M5 Ultra GPU firmware before use On the M5 Ultra Mac Studio (J775D) the driver would write its firmware tables to whatever GPU firmware the boot loader left loaded, unchecked. It now drives only the known build, and any other image fails the GPU probe before a single GPU write, with the rest of boot unaffected. On bare metal nothing vouches for the firmware the boot loader left in fw-text: the driver would write the 26A428 firmware's InitData graph to whatever was loaded. J775D now names the build it admits (Firmware::Identified), so Gpu::new identifies the loaded TEXT among its images before anything is written to the GPU, as for the other identified G17 SoCs. fw-text holds the primary's TEXT then the secondary's, each with its own pair of entropy records, and the boot loader refills eight ranges per boot (two sections and two header load fields per role). The build zeroes all of them, so the live image and the firmware file hash alike (c35354407602...ce9cc8251). A mismatch logs the measured identity and the probe fails before any GPU write; the rest of boot continues. A digest pinned per boot would refuse the genuine firmware on the next boot. Signed-off-by: Jeff Queisser (cherry picked from commit 1f265079d18ae8a9d41b02dd1621e475f353ef76 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/fwid.rs | 38 +++++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/hw/t6050.rs | 29 ++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index c5ee5c1bec2bc6..ff82f760d2f976 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -492,4 +492,42 @@ mod tests { } assert!(measured(builds, 0x13c000, &[[0x5c8df, 0x5c9c2]], [0; 32]).is_none()); } + + #[test] + fn t6050_admits_only_its_pinned_image() { + let crate::g17::fw::variant::Firmware::Identified(builds) = + crate::hw::t6050::CONFIG_ULTRA.firmware + else { + panic!("J775D on bare metal must identify its loaded firmware"); + }; + for image in builds.iter().flat_map(|build| build.images) { + // Both roles' records, inside TEXT. + assert_eq!(image.entropy.len(), 2); + assert!(image + .entropy + .iter() + .flatten() + .all(|&o| u64::from(o) + 16 <= image.text_size)); + let admit = |text, data, at: &[[u32; 2]], digest| { + lookup(core::slice::from_ref(image), text, data, |m| { + (m.entropy == at).then_some(digest) + }) + }; + // The header fields and both boot-filled sections of each role. + assert_eq!(image.boot_filled.len(), 8); + let (t, d, e, h) = ( + image.text_size, + image.data_size, + image.entropy, + image.text_sha256, + ); + assert!(core::ptr::eq(admit(t, d, e, h).unwrap(), image)); + let mut wrong = h; + wrong[31] ^= 1; + assert!(admit(t, d, e, wrong).is_none()); + assert!(admit(t, d + 0x4000, e, h).is_none()); + assert!(admit(t + 0x4000, d, e, h).is_none()); + assert!(admit(t, d, &e[..1], h).is_none()); + } + } } diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index b3257a7f18437f..3fc02fbc8efdb7 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -149,7 +149,34 @@ pub(crate) const CONFIG_ULTRA: Config = Config { fabric_bridge: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, - firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428_M5), + // On bare metal nothing vouches for the firmware the boot loader left in `fw-text`: it + // must be this build, or the GPU is refused before anything is written to it. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428)", + images: &[crate::fwid::FirmwareBuild { + // The boot loader loads both roles' TEXT back to back into `fw-text`: the primary's + // 0x50000, then the secondary's 0x58000, each with its own GKTS/ECAP entropy + // records. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed, so a loaded copy normalizes to it on every boot. + name: "26A428 for T6050, primary then secondary", + text_size: 0xa8000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0xa1918, 0xa19fb]], + // Per role: two load fields of the image header, then the two sections the boot + // loader fills with the boot arguments (the entropy included). + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0xa11d8, 0x740), (0xa1918, 0x231), + ], + text_sha256: [ + 0xc3, 0x53, 0x54, 0x40, 0x76, 0x02, 0x48, 0x44, 0x04, 0xaa, 0x6f, 0xc9, 0x46, + 0xcd, 0xd2, 0x4c, 0xb8, 0x2c, 0x64, 0xa8, 0xbb, 0xd1, 0x07, 0xa4, 0x94, 0x79, + 0xa0, 0x8c, 0xe9, 0xcc, 0x82, 0x51, + ], + }], + variant: &crate::g17::fw::variant::V26A428_M5, + }]), // Eight clusters of ten cores. M5 renders lay out their tiler context store themselves // (`fw::m5::render_registers`), so the deflake layout is the M5 Pro's. As on the M5 Pro, the // buffers are sized for all 80 core slots and the board admits the GPU by clusters and slots: From aeb61e27d8a7f2b60e3a7261724dfbd1aff8c339 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 22:59:25 -0800 Subject: [PATCH 084/171] rust: drm: gem: shmem: Fix memory corruption on a foreign dma-buf import A Rust DRM driver without core_object_args() read and wrote past the end of objects the DRM core created for it, such as a dma-buf imported from another device; on the M5 Ultra Mac Studio (J775D) this bug in an earlier GPU driver tree panicked the kernel and kept the desktop from starting. Such a driver (tyr, in this tree) now refuses the import with EINVAL; asahi has core_object_args() and is not affected. Handle lookups and every Object accessor assume that each GEM object of the device is a whole Object. The DRM core creates imported and dumb objects through gem_create_object, which the shmem abstraction installs only for drivers with core_object_args(). For any other driver the core allocates a bare drm_gem_shmem_object instead, and the Rust side then reads and writes past its end. Commit 2dc22a191a5c ("rust: drm: gem: shmem: let drivers allocate the objects the core creates") lets drivers with core_object_args() allocate them; this completes it for the others. Install gem_prime_import_sg_table and dumb_create only with gem_create_object. A driver without core_object_args() then refuses an import from another device with -EINVAL instead of corrupting memory. In this tree that is tyr; asahi has core_object_args() and keeps both. Signed-off-by: Jeff Queisser (cherry picked from commit f360e62b51c14fb2a3f65686e93aaa2cfe66d5e0 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- rust/kernel/drm/gem/shmem.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 83eca852139beb..095f798e77d31d 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -604,8 +604,19 @@ impl driver::AllocImpl for Object { prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, - gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), - dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + // The core creates imported and dumb objects through `gem_create_object`. Without it they + // would be bare `drm_gem_shmem_object`s, which every handle lookup still treats as an + // `Object`: offer them only to drivers that can allocate the whole object. + gem_prime_import_sg_table: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_prime_import_sg_table) + } else { + None + }, + dumb_create: if T::HAS_CORE_OBJECT_ARGS { + Some(bindings::drm_gem_shmem_dumb_create) + } else { + None + }, dumb_map_offset: None, }; } From ed4b8770d7577893a9194e1dcc81e166c5b6c79c Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:16:21 -0800 Subject: [PATCH 085/171] drm/asahi: g17: M5: Fix the M5 Ultra GPU staying at its lowest clock On the M5 Ultra Mac Studio (J775D) the GPU never left its base performance state (338 MHz) under any load, so every GPU workload ran at the lowest clock. The firmware's power controllers now get the Ultra's own power model and ramp the GPU up. The M5 builder installs the policy controllers' configuration only for a board whose leakage calibration words match (J714). J775D has none, so every controller word stayed zero, and the firmware never left its base performance state (P1, 338 MHz) under any load. Give J775D its own arm, as J714 has one: encode_m5_policy_controller_defaults_j775d starts from the J714 defaults and writes the Ultra's power model over them: - the modeled maximum power, 345262 mW, in the policy's maximum and ceiling words, and from there in Region C's host power budget; - the relative power curves of the main and auxiliary domains; - the leakage inputs of eight clusters and two dies; - the controllers' gains and utilization thresholds; - the temperature-sensor channel masks of both dies. The model uses synthetic leakage inputs, valid for every J775D, so the J775D arm needs no calibration words. J714 is unchanged. The J775D arm follows the shape of Erik Bourget's J716c controller defaults (f7f650b887cd, not carried here): the J714 defaults first, then the board's own values over them. Signed-off-by: Jeff Queisser (cherry picked from commit 86eb1558b4125a234f3fda5bccd6de50d09ab647 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 58 ++++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index f2766b0fe5c3c9..9ca7ff6a6b8558 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1832,9 +1832,9 @@ pub(crate) const J775D: Board = Board { // The minimum parameter buffer for eight clusters. min_parameter_buffer: 0x0050_0000, render_pool: (0x40_0000, 1 << 32), + // The power model's synthetic leakage inputs hold for every J775D: no calibration words. calibration: None, - // No controller configuration. - controller_defaults: None, + controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), }; const _: () = { @@ -2686,3 +2686,57 @@ pub(crate) fn encode_m5_policy_controller_defaults(out: &mut [u8]) -> Result<(), for &(offset, value) in INITIAL { put_u32(policy, offset, value); } Ok(()) } + +/// Controller inputs for J775D: the J714 defaults, with the Ultra's power model over them. The +/// model covers the main and auxiliary operating points of both dies (64 cores, 110 C) with +/// synthetic leakage inputs (1350 per cluster core and per die auxiliary domain, 51 per cluster +/// SRAM), which hold for every J775D: P13 sums to 345262 mW. Then the relative power curves of +/// that model, the controllers' gains and utilization thresholds, and the temperature-sensor +/// channel masks of both dies (160 GPU and 8 auxiliary-domain channels; the hotspot set is both). +pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + const RELATIVE_POWER_MAIN: [u32; 14] = [ + 0x0000_0000, 0x4107_2435, 0x414b_7ecd, 0x4187_dfa6, 0x41be_759c, 0x41d6_f4a2, 0x41ff_340b, + 0x4215_4139, 0x4226_cc22, 0x4247_2db8, 0x426b_3fa4, 0x428b_aed3, 0x42b6_c65b, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_MAIN.iter().enumerate() { put_u32(out, 0x19e4 + index * 4, value); } + const RELATIVE_POWER_AUX: [u32; 14] = [ + 0x0000_0000, 0x4106_2d7e, 0x4154_96b9, 0x4189_3d11, 0x41ba_e22e, 0x41d5_33ed, 0x4202_7bcd, + 0x421c_12dd, 0x4230_6915, 0x4250_6a36, 0x427d_e40c, 0x4294_0cc0, 0x42bd_bd8e, 0x42c8_0000, + ]; + for (index, &value) in RELATIVE_POWER_AUX.iter().enumerate() { put_u32(out, 0x1a64 + index * 4, value); } + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 345_262; + const MAXIMUM_POWER_F32: u32 = 0x48a8_95c0; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + // Leakage: eight clusters' cores (1350.0) and SRAM (51.0), then each die's auxiliary domain. + for index in 0..8 { + put_u32(policy, 0x121b0 + index * 4, 0x44a8_c000); + put_u32(policy, 0x12230 + index * 4, 0x424c_0000); + } + for index in 0..2 { + put_u32(policy, 0x12280 + index * 4, 0x44a8_c000); + } + // The power controller's integral and proportional gains (0.08, 0.005), the second + // controller's (0.01, 0.28), and the utilization thresholds 75 and 95. + for (offset, value) in [(0xaa0, 0x3da3_d70b), (0xab0, 0x3ba3_d70a), (0x45b4, 0x3c23_d70b), + (0x45c4, 0x3e8f_5c29), (0xb44, 75), (0xb48, 95)] { + put_u32(policy, offset, value); + } + const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; + for (offset, words) in [ + (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + (0x2cfc, [0x0303_0303, 0, 0, 0, 0]), + (0x4ba4, [0x6fcc_3fcc_0303_0303, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), + ] { + for (index, word) in words.into_iter().enumerate() { + put_u64(policy, offset + index * 8, word); + } + } + Ok(()) +} From 8bc252f164b8d776320de3abc5806244c883cacf Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:17:05 -0800 Subject: [PATCH 086/171] drm/asahi: g17: M5: Fix light work settling at low clocks on the M5 Ultra On the M5 Ultra Mac Studio (J775D) the idle GPU sat at 338 MHz and light passes, ramping from there, settled at the second to fourth performance states. The idle and ramp floor is now the Ultra's own P3. The policy's performance-controller floor came from the J714 defaults: P1, in the policy's three floor words and as the base of HwData's relative-frequency tables. J775D's floor is P3. With P1 the idle GPU sat at 338 MHz, and a busy pass ramped from there in steps of three states, so light passes lost the first steps and settled at P2 to P4. J775D's controller arm now writes its floor (state x 100) into the three floor words, and its relative-frequency tables are based at P3: 0 at and below the floor, 100 at P13. relative_frequency() takes the floor; J714 keeps P1. Signed-off-by: Jeff Queisser (cherry picked from commit c769c2b82acd9d721aa5a4785c4d0836d71188bf in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 9ca7ff6a6b8558..0e87effa89c7f1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1517,13 +1517,15 @@ const fn one_lane(states: [(u32, u32, u32); 14]) -> [M5AuxPerfState; 14] { } // Evaluate the frequency-only curve at compile time; no kernel FPU use. -// Conversion uses Hz before the single-precision multiply and divide. -const fn relative_frequency(states: &[M5PerfState; 14]) -> [u32; 16] { +// Conversion uses Hz before the single-precision multiply and divide. The curve is based at +// `floor`, the lowest state the performance controllers run the GPU at: states at and below it +// read 0. +const fn relative_frequency(states: &[M5PerfState; 14], floor: usize) -> [u32; 16] { let mut values = [0; 16]; - let base = states[1].0 as u64 * 1_000_000; + let base = states[floor].0 as u64 * 1_000_000; let maximum = states[13].0 as u64 * 1_000_000; let denominator = (maximum - base) as f32; - let mut state = 2; + let mut state = floor + 1; while state < 14 { let frequency = states[state].0 as u64 * 1_000_000; values[state] = (((frequency - base) as f32 * 100.0) / denominator).to_bits(); @@ -1789,7 +1791,7 @@ pub(crate) const J714: Board = Board { perf_states: &J714_GPU_PERF_STATES, perf_tables: 2, aux_perf_states: &J714_AUX_PERF_STATES, - relative_frequency: relative_frequency(&J714_GPU_PERF_STATES), + relative_frequency: relative_frequency(&J714_GPU_PERF_STATES, 1), aon_records: &J714_AON_RECORDS, register_windows: &J714_REGISTER_WINDOWS, min_parameter_buffer: 0x0020_0000, @@ -1806,7 +1808,7 @@ pub(crate) const J716C: Board = Board { perf_states: &J716C_GPU_PERF_STATES, perf_tables: 4, aux_perf_states: &J716C_AUX_PERF_STATES, - relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES), + relative_frequency: relative_frequency(&J716C_GPU_PERF_STATES, 1), aon_records: &J716C_AON_RECORDS, register_windows: &J716C_REGISTER_WINDOWS, min_parameter_buffer: 0x0030_0000, @@ -1816,6 +1818,9 @@ pub(crate) const J716C: Board = Board { controller_defaults: Some(encode_m5_policy_controller_defaults), }; +/// J775D's performance controllers run from P3. +const J775D_PERF_FLOOR: usize = 3; + /// M5 Ultra: two dies of four clusters of ten cores (G17D). Values from the J775D ADT. pub(crate) const J775D: Board = Board { clusters: 8, @@ -1826,7 +1831,7 @@ pub(crate) const J775D: Board = Board { // One voltage table per cluster. perf_tables: 8, aux_perf_states: &J775D_AUX_PERF_STATES, - relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES), + relative_frequency: relative_frequency(&J775D_GPU_PERF_STATES, J775D_PERF_FLOOR), aon_records: &J775D_AON_RECORDS, register_windows: &J775D_REGISTER_WINDOWS, // The minimum parameter buffer for eight clusters. @@ -2728,6 +2733,11 @@ pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Resu (0x45c4, 0x3e8f_5c29), (0xb44, 75), (0xb48, 95)] { put_u32(policy, offset, value); } + // The performance controllers' floor, P3, as state x 100 in the three floor words (the last + // byte-packed in the boost block); HwData's relative-frequency tables are based at it too. + for offset in [0xb94, 0xbd4, 0xbf9] { + put_u32(policy, offset, J775D_PERF_FLOOR as u32 * 100); + } const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; for (offset, words) in [ (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), From 5b64c11e72c1748e13786e9bc417ee4353fddc75 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 17:24:57 -0800 Subject: [PATCH 087/171] drm/asahi: Fix refused M5 renders after 255 reuses of a context slot On the M5 Ultra Mac Studio (J775D) renders were refused once a context slot had been reused 255 times: a render-churn test failed at its 258th run. Renders now keep working however often a slot is reused; other GPUs are unchanged. An execution context slot's reuse generation is 8 bits and wrapped from 255 to 0, which M5 reads as "no generation": the 256th reuse of one context slot refused its next render's context token (render churn failed at run 258). On M5 it now wraps from 255 to 1. Other GPUs keep the wrapping generation: the change is gated on the M5 UAT. Signed-off-by: Jeff Queisser (cherry picked from commit d59a8023631bb414c71bdc9e41f4185362c64ef1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 1949a0d88c7613..f67cf7af5715bb 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3544,8 +3544,16 @@ impl Uat { } let id = free.trailing_zeros() as usize; inner.execution_contexts |= 1u64 << id; - inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); - (id, inner.context_generations[id]) + // On M5 a slot's reuse generation is never zero, which M5's context token reserves + // for none: it wraps from 255 to 1. Slots reused past 255 times (reclaimed M5 queues) + // would otherwise hand out zero. + let generation = &mut inner.context_generations[id]; + *generation = if self.m5 { + generation.checked_add(1).unwrap_or(1) + } else { + generation.wrapping_add(1) + }; + (id, *generation) }; let low = tagged_root(vm.ttb(), id); let high = tagged_root(self.kernel_vm.ttb(), id); From e65fea1679ee29937f682250de8493e8661192d9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 17:46:55 -0800 Subject: [PATCH 088/171] drm/asahi: mmu: Fix Vulkan device creation failing on bare-metal M5 GPUs On the M5 Ultra Mac Studio (J775D) every Vulkan application failed at vkCreateDevice: Mesa's sparse-tile bind for its descriptor table was refused with ENOTSUPP on the driver's own page tables. The bind now succeeds and devices are created. DRM_ASAHI_FEATURE_SPARSE_TILES is reported for every M5 firmware profile, but only SPTM-provided tables could map a sparse tile; on a directly owned table every BIND_SPARSE_TILE returned ENOTSUPP, so Mesa could not bind its descriptor table reservation and vkCreateDevice failed. A directly owned table has no per-frame mapping bound. Map the range by repeating the tile's first page, which is zeroed: the read-only fill reads zeros and the writable fill writes to this VM's own sink page, the same semantics the SPTM tables provide. Signed-off-by: Jeff Queisser (cherry picked from commit e04c62f03b4be65e37c8257a74a28ec6274db04c in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu/table.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/table.rs b/drivers/gpu/drm/asahi/mmu/table.rs index 3414c99774bd0e..53d6c30273b5e4 100644 --- a/drivers/gpu/drm/asahi/mmu/table.rs +++ b/drivers/gpu/drm/asahi/mmu/table.rs @@ -4,7 +4,7 @@ use super::{sparse_tile::SparseTile, Prot, UatPageTable}; use core::ops::Range; -use kernel::{prelude::*, sync::Arc}; +use kernel::{addr::PhysicalAddr, prelude::*, sync::Arc}; impl UatPageTable { pub(super) fn visit_runs( @@ -16,8 +16,9 @@ impl UatPageTable { Err(EOPNOTSUPP) } + /// Fill `range` from a sparse tile: its first (zeroed) page repeats across + /// the range, as a SINGLE_PAGE bind repeats its GEM's first page. pub(super) fn map_tiled(&mut self, range: Range, tile: &Arc, prot: Prot) -> Result { - let _ = (range, tile, prot); - Err(EOPNOTSUPP) + self.map_pages(range, tile.physical() as PhysicalAddr, prot, true) } } From 42162ccd5eac0b1ac4acd2cdfb93dcfd1ac4f447 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:17:48 -0800 Subject: [PATCH 089/171] drm/asahi: g17: M5: Give the M5 Ultra GPU its 3500 us idle standby delay On the M5 Ultra (J775D) the idle GPU went to standby after 1500 us, the smaller M5 part's delay, instead of the Ultra's own 3500 us. Region C's idle standby timer, how long the idle GPU waits before it goes to standby, came from the shared M5 Region C image: 1500 us, the value of the smaller M5 part. J775D's is 3500 us. Board::idle_standby_us names the boards whose timer differs, and the builder writes it into Region C (set_m5_region_idle_standby). The rest of the idle configuration is unchanged. J714 keeps 1500 us. Signed-off-by: Jeff Queisser (cherry picked from commit ef5a339878445ec55fbe4a9b99a2c14d56fcf814 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 14 ++++++++++++++ drivers/gpu/drm/asahi/g17/initdata/m5.rs | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 0e87effa89c7f1..66ae15dab7046f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1781,6 +1781,8 @@ pub(crate) struct Board { pub(crate) calibration: Option<[u32; 3]>, /// The policy controllers' configuration, where the board has one. pub(crate) controller_defaults: Option Result<(), InitdataError>>, + /// Region C's idle standby timer where the board's differs from the image's 1500 us, us. + pub(crate) idle_standby_us: Option, } /// M5 Pro: two clusters of ten cores. @@ -1798,6 +1800,7 @@ pub(crate) const J714: Board = Board { render_pool: (0x40_0000, 1 << 32), calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), controller_defaults: Some(encode_m5_policy_controller_defaults), + idle_standby_us: None, }; /// M5 Max: four clusters of ten cores. @@ -1816,6 +1819,7 @@ pub(crate) const J716C: Board = Board { // J714's controller configuration, for a part with J714's calibration only. calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), controller_defaults: Some(encode_m5_policy_controller_defaults), + idle_standby_us: None, }; /// J775D's performance controllers run from P3. @@ -1840,6 +1844,7 @@ pub(crate) const J775D: Board = Board { // The power model's synthetic leakage inputs hold for every J775D: no calibration words. calibration: None, controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), + idle_standby_us: Some(3500), }; const _: () = { @@ -2266,6 +2271,15 @@ pub(crate) fn encode_m5_hw_data( pub(crate) const M5_POLICY_MAXIMUM_POWER: usize = 0xa18; +/// Region C's idle standby timer: how long the idle GPU waits before it goes to standby, us. +pub(crate) fn set_m5_region_idle_standby(out: &mut [u8], microseconds: u32) + -> Result<(), InitdataError> { + if out.len() < M5_REGION_C_ALLOCATION { return Err(InitdataError::BufferSize); } + if microseconds == 0 { return Err(InitdataError::InterfaceMismatch); } + put_u32(out, 0x7c8, microseconds); + Ok(()) +} + /// Apply the calibrated maximum shared by the policy and host power input. pub(crate) fn set_m5_region_maximum_power(out: &mut [u8], maximum: u32) -> Result<(), InitdataError> { diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index e97cc01947b154..d7d28ee3f6fbee 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -190,6 +190,10 @@ impl InitData { m::set_m5_region_maximum_power(self.object_mut(self.region_c)?.bytes_mut(), maximum) .map_err(|_| EINVAL)?; } + if let Some(microseconds) = board.idle_standby_us { + m::set_m5_region_idle_standby(self.object_mut(self.region_c)?.bytes_mut(), microseconds) + .map_err(|_| EINVAL)?; + } for (index, role) in [m::InstanceRole::Primary, m::InstanceRole::Secondary].into_iter().enumerate() { let instance = m::InstanceAddresses { root: ROOTS + index as u64 * 0x8000, From e219492580d0eb4f311c688264843d7f2e52deeb Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:17:50 -0800 Subject: [PATCH 090/171] drm/asahi: g17: M5: Give the M5 Ultra's GPU boost its 20% idle tolerance On the M5 Ultra (J775D) the firmware's fast boost stopped raising the GPU clock once the GPU idled more than 10 percent of a boost window, where the Ultra's tolerance is 20 percent. The firmware's fast-active boost keeps raising the performance state while the GPU idles for no more than a tolerated share of each boost window. The policy's tolerance, a byte in the boost block, kept the controller default of 10 percent. J775D's core type takes 20 percent: its controller arm writes it. J714 keeps 10. Signed-off-by: Jeff Queisser (cherry picked from commit 51eba46385c70802bdb891d551ae6208a989c8ce in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 66ae15dab7046f..23ef0aa7e789d8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -2752,6 +2752,9 @@ pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Resu for offset in [0xb94, 0xbd4, 0xbf9] { put_u32(policy, offset, J775D_PERF_FLOOR as u32 * 100); } + // The fast-active boost may continue while the GPU idles for 20 percent of a boost window + // (byte-packed in the boost block) on the Ultra's core type; the default is 10. + policy[0xbfd] = 20; const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; for (offset, words) in [ (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), From 8d1f2322a2d4aabe484e93abc7f9f84e4321ebd7 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:18:15 -0800 Subject: [PATCH 091/171] drm/asahi: g17: M5: Fix the Ultra power policy ignoring die 1's sensors On the M5 Ultra (J775D) the GPU power policy read the temperature sensors of only one of its two dies. It now reads all six records, both dies'. The policy's AON telemetry state names the records the GPU's and the auxiliary domain's controllers read: sixteen GPU slots with a found mask, and two auxiliary-domain slots with a mask. J775D's came from the J714 defaults: g0TR, g1TR and r0TR, one die's sensors, and no masks. J775D has two dies, and its AON table (encode_common26_aon_ptd) holds six records: g0TR-g3TR and r0TR-r1TR. J775D's controller arm now names all six, with masks 0xf and 0x3. J714 is unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit d914e782a7f9a1eab04fbe372a9d9255f9f4a71a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 23ef0aa7e789d8..c5284538aaed7d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -2755,6 +2755,17 @@ pub(crate) fn encode_m5_policy_controller_defaults_j775d(out: &mut [u8]) -> Resu // The fast-active boost may continue while the GPU idles for 20 percent of a boost window // (byte-packed in the boost block) on the Ultra's core type; the default is 10. policy[0xbfd] = 20; + // The AON temperature records the GPU's and the auxiliary domain's controllers read, with + // each list's found mask: both dies' g0TR-g3TR and r0TR-r1TR, the six records of J775D's + // AON table (encode_common26_aon_ptd). + for (index, tag) in [0x6730_5452, 0x6731_5452, 0x6732_5452, 0x6733_5452].into_iter().enumerate() { + put_u32(policy, 0x11a24 + index * 4, tag); + } + put_u32(policy, 0x11a64, 0xf); + for (index, tag) in [0x7230_5452, 0x7231_5452].into_iter().enumerate() { + put_u32(policy, 0x11a6c + index * 4, tag); + } + put_u32(policy, 0x11a74, 0x3); const DIE_CHANNELS: u64 = 0x6fcc_3fcc_6fcc_3fcc; for (offset, words) in [ (0x2324, [0x6fcc_3fcc_0000_0000, DIE_CHANNELS, DIE_CHANNELS, DIE_CHANNELS, 0x6fcc_3fcc]), From c9216e13df2cf83a4ecf52e834ca25590ca52592 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:18:27 -0800 Subject: [PATCH 092/171] drm/asahi: g17: M5: Fix the Ultra firmware seeing half its cache controllers On the M5 Ultra (J775D) the GPU firmware was told that four memory cache controllers were active, where the chip has eight. It now gets all eight. The M5 hardware data carries the active memory-cache-controller mask, which the firmware copies into its runtime configuration at initialization. J775D kept the inherited 0xf, four controllers. Its ADT GPU node's active-mcc-bit-vector is 0xff: eight. Board::mcc_mask now gives J775D 0xff. J714 is unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 9fa8586f7d5dce0d1c5b955e1bf27bcd870ce1ea in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index c5284538aaed7d..5c658eb657a1c9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1783,6 +1783,8 @@ pub(crate) struct Board { pub(crate) controller_defaults: Option Result<(), InitdataError>>, /// Region C's idle standby timer where the board's differs from the image's 1500 us, us. pub(crate) idle_standby_us: Option, + /// The active memory cache controller mask, where it differs from the inherited 0xf. + mcc_mask: Option, } /// M5 Pro: two clusters of ten cores. @@ -1801,6 +1803,7 @@ pub(crate) const J714: Board = Board { calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, + mcc_mask: None, }; /// M5 Max: four clusters of ten cores. @@ -1820,6 +1823,7 @@ pub(crate) const J716C: Board = Board { calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, + mcc_mask: None, }; /// J775D's performance controllers run from P3. @@ -1845,6 +1849,8 @@ pub(crate) const J775D: Board = Board { calibration: None, controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), idle_standby_us: Some(3500), + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), }; const _: () = { @@ -2264,6 +2270,9 @@ pub(crate) fn encode_m5_hw_data( put_u32(out, 0x2660, 0); put_u32(out, 0x2728, 0); put_u32(out, 0x272c, 1); + if let Some(mask) = board.mcc_mask { + put_u64(out, 0x10a8, mask); + } // The inherited J813 minimum parameter-buffer size is replaced per board. put_u32(out, 0x26b0, board.min_parameter_buffer); Ok(()) From a091f3781c023c0fddf00249dc6a34ffa1a4577d Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 19:01:26 -0800 Subject: [PATCH 093/171] drm/asahi: g17: Fix the M5 Ultra thermal governor never seeing a temperature With the opt-in M5 thermal governor on, on the M5 Ultra Mac Studio (J775D) it found neither thermal zone it looked for, saw no temperature and never acted. It now reads the hotspot from the SoC die zone. The thermal governor reads the SoC hotspot from the thermal zone smc-TCMb, or from J714's GPU die zones plus their offset. On the Mac Studio (M5 Ultra) the SMC hwmon driver discovers its temperatures, and the device tree names the hotspot key, TCMb, as the only key of the macsmc_soc_die zone, so neither of those zones exists: the governor saw no temperature and stayed released. Read macsmc_soc_die after smc-TCMb and before the die fallback. Signed-off-by: Jeff Queisser (cherry picked from commit 4fe427b4afefe7923dfbfc1c818fb0ef493324c4 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/thermal.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs index bea0b9708352d4..9ff7d5c3f6c944 100644 --- a/drivers/gpu/drm/asahi/g17/thermal.rs +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -42,6 +42,9 @@ const J714_GPU_DIE_ZONES: [&CStr; 8] = [ c"smc-Tg43", ]; const GPU_DIE_TO_HOTSPOT_MC: i32 = 6000; +/// Where the SMC driver discovers the temperatures and the device tree names the hotspot key +/// (TCMb) as the SoC die zone's only key (J775D), that zone reads the hotspot. +const SOC_DIE_ZONE: &CStr = c"macsmc_soc_die"; /// Region C word holding the host GPU power budget (mW). pub(crate) const REGION_C_POWER_BUDGET: usize = 0x88; @@ -195,10 +198,12 @@ fn zone_temperature_mc(_provider: &SensorProvider, name: &CStr) -> Option { /// lookups and sensor reads may sleep. pub(crate) fn hotspot_temperature_mc() -> Option { let provider = SensorProvider::acquire()?; - zone_temperature_mc(&provider, J714_HOTSPOT_ZONE).or_else(|| { - J714_GPU_DIE_ZONES.iter().filter_map(|name| zone_temperature_mc(&provider, name)).max() - .map(|temp| temp + GPU_DIE_TO_HOTSPOT_MC) - }) + zone_temperature_mc(&provider, J714_HOTSPOT_ZONE) + .or_else(|| zone_temperature_mc(&provider, SOC_DIE_ZONE)) + .or_else(|| { + J714_GPU_DIE_ZONES.iter().filter_map(|name| zone_temperature_mc(&provider, name)).max() + .map(|temp| temp + GPU_DIE_TO_HOTSPOT_MC) + }) } /// Keeps the fans boosted at least this long once boosted, so a short release or idle period From 9144c1d8295ea176adfc2128493cfe536854f8e4 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 19:02:53 -0800 Subject: [PATCH 094/171] drm/asahi: g17: Fix the M5 governor overriding fans userspace controls With the opt-in M5 thermal governor on, it could take over a fan userspace was controlling, forced the fans to automatic mode on release whatever mode they had started in, and skipped the handshake an SMC that keeps its fans in system mode needs. It now goes through the SMC hwmon driver, which leaves userspace's fans alone and hands the rest back properly. The governor wrote each fan's mode and target keys itself, through the SMC core it took from the hwmon device's parent: a second owner of the mode key beside macsmc-hwmon's manual mode. It wrote manual mode without the system mode handshake an SMC that keeps its fans in system mode needs, wrote automatic mode on release whatever mode the firmware had started in, and could take a fan userspace held. Set the fans with macsmc_hwmon_fans_request() instead, as a share of each fan's range: the curve's permille, the full range for the boost, 0 to hand them back, with the curve's 150 rpm step left to the request. The SMC float conversions go with the direct writes. The fan request and the zone reads go through a small C file, where the thermal core and the hwmon cell have their stubs, so the driver links without either. Signed-off-by: Jeff Queisser [Eduard Moldovan: g17/fans.o goes to integration/aurora's asahi-y] (cherry picked from commit 35418a67f03ee352e69e708387588e933ceac0a1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/g17/fans.c | 24 ++++ drivers/gpu/drm/asahi/g17/fans.h | 7 ++ drivers/gpu/drm/asahi/g17/thermal.rs | 158 ++++----------------------- 4 files changed, 54 insertions(+), 137 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fans.c create mode 100644 drivers/gpu/drm/asahi/g17/fans.h diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 1041f0ded2ed41..641be300ddb6ee 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o g17/fans.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/g17/fans.c b/drivers/gpu/drm/asahi/g17/fans.c new file mode 100644 index 00000000000000..6178a620686084 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The M5 thermal governor's sensor and fan calls (thermal.rs). The thermal + * core and the SMC hwmon cell have stubs where they are not built; Rust + * reaches them through these. + */ +#include +#include +#include +#include "fans.h" + +int asahi_zone_temp(const char *name, int *temp) +{ + struct thermal_zone_device *zone = thermal_zone_get_zone_by_name(name); + + if (IS_ERR_OR_NULL(zone)) + return zone ? PTR_ERR(zone) : -ENODEV; + return thermal_zone_get_temp(zone, temp); +} + +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm) +{ + return macsmc_hwmon_fans_request(dev, permille, min_step_rpm); +} diff --git a/drivers/gpu/drm/asahi/g17/fans.h b/drivers/gpu/drm/asahi/g17/fans.h new file mode 100644 index 00000000000000..4bc18d80d3fe07 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fans.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _ASAHI_G17_FANS_H +#define _ASAHI_G17_FANS_H +struct device; +int asahi_zone_temp(const char *name, int *temp); +int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm); +#endif diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs index 9ff7d5c3f6c944..8ba984ec3f87a5 100644 --- a/drivers/gpu/drm/asahi/g17/thermal.rs +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -22,14 +22,15 @@ //! restores the calibrated maximum on release, when the GPU goes idle, when no sensor can be read //! and at shutdown. Below the engage temperature nothing is written. -use core::ffi::{c_char, c_int, c_void, CStr}; +use core::ffi::{c_char, c_int, c_uint, c_void, CStr}; +use core::sync::atomic::{AtomicI32, Ordering}; use kernel::{bindings, prelude::*}; +// fans.c: the SMC hwmon cell's fan requests and thermal zone reads, which have stubs where +// their drivers are not built. extern "C" { - fn thermal_zone_get_zone_by_name(name: *const c_char) -> *mut c_void; - fn thermal_zone_get_temp(zone: *mut c_void, temp: *mut c_int) -> c_int; - fn apple_smc_read(smc: *mut c_void, key: u32, buf: *mut c_void, size: usize) -> c_int; - fn apple_smc_write(smc: *mut c_void, key: u32, buf: *const c_void, size: usize) -> c_int; + fn asahi_zone_temp(name: *const c_char, temp: *mut c_int) -> c_int; + fn asahi_fans_request(dev: *mut c_void, permille: c_uint, min_step_rpm: c_uint) -> c_int; } /// SoC hotspot temperature computed by the SMC (registered as a thermal zone by the SMC driver): @@ -180,15 +181,10 @@ impl Drop for SensorProvider { /// The provider guard prevents its managed zones from disappearing during /// lookup and read. Do not retain a zone pointer after dropping the guard. fn zone_temperature_mc(_provider: &SensorProvider, name: &CStr) -> Option { - // SAFETY: name is a NUL-terminated string that outlives the call. - let zone = unsafe { thermal_zone_get_zone_by_name(name.as_ptr()) }; - if zone.is_null() || (zone as usize) >= usize::MAX - 4095 { - return None; - } let mut temp: c_int = 0; - // SAFETY: the provider guard protects the zone's lifetime, and temp is a - // valid destination for this call. - (unsafe { thermal_zone_get_temp(zone, &mut temp) } == 0) + // SAFETY: name is a NUL-terminated string that outlives the call, and temp is a valid + // destination; the zone is looked up and read under the provider guard. + (unsafe { asahi_zone_temp(name.as_ptr(), &mut temp) } == 0) .then_some(temp) .filter(|temp| (0..=125_000).contains(temp)) } @@ -238,46 +234,15 @@ fn fan_curve_permille(temp_mc: i32, target_mc: i32) -> i32 { } } -/// The integer value of a non-negative SMC float (truncated), without FPU use. -fn f32_bits_to_u32(bits: u32) -> u32 { - let exponent = ((bits >> 23) & 0xff) as i32 - 127; - if bits >> 31 != 0 || exponent < 0 { - return 0; - } - let mantissa = (bits & 0x7f_ffff) | 0x80_0000; - if exponent >= 23 { - mantissa.checked_shl((exponent - 23) as u32).unwrap_or(u32::MAX) - } else { - mantissa >> (23 - exponent) - } -} - -/// The SMC float encoding of `value` (exact below 2^24), without FPU use. -fn u32_to_f32_bits(value: u32) -> u32 { - if value == 0 { - return 0; - } - let exponent = 31 - value.leading_zeros(); - let mantissa = if exponent >= 23 { value >> (exponent - 23) } else { value << (23 - exponent) }; - ((exponent + 127) << 23) | (mantissa & 0x7f_ffff) -} - -fn smc_key(name: &[u8; 4]) -> u32 { - u32::from_be_bytes(*name) -} - -/// Runs `f` with the SMC core of the sensor provider, under the provider lock (which serializes -/// with its unbinding). -fn with_smc(f: impl FnOnce(*mut c_void) -> Option) -> Option { +/// Sets every fan the SMC hwmon cell lets a kernel policy set to `permille` of its +/// minimum-to-maximum range, or hands them back to the SMC for 0, under the provider lock (which +/// serializes with its unbinding). A fan whose target is within `min_step_rpm` of the new one +/// keeps it. Returns the first fan's target rpm, 0 once handed back. +fn request_fans(permille: u32, min_step_rpm: u32) -> Option { let provider = SensorProvider::acquire()?; - // SAFETY: the provider lock keeps the device bound; the SMC core is its parent, whose driver - // data is the core's state for as long as a child is bound. - let smc = unsafe { - let parent = (*provider.0).parent; - if parent.is_null() { return None; } - (*parent).driver_data - }; - if smc.is_null() { None } else { f(smc) } + // SAFETY: the provider lock keeps the device bound, as the request requires. + let rpm = unsafe { asahi_fans_request(provider.0.cast(), permille, min_step_rpm) }; + u32::try_from(rpm).ok() } /// Fan boost: while the governor limits the GPU, every fan runs at its hardware maximum instead of @@ -299,43 +264,7 @@ impl FanBoost { /// Puts every fan in manual mode at `permille` of its minimum-to-maximum range, or back in /// automatic mode for `None`. Returns the first fan's target rpm, 0 for automatic. fn set_curve(&mut self, permille: Option) -> Option { - with_smc(|smc| { - let mut count = 0u8; - // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. - if unsafe { apple_smc_read(smc, smc_key(b"FNum"), (&raw mut count).cast(), 1) } < 0 { - return None; - } - let mut first = 0; - for fan in 0..count.min(4) { - let digit = b'0' + fan; - let mode = u8::from(permille.is_some()); - let (mut minimum, mut maximum) = ([0u8; 4], [0u8; 4]); - // SAFETY: as above. - unsafe { - if let Some(permille) = permille { - if apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'n']), minimum.as_mut_ptr().cast(), 4) < 0 - || apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'x']), maximum.as_mut_ptr().cast(), 4) < 0 - { - return None; - } - let low = f32_bits_to_u32(u32::from_le_bytes(minimum)); - let high = f32_bits_to_u32(u32::from_le_bytes(maximum)).max(low); - let rpm = low + (high - low) * permille.clamp(0, 1000) as u32 / 1000; - if fan == 0 { - first = rpm; - } - let target = u32_to_f32_bits(rpm).to_le_bytes(); - if apple_smc_write(smc, smc_key(&[b'F', digit, b'T', b'g']), target.as_ptr().cast(), 4) < 0 { - return None; - } - } - if apple_smc_write(smc, smc_key(&[b'F', digit, b'm', b'd']), (&raw const mode).cast(), 1) < 0 { - return None; - } - } - } - Some(first) - }) + request_fans(permille.map_or(0, |permille| permille.clamp(1, 1000) as u32), 0) } /// One governor step of the proportional curve: `busy` while the governor runs a busy GPU. @@ -357,56 +286,13 @@ impl FanBoost { } fn set_curve_if_moved(&mut self, permille: i32) -> Option { - // The range is read on every write; skip the write while the target barely moves. - let estimate = with_smc(|smc| { - let mut bytes = [[0u8; 4]; 2]; - // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. - unsafe { - if apple_smc_read(smc, smc_key(b"F0Mn"), bytes[0].as_mut_ptr().cast(), 4) < 0 - || apple_smc_read(smc, smc_key(b"F0Mx"), bytes[1].as_mut_ptr().cast(), 4) < 0 - { - return None; - } - } - let low = f32_bits_to_u32(u32::from_le_bytes(bytes[0])); - let high = f32_bits_to_u32(u32::from_le_bytes(bytes[1])).max(low); - Some(low + (high - low) * permille.clamp(0, 1000) as u32 / 1000) - })?; - if self.curve_rpm != 0 && estimate.abs_diff(self.curve_rpm) < FAN_CURVE_STEP_RPM { - return None; - } - self.set_curve(Some(permille)) + // A fan already on the curve keeps its target while the new one is within a step. + let step = if self.curve_rpm != 0 { FAN_CURVE_STEP_RPM } else { 0 }; + request_fans(permille.clamp(1, 1000) as u32, step) } fn set(&mut self, boost: bool) -> bool { - with_smc(|smc| { - let mut count = 0u8; - // SAFETY: smc is the bound SMC core; the buffers are valid for their sizes. - if unsafe { apple_smc_read(smc, smc_key(b"FNum"), (&raw mut count).cast(), 1) } < 0 { - return None; - } - let mut ok = count != 0; - for fan in 0..count.min(4) { - let digit = b'0' + fan; - let mode = u8::from(boost); - let mut maximum = [0u8; 4]; - // SAFETY: as above. - unsafe { - if boost - && apple_smc_read(smc, smc_key(&[b'F', digit, b'M', b'x']), maximum.as_mut_ptr().cast(), 4) < 0 - { - ok = false; - continue; - } - ok &= apple_smc_write(smc, smc_key(&[b'F', digit, b'm', b'd']), (&raw const mode).cast(), 1) >= 0; - if boost { - ok &= apple_smc_write(smc, smc_key(&[b'F', digit, b'T', b'g']), maximum.as_ptr().cast(), 4) >= 0; - } - } - } - Some(ok) - }) - .unwrap_or(false) + request_fans(if boost { 1000 } else { 0 }, 0).is_some() } /// Applies one governor step's request. Called from the governor work without driver locks: From 1a08f405ab8673c76fe0be49ff9ec3cdaf0b7adb Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:20:21 -0800 Subject: [PATCH 095/171] drm/asahi: g17: Fix the M5 Ultra governor spinning the fans up early With the opt-in M5 thermal governor on, on the M5 Ultra Mac Studio (J775D) it spun the fans up from an 84 C hotspot, well before the SMC itself would (97 C), for no gain. The curve now starts at 97 C there; J714 keeps 84 C. The governor's fan curve starts at an 84 C hotspot: on J714 the SMC's automatic mode keeps the fans near their minimum under sustained GPU load. The M5 Ultra's SMC raises the fans itself, from a 97 C hotspot under a sustained full-chip load (213-218 W DC input), and the CPU and GPU hold their top states to at least 103 C. A curve from 84 C there spins the fans up well before the SMC would, for no gain. Make the curve start a board setting (Board::fan_curve_start_mc), selected at probe: 97 C on J775D, above the SMC's own start rather than below it; J714 keeps 84 C. m5_thermal_governor already defaults to 0, so this applies only when the governor is turned on. Signed-off-by: Jeff Queisser (cherry picked from commit c0935a5cf0bf8e6ad648b8ce023f075faefce2d0 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 8 +++++++ drivers/gpu/drm/asahi/g17/mod.rs | 1 + drivers/gpu/drm/asahi/g17/thermal.rs | 24 ++++++++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 5c658eb657a1c9..23745995e95c46 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1785,6 +1785,8 @@ pub(crate) struct Board { pub(crate) idle_standby_us: Option, /// The active memory cache controller mask, where it differs from the inherited 0xf. mcc_mask: Option, + /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. + pub(crate) fan_curve_start_mc: i32, } /// M5 Pro: two clusters of ten cores. @@ -1804,6 +1806,7 @@ pub(crate) const J714: Board = Board { controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, mcc_mask: None, + fan_curve_start_mc: 84_000, }; /// M5 Max: four clusters of ten cores. @@ -1824,6 +1827,7 @@ pub(crate) const J716C: Board = Board { controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, mcc_mask: None, + fan_curve_start_mc: 84_000, }; /// J775D's performance controllers run from P3. @@ -1851,6 +1855,10 @@ pub(crate) const J775D: Board = Board { idle_standby_us: Some(3500), // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. mcc_mask: Some(0xff), + // The SMC raises the fans in its automatic mode from a 97 C hotspot under a sustained + // full-chip load, with the CPU and GPU still at their top states: the curve starts there, + // above the SMC's own start rather than below it. + fan_curve_start_mc: 97_000, }; const _: () = { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index caca2dfc29b207..f35bf12ca29999 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1416,6 +1416,7 @@ impl Gpu { return Err(ENODEV); } fw::m5::pool::select_render_pool(board); + thermal::select_board(board); } let geometry = mmu::UatGeometry { diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs index 8ba984ec3f87a5..0f5c21a77488ae 100644 --- a/drivers/gpu/drm/asahi/g17/thermal.rs +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -206,11 +206,13 @@ pub(crate) fn hotspot_temperature_mc() -> Option { /// (a loading screen) does not cycle them. const FAN_HOLD_NS: u64 = 30_000_000_000; -/// Proportional fan curve on the hotspot while the GPU is busy: the SMC's automatic mode keeps -/// the fans near their minimum under sustained GPU load on Linux. From FAN_CURVE_START_MC to -/// the governor target the fans ramp from the minimum to two thirds of the range, and on to -/// FAN_CURVE_TOP_PERMILLE at FAN_CURVE_TOP_MC. -const FAN_CURVE_START_MC: i32 = 84_000; +/// Proportional fan curve on the hotspot while the GPU is busy: on J714 the SMC's automatic mode +/// keeps the fans near their minimum under sustained GPU load on Linux. From the board's curve +/// start ([`super::fw::m5::initdata::Board::fan_curve_start_mc`]) to the governor target the fans +/// ramp from the minimum to two thirds of the range, and on to FAN_CURVE_TOP_PERMILLE at +/// FAN_CURVE_TOP_MC. +static FAN_CURVE_START_MC: AtomicI32 = + AtomicI32::new(super::fw::m5::initdata::J714.fan_curve_start_mc); const FAN_CURVE_TOP_MC: i32 = 98_000; const FAN_CURVE_TARGET_PERMILLE: i32 = 660; const FAN_CURVE_TOP_PERMILLE: i32 = 800; @@ -219,13 +221,19 @@ const FAN_CURVE_RELEASE_MC: i32 = 82_000; /// A new target is written only when it moves at least this far. const FAN_CURVE_STEP_RPM: u32 = 150; +/// Selects the board's thermal settings at probe; J714's until then. +pub(crate) fn select_board(board: &super::fw::m5::initdata::Board) { + FAN_CURVE_START_MC.store(board.fan_curve_start_mc, Ordering::Relaxed); +} + /// The permille of the fan range the curve asks for at hotspot `temp_mc`. fn fan_curve_permille(temp_mc: i32, target_mc: i32) -> i32 { - let target_mc = target_mc.max(FAN_CURVE_START_MC + 1000); - if temp_mc <= FAN_CURVE_START_MC { + let start_mc = FAN_CURVE_START_MC.load(Ordering::Relaxed); + let target_mc = target_mc.max(start_mc + 1000); + if temp_mc <= start_mc { 0 } else if temp_mc <= target_mc { - (temp_mc - FAN_CURVE_START_MC) * FAN_CURVE_TARGET_PERMILLE / (target_mc - FAN_CURVE_START_MC) + (temp_mc - start_mc) * FAN_CURVE_TARGET_PERMILLE / (target_mc - start_mc) } else { let top = FAN_CURVE_TOP_MC.max(target_mc + 1000); (FAN_CURVE_TARGET_PERMILLE + (temp_mc.min(top) - target_mc) From d31655e29e0a5320e8b4275031528de134f5d316 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 18:29:59 -0800 Subject: [PATCH 096/171] drm/asahi: g17: Fix large M5 render passes corrupting a client buffer On the M5 Ultra (J775D) a render pass that overflowed its parameter buffer and was rendered in parts wrote the metadata of every segment after the first into whatever client object was mapped at the start of the render's window. That object is no longer overwritten. When an M5 pass overflows its parameter buffer and the firmware renders it partially, the tiler starts each later segment from roots it advances inside the target's slices: the tilemap from the slice-1 base, scratch from the slice-6 base, and the layer metadata from the slice-2 base, one 256-byte record per segment. The firmware saves these roots at each split and appends them to the next partial program. A single-layer pass names its first segment's metadata from the target records, so TargetSlices sized slice 2 only for layered targets. Its base was then 0, the compact window's first byte, and the metadata of every segment after the first was written into the object mapped there. Slice 2 now holds one 256-byte record per segment for every M5 target. Signed-off-by: Jeff Queisser (cherry picked from commit 7b9b5647fae32f7e3ef60f919544bbb3c69cf679 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/target.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs index a21fb72e2b7e34..d03ecbfec77ecc 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/target.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/target.rs @@ -40,7 +40,14 @@ impl TargetSlices { if segments == 0 || pb_bytes == 0 { return None; } let segments = segments as usize; sizes[1] = tilemap_bytes.checked_mul(segments)?; - sizes[2] = if layers > 1 { 0x100usize.checked_mul(segments)? } else { 0 }; + // Each segment's layer metadata, single-layer passes included. When the parameter + // buffer runs out and the firmware renders partially, the tiler continues each later + // segment k at this slice's base (named in the TA command) plus 0x100 k, and the + // firmware names that root in the next partial program. A single-layer pass's first segment + // takes its metadata from the target records instead, so the slice is used only once + // a pass splits: left unsized, its base was the compact window's first byte, and the + // segments' metadata landed in whatever lived there. + sizes[2] = 0x100usize.checked_mul(segments)?; sizes[4] = (kind as usize).checked_mul(0x300)?.checked_add(0x680)?; sizes[5] = (kind as usize).checked_mul(0x5c0)?; From f91551c0bb3837e3c809c5bf6bcaa98534b9fdc7 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:22:21 -0800 Subject: [PATCH 097/171] drm/asahi: g17: M5: Run compute and render jobs on the bare-metal M5 Ultra On the M5 Ultra Mac Studio (J775D) the GPU driver bound but accepted no compute or render jobs. The bare-metal M5 now takes both by default; other M5 profiles still take none. The bare-metal boot (published roots) runs compute and render; other M5 profiles still take no submission. Signed-off-by: Jeff Queisser (cherry picked from commit bfc8c0227436f47b953ddb85860eede1344c57d1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f35bf12ca29999..5cde7d520ec565 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1017,9 +1017,9 @@ struct Shared { deflake: fw::render::Deflake, compute_blocks: usize, descriptor_flags: [u32; 2], - /// Firmware events and work run: off for an M5 GPU until the M5 queues take submission. + /// Firmware events and work run: off for an M5 profile without the bare-metal boot. submission_enabled: bool, - /// An M5 GPU before its render queues take submission: compute only. + /// Such an M5 profile admits no render command. firmware_test: bool, /// The M5 shared-queue resource policy (frozen device topology). render_policy: Option, @@ -1475,6 +1475,8 @@ impl Gpu { ); } let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; + // The bare-metal M5 boot runs both engines; other M5 profiles take no submission. + let submits = !m5 || cfg.profile == FirmwareProfile::M5PublishedRoots; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1486,8 +1488,8 @@ impl Gpu { deflake: cfg.core_layout.deflake, compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], - submission_enabled: !m5, - firmware_test: m5, + submission_enabled: submits, + firmware_test: !submits, render_policy, crashed: AtomicBool::new(false), feed, From fe3e38cf6c44efa14ee3476cf041ee54abeeaabb Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:22:59 -0800 Subject: [PATCH 098/171] drm/asahi: g17: M5: Fix Ultra compute operand state sized for the M5 Pro On the M5 Ultra (J775D) compute queues sized their operand state with the M5 Pro's constants (two clusters) instead of the Ultra's eight-cluster layout. It now follows each board's core layout. The board's OperandState (TWO_BY_TEN on the M5 Pro, FOUR_BY_TEN on the M5 Max, EIGHT_BY_TEN on the M5 Ultra) gives the bank size, tail offset and buffer size, as on G17P, instead of the M5 Pro's constants. A build-time assert ties each T6050 bank to its board's core slots. Signed-off-by: Jeff Queisser (cherry picked from commit 86fffd4c689d9a9ca584ad951f0fdf5a9175f7ca in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/compute.rs | 10 ++++++++++ drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 9 +++++---- drivers/gpu/drm/asahi/g17/queue/compute.rs | 10 ++++++---- drivers/gpu/drm/asahi/hw/t6050.rs | 8 ++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 819427d8af58f4..76f4f96bd6ea67 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -180,6 +180,16 @@ pub(crate) struct OperandState { } impl OperandState { + /// Size of one preemption bank, the region the M5 program exchanges. + pub(crate) const fn bank_bytes(&self) -> u32 { + self.r0a5c9 as u32 + } + + /// Offset of the per-cluster tails, the M5 program's auxiliary area. + pub(crate) const fn tail_offset(&self) -> u32 { + self.r140a8 as u32 + } + /// One cluster of six cores (A18 Pro). pub(crate) const ONE_BY_SIX: OperandState = OperandState { r10229: 0x12800, diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index b2e439398d826f..d5a18d64e338b3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -10,7 +10,6 @@ use super::super::{kick::RegisterWrite, offset_va, work_key}; pub(crate) const MAIN_COUNT: usize = 40; pub(crate) const ALTERNATE_COUNT: usize = 4; pub(crate) const PREEMPT_BYTES: usize = 0x4000; -pub(crate) const OPERAND_BYTES: usize = 0x40000; pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; /// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent @@ -150,6 +149,8 @@ pub(crate) struct RegisterArgs { /// Size of one operand-state region; the program exchanges two regions. pub(crate) operand_region_bytes: u32, pub(crate) operand_aux_offset: u32, + /// Size of the operand-state buffer at `operand_va`. + pub(crate) operand_bytes: usize, pub(crate) resource_control: u64, pub(crate) dispatch_control: u64, pub(crate) auxiliary_control: u64, @@ -162,13 +163,13 @@ impl RegisterArgs { return Err(EINVAL); } offset_va(self.preempt_va, PREEMPT_BYTES as u64)?; - offset_va(self.operand_va, OPERAND_BYTES as u64)?; + offset_va(self.operand_va, self.operand_bytes as u64)?; if self.preempt_state_offset & 7 != 0 || u64::from(self.preempt_state_offset) + 32 > PREEMPT_BYTES as u64 || self.operand_region_bytes == 0 || self.operand_region_bytes & 0xfff != 0 - || u64::from(self.operand_region_bytes) * 2 + 0x808 > OPERAND_BYTES as u64 + || u64::from(self.operand_region_bytes) * 2 + 0x808 > self.operand_bytes as u64 || self.operand_aux_offset & 7 != 0 - || u64::from(self.operand_aux_offset) + 8 > OPERAND_BYTES as u64 { + || u64::from(self.operand_aux_offset) + 8 > self.operand_bytes as u64 { return Err(EINVAL); } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index d894e402462a0c..ab745a0b835bda 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -168,8 +168,7 @@ impl Graph { let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(if m5 { crate::g17::fw::m5::compute::OPERAND_BYTES } - else { operand_state.size }, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, prot, cpu)?; if !m5 { let support_va = support.gpu_va(); @@ -2438,8 +2437,11 @@ impl Queue { context_id: args.context_id, work_state: work_word, object_ids: 0x0000_0001_0000_0001, resource_policy: plan.map_or(0xf5_0040_0020, |p| u64::from(p.pressure) << 32 | 0x0040_0020), - preempt_state_offset: 0x1b80, operand_region_bytes: 0x1e000, - operand_aux_offset: 0x3d800, resource_control: plan.map_or(0x54b, |p| p.resource_flags), + preempt_state_offset: 0x1b80, + // The board's core layout sizes the operand state, as on G17P. + operand_region_bytes: self.graph.operand_state.bank_bytes(), + operand_aux_offset: self.graph.operand_state.tail_offset(), + operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), dispatch_control: 0x14c08a60, auxiliary_control: 1, }, descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 3fc02fbc8efdb7..3ac6ac6d0837de 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -190,6 +190,14 @@ pub(crate) const CONFIG_ULTRA: Config = Config { }, }; +// Each compute operand bank holds 6 KiB per core slot of the board. +const _: () = { + use crate::g17::fw::m5::initdata::{J714, J716C, J775D}; + assert!(CONFIG.core_layout.operand_state.bank_bytes() == J714.gpu_cores * 0x1800); + assert!(CONFIG_MAX.core_layout.operand_state.bank_bytes() == J716C.gpu_cores * 0x1800); + assert!(CONFIG_ULTRA.core_layout.operand_state.bank_bytes() == J775D.gpu_cores * 0x1800); +}; + /// Every T6050 board binds the same compatible; the ID registers select the configuration. pub(crate) fn config(id: &G17Id) -> Option<&'static Config> { [&CONFIG, &CONFIG_MAX, &CONFIG_ULTRA].into_iter().find(|cfg| id.family == cfg.identity.family From 6ddfb6778725c096a384b200e28422cafab1bb9b Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:25:44 -0800 Subject: [PATCH 099/171] drm/asahi: g17: Fix the recovery check of lockless submits on the T6041 On the T6041 (25G83 firmware), the lockless submission paths checked a word other than the firmware's recovery state, so their test for a GPU recovery in progress did not test it. They now read the same word as InitData; M5, with its own layout, gets the right word too. The port's recovery word, which the lockless publication paths check, was taken at the G17P status block's position on every build. The 25G83 build's status block sits elsewhere and M5's has its own layout, so on those builds the port read a different word than InitData::recovery_state(). Take the port's view from the same recovery_state_offset(). Signed-off-by: Jeff Queisser (cherry picked from commit 3712809ed4fa6876100da2bdbf820131f0c4d1e5 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index ad56900418b5ea..86c4443a28c745 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -959,9 +959,8 @@ impl InitData { qos: self .qos()? .pointer(0, core::mem::size_of::())?, - recovery: self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + // The word recovery_state() reads: the status block moves with the firmware build. + recovery: self.object(self.cluster)?.word(self.recovery_state_offset())?, mailbox, }) } From bd9b8ba34f7f16389baeb6de18b202f812ad20ae Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:29:30 -0800 Subject: [PATCH 100/171] drm/asahi: g17: Fix jobs succeeding on the output of a failed producer On G17 GPUs (the M5 Ultra and Neo), a job that waited in firmware for another engine's job completed successfully even when that producer failed, so the application used whatever the producer had or had not written. It now fails with the producer's error. A command whose other-engine producer was published but not complete when it was scheduled waits for that producer in firmware, and checks the producer's status only before publication. A producer that failed after that left its consumer to retire successfully on whatever the producer had (not) written. Check the firmware-waited producers again when the consumer retires: a failed one fails the consumer with the inherited error, as an input fence's failure does, without recording a fault in the consumer's VM. A producer whose failure is not yet signalled when its consumer retires is not caught; the firmware ran the consumer after the producer's stamp. Signed-off-by: Jeff Queisser (cherry picked from commit c5381fe81cc954524e4be4f1892e4d9158886e0e in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/job.rs | 11 ++++++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 19 +++++++++---------- drivers/gpu/drm/asahi/g17/queue/render.rs | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 93761c025e4c35..54bd823eda094b 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -201,10 +201,19 @@ impl Packet { } check_status(fence)?; } + self.check_firmware_producers()?; + self.completion.check_source_inputs()?; + Ok(()) + } + + /// Producers this packet waits for in firmware (another engine's commands that + /// were published, not complete, when it was scheduled). Checked before + /// publication, and again at retirement: a producer that fails after this + /// packet was published fails it too, although the firmware ran it. + pub(crate) fn check_firmware_producers(&self) -> Result { for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index ab745a0b835bda..db597899ba94a8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1115,18 +1115,17 @@ impl Queue { ); } self.retire_pending = true; - if self.graph.m5 { - defer(Deferred::RetiredCompute( - front.packet.completion.clone(), + let completion = front.packet.completion.clone(); + let producers = front.packet.check_firmware_producers(); + defer(match producers { + Ok(()) if self.graph.m5 => Deferred::RetiredCompute( + completion, [timestamps[0], timestamps[1], 0, 0], front.observed.map(|record| record.timestamps), - ))?; - } else { - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok([timestamps[0], timestamps[1], 0, 0]), - ))?; - } + ), + Ok(()) => Deferred::Retired(completion, Ok([timestamps[0], timestamps[1], 0, 0])), + Err(error) => Deferred::Retired(completion, Err(error)), + })?; self.retire_pending = false; self.submitted = submitted; let active = self.active.pop().ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 81aa4c711c8fb0..1c9856080ca07e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1843,7 +1843,7 @@ impl Pair { ); defer(Deferred::Retired( front.packet.completion.clone(), - Ok(timestamps), + front.packet.check_firmware_producers().map(|()| timestamps), ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; From 086f288c87f1e4d58f419cd40747f7e8fcab0e94 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:29:32 -0800 Subject: [PATCH 101/171] drm/asahi: g17: M5: Fix waited-on work running at the lowest busy clock On the M5 Ultra Mac Studio (J775D) a client that submitted work and waited for it kept the GPU at its lowest busy performance state, because the driver gave the firmware no effort. While a thread waits on GPU work, the firmware now gets full effort. The firmware takes a performance effort from the host, a fraction of the top clock, and holds it as a floor for the performance state. On M5 the driver publishes none (InitData::submission_effort), so a client that submits work and waits for it kept the GPU at its lowest busy state. Count the waiters on the ioctls' output fences. The first waiter that enables signalling on an unsignalled output fence counts it, until the fence signals or is released; other completion fences are not counted. While a thread waits, or has waited within the effort's idle hold, M5 publishes the full effort. Once no thread has waited for that long it goes back to none. The effort is re-evaluated at each submission and at each idle check while work streams. Neo publishes the full effort at every submission, as before. Signed-off-by: Jeff Queisser (cherry picked from commit f1898099cd139101452052d02394efab4650ec22 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fence.rs | 44 ++++++++++++++++++++++++++- drivers/gpu/drm/asahi/g17/initdata.rs | 9 ++++-- drivers/gpu/drm/asahi/g17/mod.rs | 7 ++++- drivers/gpu/drm/asahi/g17/power.rs | 36 ++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 5 files changed, 92 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 96043f736c5c46..47e481e6880d26 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,7 +7,7 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, @@ -63,9 +63,44 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { pub(crate) struct ExportFence { source: Option>, scoped: bool, + /// A waiter enabled signalling on this ioctl output fence before it signalled ([`waiters`]). + waited: AtomicBool, } static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); +/// Output fences a waiter is waiting on: signalling enabled and not yet signalled. +static WAITERS: AtomicU32 = AtomicU32::new(0); +/// Waits begun on output fences, ever. +static WAITS: AtomicU64 = AtomicU64::new(0); + +/// The output fences a waiter is blocked on now, and the waits begun so far: the work a host +/// thread waits for, which the M5 performance effort follows ([`super::power::Effort`]). +pub(crate) fn waiters() -> (u32, u64) { + (WAITERS.load(Ordering::Acquire), WAITS.load(Ordering::Acquire)) +} + +/// The first waiter on an unsignalled output fence (the fence core calls this once, under the +/// fence's lock). +fn waited(flag: &AtomicBool) { + if !flag.swap(true, Ordering::AcqRel) { + WAITERS.fetch_add(1, Ordering::AcqRel); + WAITS.fetch_add(1, Ordering::AcqRel); + } +} + +/// The fence signalled or went away: a waiter on it no longer waits. +fn unwaited(flag: &AtomicBool) { + if flag.swap(false, Ordering::AcqRel) { + WAITERS.fetch_sub(1, Ordering::AcqRel); + } +} + +impl Drop for ExportFence { + fn drop(&mut self) { + unwaited(&self.waited); + } +} + #[vtable] impl FenceOps for ExportFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -74,6 +109,10 @@ impl FenceOps for ExportFence { fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } + fn enable_signaling(self: &FenceObject) -> bool { + waited(&self.waited); + true + } } #[inline(never)] @@ -83,6 +122,7 @@ fn export_fallback(contexts: &FenceContexts) -> Result> ExportFence { source: None, scoped: false, + waited: AtomicBool::new(false), }, )?; // SAFETY: Unique, initialized and still private. Only this constructor @@ -238,6 +278,7 @@ impl Submission { } else { fence.signal(); } + unwaited(&fence.waited); } } } @@ -510,6 +551,7 @@ impl Drop for Outputs { ); } fallback.signal(); + unwaited(&fallback.waited); } } } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 86c4443a28c745..2693266ed91349 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1560,9 +1560,12 @@ impl InitData { self.m5 } - /// The effort value the firmware is given for a requested `effort`: M5 runs with 0. - pub(super) fn submission_effort(&self, effort: u32) -> u32 { - if self.m5 { 0 } + /// The effort value the firmware is given for a requested `effort`. M5 publishes none while + /// work streams, except while a host thread waits for it (`waited`): then the full effort, + /// which the firmware takes as a floor for its performance state, so a submit-and-wait loop + /// runs at the top state. + pub(super) fn submission_effort(&self, effort: u32, waited: bool) -> u32 { + if self.m5 && !waited { 0 } else { effort } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 5cde7d520ec565..767b9aa1f04c7f 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -814,6 +814,7 @@ impl Firmware { self.port.note_gpu_activity(); self.primary.state.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(now_ns(), fence::waiters()); self.apply_effort(power::FULL_EFFORT) } @@ -870,7 +871,7 @@ impl Firmware { ) -> Result { // The state machine and the performance-controller demand follow the requested effort; // the firmware is given the M5 value (InitData::submission_effort). - let published = init.submission_effort(effort); + let published = init.submission_effort(effort, state.waited()); if state.applied() == effort && init.effort()? == published { return Ok(()); } @@ -901,6 +902,10 @@ impl Firmware { if submitted { self.effort.submit(); } + // A waiter that began or ended since the last note moves the effort while work streams. + if self.effort.streaming() && self.effort.note_waiters(now_ns, fence::waiters()) { + self.apply_effort(power::FULL_EFFORT)?; + } let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510f..5c57f6a2894400 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -40,6 +40,13 @@ pub(super) struct Effort { idle_since: Option, applied: u32, submissions: u64, + /// A host thread waits, or waited within [`IDLE_HOLD_NS`], for submitted work + /// ([`Self::note_waiters`]). + waited: bool, + /// The waits begun when the waiters were last noted. + seen_waits: u64, + /// When a waiter was last seen. + last_wait_ns: u64, } impl Effort { @@ -49,9 +56,36 @@ impl Effort { idle_since: None, applied: 0, submissions: 0, + waited: false, + seen_waits: 0, + last_wait_ns: 0, } } + pub(super) fn streaming(&self) -> bool { + self.streaming + } + + /// Whether a host thread waits for the work ([`Self::note_waiters`]). + pub(super) fn waited(&self) -> bool { + self.waited + } + + /// Notes the waiters on submitted work at `now_ns` ([`super::fence::waiters`]): a thread + /// waits now or began a wait since the last note. The note holds for [`IDLE_HOLD_NS`] after + /// the last waiter, across a submit-and-wait loop's turns between its waits. Returns whether + /// that changed. + pub(super) fn note_waiters(&mut self, now_ns: u64, (waiters, waits): (u32, u64)) -> bool { + if waiters > 0 || waits != self.seen_waits { + self.last_wait_ns = now_ns.max(1); + } + self.seen_waits = waits; + let waited = self.last_wait_ns != 0 && now_ns.saturating_sub(self.last_wait_ns) < IDLE_HOLD_NS; + let changed = waited != self.waited; + self.waited = waited; + changed + } + pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; @@ -84,6 +118,8 @@ impl Effort { let idle = now_ns.saturating_sub(since); if idle >= IDLE_HOLD_NS { self.streaming = false; + self.waited = false; + self.last_wait_ns = 0; self.idle_since = None; IdleAction::Clear } else { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ff88b43b1d0d6a..a528618166218f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1442,6 +1442,7 @@ impl compute::Host for ComputeHost<'_> { fn note_submission(&mut self) -> Result { self.shared.note_gpu_activity(); self.effort.submit(); + self.effort.note_waiters(super::now_ns(), super::fence::waiters()); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) } fn submit_m5(&mut self, submission: &super::fw::m5::queue::HardwareSubmission) -> Result { From 1cf7b42274a089de6bed86cad99871e76a618723 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:33:57 -0800 Subject: [PATCH 102/171] drm/asahi: g17: Fix false timeouts of M5 work behind its own context On the M5 Ultra (J775D) a compute job waiting behind an older render of its own context could be failed by the watchdog with a timeout while the firmware was still making progress. Such a job is now kept alive while the firmware progresses. The firmware lists a context's work-state nodes by their words and flushes them from the head, stopping at the first node not yet flushable: a later node's completion stamps wait behind an older node until that node drains. With the words drawn in submission order, a compute kick can wait behind an older render of its own context. The watchdog expired such a wait: its renewals are bounded, and the older-blocker attribution looks only at other VMs, so the same context's older node was never a blocker. That wait has no ceiling while the firmware works. The firmware's progress word, in the status block beside its recovery state, moves as it works; a watchdog that finds it changed within five seconds counts the GPU as progressing. So a published job whose context has an older node in use ahead of its own is renewed, without a count, while the progress word changed within the interval; the word is read at every poll. The first renewal logs the older node and the job's own. A firmware that stops progressing expires the job as before, and an older node of another VM still goes through the cross-VM attribution. Firmware without the word renews nothing by this rule. Signed-off-by: Jeff Queisser (cherry picked from commit 3a1c7871fc228e05315573670875a5416c5434a4 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/context.rs | 9 +++++ drivers/gpu/drm/asahi/g17/context/identity.rs | 12 +++++++ drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 4 +++ drivers/gpu/drm/asahi/g17/initdata.rs | 14 ++++++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 8 +++++ .../drm/asahi/g17/runtime/compute_control.rs | 21 ++++++++++++ drivers/gpu/drm/asahi/g17/timeout.rs | 33 +++++++++++++++++++ 7 files changed, 101 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 7ac04bb34710ed..96e1804d9b573e 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -578,6 +578,15 @@ impl WorkStateLease { pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } + /// The oldest node of this ioctl's context still in use ahead of this ioctl's node, whose + /// draining the firmware's flush of this node waits for; none without a node of its own. + pub(crate) fn older_node(&self) -> Option { + let word = self.word(); + if word == FREE { + return None; + } + self.context.states.older_live(word) + } pub(crate) fn node_va(&self) -> Result { let word = self.word(); if word == FREE { diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index b3ae8831a6523b..7646637bd6d0fa 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -251,6 +251,18 @@ impl RenderSlots { } impl WorkStates { + /// The oldest node in use that the firmware lists ahead of `word`: a live word older than + /// it by the signed 32-bit difference the firmware sorts by. Its ioctl was submitted + /// earlier ([`Order`]), and until that node drains the firmware withholds every stamp of + /// `word`'s node behind it. + pub(super) fn older_live(&self, word: u32) -> Option { + self.live + .iter() + .map(|live| live.load(Ordering::Acquire)) + .filter(|&live| live != FREE && (live.wrapping_sub(word) as i32) < 0) + .min_by_key(|&live| live.wrapping_sub(word) as i32) + } + pub(super) fn used(&self) -> usize { self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() } diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 23745995e95c46..5c9e333d426ae7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1737,6 +1737,10 @@ pub(crate) const M5_STATUS_B_SPAN: usize = M5_PRIVATE_LAYOUT.roles[0].status_a - M5_PRIVATE_LAYOUT.status_b; pub(crate) const M5_RECOVERY_STATE: usize = 0x4ae0; +/// The firmware's progress word in the primary status B, beside the recovery state: the +/// firmware moves it as it works, and a reader that finds it changed since its previous read +/// counts the GPU as progressing. +pub(crate) const M5_PROGRESS: usize = 0x4b00; pub(crate) const M5_HOST_RECOVERY: usize = 0x4af0; pub(crate) const M5_RECOVERY_INFO: usize = 0x4298; pub(crate) const M5_RECOVERY_SLOT_MASK: usize = 0x4ab8; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 2693266ed91349..6c383b6e35be4e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1812,6 +1812,20 @@ impl InitData { } impl InitData { + /// The M5 firmware's progress word ([`super::fw::m5::initdata::M5_PROGRESS`]); ENODEV on + /// other firmware. + pub(super) fn m5_progress(&self) -> Result { + use super::fw::m5::initdata as m; + if !self.m5 { + return Err(ENODEV); + } + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(m::M5_PRIVATE_LAYOUT.status_b + m::M5_PROGRESS)? + .load(Ordering::Relaxed)) + } + fn recovery_state_offset(&self) -> usize { if self.m5 { super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index f2c781c668c97c..5482f2b96b0f37 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -372,6 +372,14 @@ impl job::Backend for Backend { if let Some(deferred) = deferred.as_mut() { firmware.scan_renders(deferred)?; } + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Render watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(true); + } // Borrowed-graph and queued-retirement deferrals share the renewal bound. if timeout::transaction_grace(renewals) && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed530..94d1e68197c06b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,6 +20,14 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init) { + if renewals == 0 { + dev_info!(self.shared.dev.as_ref(), + "Compute watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", + self.owner, older, own); + } + return Ok(Decision::Defer); + } // Each queue is locked on its own: release this one before visiting the others. let Some(own) = firmware.queues.compute(self.owner) .filter(|queue| queue.can_classify_timeout(packet)) @@ -49,6 +57,19 @@ impl Backend { matches!(result, Ok(Decision::Defer)) } + /// The older node of the packet's context ahead of its own, with its own word, when the + /// firmware's progress word moved within the watchdog's interval + /// ([`crate::g17::timeout::behind_own_head`]). The word is read at every poll. Firmware + /// without the word renews nothing by this rule. + pub(super) fn behind_own_head(&self, packet: &Arc, + init: &crate::g17::initdata::InitData) -> Option<(u32, u32)> { + let work = packet.completion.work_state().ok()?; + let older = work.older_node(); + let progressing = init.m5_progress().is_ok_and(crate::g17::timeout::note_progress); + crate::g17::timeout::behind_own_head(packet.is_published(), older, progressing) + .then(|| (older.unwrap_or(0), work.word())) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ce003e6b181961..077bf97567de8f 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -157,6 +157,39 @@ pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator bool { + use core::sync::atomic::Ordering; + let now = crate::g17::now_ns(); + if PROGRESS.swap(word, Ordering::Relaxed) != word { + LAST_PROGRESS_NS.store(now, Ordering::Relaxed); + } + match LAST_PROGRESS_NS.load(Ordering::Relaxed) { + 0 => false, + last => now.saturating_sub(last) < PROGRESS_INTERVAL_NS, + } +} + +/// Work whose context has an older node in use ahead of its own is not late: the firmware +/// lists a context's nodes by their words and flushes from the head, so this work's stamps wait +/// behind the older node until it drains. That wait has no ceiling while the firmware works, so +/// a published job behind its context's older node is renewed, without a count, while the +/// firmware's progress word moves; a firmware that stops progressing for the interval expires it +/// as before, and an older node of another VM still goes through [`attribute`]. +pub(crate) fn behind_own_head(published: bool, older: Option, progressing: bool) -> bool { + published && older.is_some() && progressing +} + #[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; From c842bac23c65afc4731d7b7cfea15b9a4b520ba8 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:32:40 -0800 Subject: [PATCH 103/171] drm/asahi: t6050: Prepare M5 compute queue installs for bare-metal contexts On the M5 Ultra (J775D), installing a compute queue invalidated its ring's aliases only in execution contexts 0 to 3, though on bare metal every context names the roots those aliases live in. It now invalidates every context that names the root. On bare metal (published roots), invalidate a compute kick ring's two aliases before installation in every execution context whose lower or upper translation table base names the alias's root, instead of a fixed list of contexts 0..3: the canonical low alias lives in the kernel lower VM, the firmware alias in the kernel upper VM, whose root sits in the upper half of every execution context. Kernel mappings gain tracks_contexts and kernel objects publish_to_contexts for this; the handoff keeps its fixed list. It is the direct path's part of the M5 compute qualification on J775D; the lower links that qualification's descriptor tables, completion rings and canonical arena need before the first CPU starts are the bootstrap's. Signed-off-by: Jeff Queisser (cherry picked from commit 587101ed12ecd959aa66b91be79b4e8d3bfa69af in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/kick.rs | 13 ++++++++- drivers/gpu/drm/asahi/g17/object.rs | 7 +++++ drivers/gpu/drm/asahi/mmu.rs | 45 +++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 5e940ab70ad190..d3f05657d78cff 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -682,9 +682,20 @@ impl Queue { /// Both aliases are owned mappings of the same full ring. Publish their /// page-table updates to each live translation regime before installation. + /// + /// With published roots each alias reaches every context whose lower or + /// upper half names its VM's root: the low alias lives in the kernel lower + /// VM, the firmware alias in the kernel upper VM, whose root sits in the + /// upper half of every execution context. Under the handoff the first four + /// contexts are invalidated. pub(crate) fn prepare_compute_install(&self) -> Result { - if self.id.engine != DataMaster::Compute { + let Backing::Compute { high, low } = &self.backing else { return Err(EINVAL); + }; + if high.tracks_contexts() { + low.publish_to_contexts()?; + high.publish_to_contexts(); + return Ok(()); } mem::sync(); for asid in 0..4 { diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index b952c998c35ef1..d96b70da262ece 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -229,6 +229,13 @@ impl KernelObject { self.mapping.take().ok_or(EINVAL) } + /// Publishes the page-table updates of the object's own mapping to every context naming its + /// VM's root (see [`mmu::KernelMapping::publish_to_contexts`]). + pub(crate) fn publish_to_contexts(&self) -> Result { + self.mapping.as_ref().ok_or(EINVAL)?.publish_to_contexts(); + Ok(()) + } + /// Returns the GPU address of the object. pub(crate) fn gpu_va(&self) -> u64 { self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f67cf7af5715bb..124f49d467c945 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -948,6 +948,21 @@ impl VmInner { self.tlbi_context_mask(iova, size, contexts); } + /// Invalidate a VA range in every TTBAT context that names this Vm's root in either half: a + /// kernel upper mapping reaches the firmware contexts and every execution context, a kernel + /// lower one the contexts that still name the kernel lower root. + fn tlbi_contexts_naming_root_in_either_half(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self + .uat_inner + .lock() + .contexts_naming_root_in_either_half(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { // Complete new leaves even if no context currently names this root: // a later context can publish it without repeating the map operation. @@ -1414,6 +1429,23 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Whether the UAT tracks which TTBAT contexts name each root (published roots), so that + /// [`Self::publish_to_contexts`] reaches every context that can walk this mapping. Under the + /// handoff the caller keeps its own invalidation. + pub(crate) fn tracks_contexts(&self) -> bool { + self.0.uat_inner.firmware != UatFirmware::Handoff + } + + /// Publishes this mapping's page-table updates to every TTBAT context whose lower or upper + /// half names the owner's root. Call once the mapping set is complete, before work naming it + /// becomes visible. + pub(crate) fn publish_to_contexts(&self) { + self.0 + .owner + .lock_inner() + .tlbi_contexts_naming_root_in_either_half(self.iova(), self.size()); + } + /// Resolves a byte in this retained mapping without creating page tables. pub(crate) fn translate_offset(&self, offset: usize) -> Result { if offset >= self.size() { @@ -1843,6 +1875,19 @@ impl UatShared { }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } + + /// Returns the mask of TTBAT contexts naming the root `ttb` in their lower or upper half. + fn contexts_naming_root_in_either_half(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + slot.ttb0.load(Ordering::Acquire) == expected + || slot.ttb1.load(Ordering::Acquire) == expected + }) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } } /// Returns the TTBAT value naming root `ttb` in context `ctx`. From 783696d4fdb7a4edda80226d732b1f743f885b7a Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:34:05 -0800 Subject: [PATCH 104/171] drm/asahi: g17: Fix M5 Ultra compute dispatch encoded for four clusters On the eight-cluster M5 Ultra (J775D), compute jobs were dispatched with the four-cluster form of the dispatch control, without the cluster-topology term. They now carry the Ultra's topology. Above four clusters the compute dispatch control carries a cluster-topology term, in the form for the firmware feature the 27.0 profile leaves clear. The DRM compute path did not apply it, so on the eight-cluster M5 Ultra it encoded the four-cluster word. Compute queues now carry the device's M5 board (Config::m5), and dispatch_control() ORs the term into the word above the board's four clusters. J714, with two clusters, keeps its word. The M5 descriptor arguments are built only for an M5 queue, from its board. The qualification's hunk is not part of this port. Signed-off-by: Jeff Queisser (cherry picked from commit a27f859d0acbaf29b1531ef6f2c85585f3e84480 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 19 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 3 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 20 ++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index d5a18d64e338b3..d79c0c1bbd3e5a 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -6,12 +6,31 @@ use kernel::prelude::*; use super::super::{kick::RegisterWrite, offset_va, work_key}; +use super::initdata::Board; pub(crate) const MAIN_COUNT: usize = 40; pub(crate) const ALTERNATE_COUNT: usize = 4; pub(crate) const PREEMPT_BYTES: usize = 0x4000; pub(crate) const CHECKPOINTS: [u16; 4] = [37, 39, 39, 40]; +/// A dispatch control word for `board`: above four clusters it carries the cluster-topology term, +/// `0x9c` in the form for the firmware feature the 27.0 profile leaves clear (`0x1c` is the +/// feature-set form). The term is an OR. Compute and render dispatch words both take it. +pub(crate) fn dispatch_control(board: &Board, base: u64) -> u64 { + topology_dispatch_control(base, board.clusters) +} + +/// [`dispatch_control`] for `clusters`. +pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { + if clusters > 4 { base | 0x9c } else { base } +} + +const _: () = { + assert!(topology_dispatch_control(0x14c0_8a60, 8) == 0x14c0_8afc); + assert!(topology_dispatch_control(0x10c0_8860, 8) == 0x10c0_88fc); + assert!(topology_dispatch_control(0x14c0_8a60, 4) == 0x14c0_8a60); +}; + /// The SCS scratch allocation. Pages are 4 KiB USC pool pages, independent /// of the host's CPU page size. Layout sections and control registers are /// specific to M5; the other G17 profile uses a different fixed region. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 767b9aa1f04c7f..bbf6a622912c1a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1018,6 +1018,8 @@ struct Shared { variant: &'static fw::Variant, /// Layout of compute queues' operand-state buffers for this GPU's cores. operand_state: &'static fw::compute::OperandState, + /// The M5 board ([`Config::m5`]) of an M5 profile. + m5_board: Option<&'static fw::m5::initdata::Board>, /// Layout of renders' deflake buffers for this GPU's cores. deflake: fw::render::Deflake, compute_blocks: usize, @@ -1490,6 +1492,7 @@ impl Gpu { active_cores, variant, operand_state: &cfg.core_layout.operand_state, + m5_board: cfg.m5.filter(|_| m5), deflake: cfg.core_layout.deflake, compute_blocks: cfg.core_layout.compute_blocks, descriptor_flags: [id.perf_control, id.perf_map[0]], diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index db597899ba94a8..469c69dbbf7f82 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -14,6 +14,7 @@ use crate::g17::{ fw::{ channels::ControlRecord, compute::{ComputeProgram, OperandState, COMPUTE_DESCRIPTOR_SIZE}, + m5::initdata::Board, queue::*, variant::PointerLayout, }, @@ -149,6 +150,8 @@ struct Graph { pointers: PointerLayout, /// Layout of the operand-state buffer for the GPU's cores. operand_state: &'static OperandState, + /// The M5 board, whose part selects the M5 descriptor words. + board: Option<&'static Board>, program: ComputeProgram, } @@ -159,6 +162,7 @@ impl Graph { context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, m5: bool, ) -> Result { @@ -243,6 +247,7 @@ impl Graph { reclaimed: 0, pointers, operand_state, + board, program, }) } @@ -652,11 +657,12 @@ impl Queue { context: Arc, pointers: PointerLayout, operand_state: &'static OperandState, + board: Option<&'static Board>, program: ComputeProgram, m5: bool, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, program, m5)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state, board, program, m5)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -2425,7 +2431,8 @@ impl Queue { KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; let matrix_ops = self.graph.m5 && *requested_matrix_ops; - let m5_args = crate::g17::fw::m5::compute::DescriptorArgs { + let m5_board = if self.graph.m5 { Some(self.graph.board.ok_or(EINVAL)?) } else { None }; + let m5_args = m5_board.map(|board| crate::g17::fw::m5::compute::DescriptorArgs { registers: crate::g17::fw::m5::compute::RegisterArgs { matrix_ops, scratch: m5_scratch, @@ -2441,7 +2448,8 @@ impl Queue { operand_region_bytes: self.graph.operand_state.bank_bytes(), operand_aux_offset: self.graph.operand_state.tail_offset(), operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), - dispatch_control: 0x14c08a60, auxiliary_control: 1, + dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, 0x14c08a60), + auxiliary_control: 1, }, descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], @@ -2450,7 +2458,7 @@ impl Queue { status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, - }; + }); let m5_dependency_types = [2u8; crate::g17::fw::kick::QID_MAX as usize + 1]; let kick_args = KickArgs { qid, @@ -2497,9 +2505,9 @@ impl Queue { accounted = true; let object = &mut self.graph.ring.as_mut().ok_or(EAGAIN)?.object; let offset = kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; - if self.graph.m5 { + if let Some(m5_args) = &m5_args { object.initialize::(offset, - |descriptor| descriptor.write_zeroed(&m5_args))?; + |descriptor| descriptor.write_zeroed(m5_args))?; } else { object.initialize::(offset, |descriptor| descriptor.write_zeroed(&args))?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index a528618166218f..0dbb529bd8ba8e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1170,6 +1170,7 @@ impl Backend { self.context.clone(), self.shared.variant.pointers, self.shared.operand_state, + self.shared.m5_board, self.shared.variant.compute, alloc.uat.is_m5(), )?; From 330991847d44591b720fefdbd851f41d69377bc2 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:34:50 -0800 Subject: [PATCH 105/171] drm/asahi: g17: Fix M5 Ultra compute jobs waiting on another context On the M5 Ultra (J775D) every compute job waited on the pool event of context 3, whatever context it ran in, and lacked the entry gate its notification relies on. Each job now waits on its own context's event, so it is no longer tied to an unrelated context (or held at a firmware restart by another queue's event). Every M5 compute kick carried one fixed event mask, [0, 0, 8, 0]: event 131, the pool event of context 3, whatever the kick's context, and without the entry-signal gate, although every kick is published with its current-kick notification. On the M5 Ultra the mask is now built for each DRM kick by kick_event_mask(): - the kick's own context's pool event, 64 x 2 + the context (the firmware's record of a compute pool names engine 2 beside the pool's context); a context of 64 or more is EINVAL, before anything is firmware-visible; - event 254, the entry-signal gate, which the firmware clears in the kick its notification names once it has handled that record. While the firmware is halted in a restart it holds the interrupted queue's context pool event; followers whose masks name it wait on it and launch once it is released at the resume. The G17D encodings are a board property (Board::g17d_forms), read from the device's board: J714 keeps [0, 0, 8, 0]. The qualification's hunk is not part of this port. Signed-off-by: Jeff Queisser (cherry picked from commit 28ed1956fda5d6c993f1ec2fc0aad76db3e8c6ed in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 28 +++++++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 12 +++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 +++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index d79c0c1bbd3e5a..b63d093318b489 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -25,6 +25,34 @@ pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { if clusters > 4 { base | 0x9c } else { base } } +/// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. +const POOL_EVENT_CONTEXTS: u32 = 64; +/// The engine of a compute pool in the event space: the firmware's record of each compute pool +/// names engine 2 beside the pool's context. +const POOL_EVENT_ENGINE: u32 = 2; +/// The entry-signal gate: event 254, which the firmware clears in the kick a current-kick +/// notification names once it has handled that record. +const ENTRY_GATE_EVENT: u32 = 254; + +/// The event mask of a compute kick of context `context_id` on `board`, published with its +/// current-kick notification. With the G17D forms ([`Board::g17d_forms`]) the kick waits on its +/// own context's pool event (64 x 2 + the context), which the firmware holds while it processes +/// that pool's dependencies, and on the entry-signal gate; a context of 64 or more is EINVAL. +/// Otherwise it is the fixed `[0, 0, 8, 0]` (event 131, context 3's pool event, without the gate). +pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4]> { + if !board.g17d_forms() { + return Ok([0, 0, 8, 0]); + } + if context_id >= POOL_EVENT_CONTEXTS { + return Err(EINVAL); + } + let mut mask = [0u64; 4]; + for event in [POOL_EVENT_ENGINE * POOL_EVENT_CONTEXTS + context_id, ENTRY_GATE_EVENT] { + mask[(event / 64) as usize] |= 1 << (event % 64); + } + Ok(mask) +} + const _: () = { assert!(topology_dispatch_control(0x14c0_8a60, 8) == 0x14c0_8afc); assert!(topology_dispatch_control(0x10c0_8860, 8) == 0x10c0_88fc); diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 5c9e333d426ae7..abf2605aa862e9 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1791,6 +1791,9 @@ pub(crate) struct Board { mcc_mask: Option, /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. pub(crate) fan_curve_start_mc: i32, + /// Whether the host encodings are the G17D ones measured on the M5 Ultra + /// ([`Board::g17d_forms`]). + g17d_forms: bool, } /// M5 Pro: two clusters of ten cores. @@ -1811,6 +1814,7 @@ pub(crate) const J714: Board = Board { idle_standby_us: None, mcc_mask: None, fan_curve_start_mc: 84_000, + g17d_forms: false, }; /// M5 Max: four clusters of ten cores. @@ -1832,6 +1836,7 @@ pub(crate) const J716C: Board = Board { idle_standby_us: None, mcc_mask: None, fan_curve_start_mc: 84_000, + g17d_forms: false, }; /// J775D's performance controllers run from P3. @@ -1863,6 +1868,7 @@ pub(crate) const J775D: Board = Board { // full-chip load, with the CPU and GPU still at their top states: the curve starts there, // above the SMC's own start rather than below it. fan_curve_start_mc: 97_000, + g17d_forms: true, }; const _: () = { @@ -2142,6 +2148,12 @@ pub(crate) struct M5RegisterWindow { pub(crate) const M5_MAX_REGISTER_WINDOWS: usize = J775D_REGISTER_WINDOWS.len(); impl Board { + /// Whether the board takes the host encodings measured on the M5 Ultra (G17D) where they + /// differ from J714's, as `super::compute` and the render registers read them. + pub(crate) const fn g17d_forms(&self) -> bool { + self.g17d_forms + } + /// The board's `index`th register window. pub(crate) fn register_window_spec(&self, index: usize) -> Option<&'static M5RegisterWindowSpec> { self.register_windows.get(index) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 469c69dbbf7f82..337aee5109c4a7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2473,7 +2473,10 @@ impl Queue { class: COMPUTE_QOS_CLASS, }, mcache: None, - event_mask: if self.graph.m5 { [0, 0, 8, 0] } else { COMPUTE_KICK_EVENT_MASK }, + event_mask: match m5_board { + Some(board) => crate::g17::fw::m5::compute::kick_event_mask(board, identity.context)?, + None => COMPUTE_KICK_EVENT_MASK, + }, register_arrays: if self.graph.m5 { crate::g17::fw::m5::compute::bindings(descriptor_low, m5_scratch, matrix_ops)? } else { From d1cfc7387de0137b6f0b8a6040547fc115df281a Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:35:09 -0800 Subject: [PATCH 106/171] drm/asahi: g17: Fix M5 Ultra compute jobs of one queue overlapping On the M5 Ultra (J775D) two compute jobs of one queue were seen running side by side, though each must wait for the previous job of its queue to complete. A queue's jobs now run one after the other. Each M5 compute kick names the previous kick of its queue as its implicit parent. It carried dependency class 2, the pipelined class, with which a kick may start while its parent still runs: on the M5 Ultra two kicks of one queue were seen running side by side. A dependency takes the consuming engine's class (2 for compute, 2 for tiling, 3 for fragment) only when it is pipelined: a barrier marked pipelined, or the implicit parent of a late-evaluated wait. Every other dependency takes class 1. No DRM compute kick is a late-evaluated wait, so with the G17D forms its parent is class 1: the kick waits for the previous kick of its queue to complete. Explicit dependencies keep class 2; J714 keeps class 2 throughout. The qualification's hunk is not part of this port. Signed-off-by: Jeff Queisser (cherry picked from commit 8b3ec6fa8e1ae8bfc2e794bb8759eeb3a80483a8 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 14 ++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 +++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index b63d093318b489..8432173d799e77 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -25,6 +25,20 @@ pub(crate) const fn topology_dispatch_control(base: u64, clusters: u32) -> u64 { if clusters > 4 { base | 0x9c } else { base } } +/// Dependency class of a pipelined compute dependency: the class the kick-queue encoder supplies +/// for compute work (tiling's is also 2, fragment's 3). +pub(crate) const COMPUTE_PIPELINED_DEPENDENCY: u8 = 2; + +/// Dependency class of a compute kick's implicit parent (the previous kick of its queue) on +/// `board`. With the G17D forms it is derived as the kick-queue encoder derives it: the compute +/// class only for a command whose pipeline dependency type is 1, which only a late-evaluated wait +/// sets, and class 1 otherwise, which holds the kick until its parent has completed. No DRM +/// compute kick is a late-evaluated wait. Otherwise it is the pipelined class, with which a kick +/// may start while its parent still runs. +pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 { + if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } +} + /// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. const POOL_EVENT_CONTEXTS: u32 = 64; /// The engine of a compute pool in the event space: the firmware's record of each compute pool diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 337aee5109c4a7..5d010c612299e2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2459,7 +2459,13 @@ impl Queue { shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, }); - let m5_dependency_types = [2u8; crate::g17::fw::kick::QID_MAX as usize + 1]; + // Explicit dependencies keep the pipelined class; the implicit parent, last, takes its + // own ([`crate::g17::fw::m5::compute::parent_dependency`]). + let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY; + crate::g17::fw::kick::QID_MAX as usize + 1]; + if let (Some(board), Some(parent)) = (m5_board, m5_dependency_types.get_mut(dependencies.len())) { + *parent = crate::g17::fw::m5::compute::parent_dependency(board, false); + } let kick_args = KickArgs { qid, timestamp: kick_timestamp, From d7f9279c9be31643abb1a796985f272bd027f205 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:35:13 -0800 Subject: [PATCH 107/171] drm/asahi: g17: Fix M5 Ultra compute jobs carrying no per-queue count On the M5 Ultra (J775D) every compute job told the firmware it was its queue's first (count 0), where the firmware expects each queue's running count of compute commands. Each job now carries its queue's count. The M5 compute descriptor's completion ordinal was 0 on every kick. The firmware's queues expect it per command: each queue counts its compute commands (zero for a new queue), and every compute command's descriptor carries the count before it. Render commands keep a count of their own. With the G17D forms every DRM compute kick now carries its hardware queue's count of earlier compute kicks. J714 keeps 0. The qualification's hunk is not part of this port. Signed-off-by: Jeff Queisser (cherry picked from commit e6344fa69fa1eb574a612f39bf7e57f426a22631 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 7 +++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index 8432173d799e77..61a355a45b47c2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -39,6 +39,13 @@ pub(crate) fn parent_dependency(board: &Board, pipelined: bool) -> u8 { if pipelined || !board.g17d_forms() { COMPUTE_PIPELINED_DEPENDENCY } else { 1 } } +/// The completion ordinal of a compute command on `board`. With the G17D forms it is the count of +/// compute commands its hardware queue published before it (zero for a new queue), wrapping as a +/// u32; render commands count separately. Otherwise it is 0 on every kick. +pub(crate) fn completion_ordinal(board: &Board, published_before: u32) -> u32 { + if board.g17d_forms() { published_before } else { 0 } +} + /// Contexts per engine in the event space: a pool's event is 64 x its engine + its context. const POOL_EVENT_CONTEXTS: u32 = 64; /// The engine of a compute pool in the event space: the firmware's record of each compute pool diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5d010c612299e2..648fdd50dd4740 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2454,7 +2454,8 @@ impl Queue { descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], stamp_value: crate::g17::fw::stamp_value(kick_timestamp.get() as u32), qid: u32::from(qid), - kick_timestamp: kick_timestamp.get(), completion_ordinal: 0, + kick_timestamp: kick_timestamp.get(), + completion_ordinal: crate::g17::fw::m5::compute::completion_ordinal(board, self.ordinal), status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, From 18cdc542bbd4da015cf3762321541e5d2aee9573 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:37:11 -0800 Subject: [PATCH 108/171] drm/asahi: g17: Fix large M5 heaps colliding with the driver's ring aliases On the M5 Ultra (J775D), once an application's GPU heap grew far enough, Mesa's read-only shadow mappings claimed the same GPU addresses as the driver's ring aliases, and any user bind could replace an alias the firmware reads. The aliases now sit outside every Mesa window and are reserved against binds. Mesa's M5 layout puts the data heap at 256 GiB and binds a read-only shadow of every allocation 128 GiB above it, up to 512 GiB. The M5 render kick rings, the compute descriptor rings' low aliases and the compute queues' client-private aliases still used the A18 Pro's arenas (just above 448 GiB), which lie inside that shadow: once the heap reached them, Mesa's shadow bind and the driver's alias claimed the same VA. An M5 client VM also reserved nothing, so a user bind could replace a kernel alias the firmware reads. Place M5 render kick rings and compute descriptor rings in the canonical arena, as compute kick rings already are, and client-private aliases in a new arena at 576 GiB, past the canonical one and outside every Mesa window. An M5 client VM reserves both arenas against VM_BIND from creation. The A18 Pro's placements are unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 789bcaa3af6925a3105e5fd8bb19953d65acfc7f in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/kick.rs | 4 ++-- drivers/gpu/drm/asahi/g17/mod.rs | 9 +++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 13 +++++++------ drivers/gpu/drm/asahi/mmu.rs | 17 +++++++++++++++++ drivers/gpu/drm/asahi/mmu/layout.rs | 5 +++++ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index d3f05657d78cff..b74012bdd37542 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -20,7 +20,7 @@ use super::{ fw::{kick::*, queue::DataMaster}, object::{Allocator, CpuMap, KernelObject}, }; -use crate::{hw::t8140, mem, mmu}; +use crate::{mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; @@ -600,7 +600,7 @@ impl Queue { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut range = t8140::dynamic::LOWER; + let mut range = alloc.uat.kernel_lower_dynamic_range(); let (low, client) = loop { let low = high.alias_in( alloc.uat.kernel_lower_vm(), diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index bbf6a622912c1a..02960b00ba7b29 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2186,6 +2186,15 @@ impl gpu::Gpu for Gpu { .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; if self.cfg.profile.is_m5() { + // The canonical arena the client's queues alias and the client-private one stay + // reserved against userspace binds from creation, so no bind replaces a kernel alias + // the firmware reads. + let ranges = firmware.uat.m5_client_reserved_ranges(); + let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; + for range in ranges { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(KVec::new(), reserved)?; return Ok(vm); } let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 648fdd50dd4740..d2cdbc7c5c0b74 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -20,7 +20,7 @@ use crate::g17::{ }, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; -use crate::{hw::t8140::dynamic, mmu}; +use crate::mmu; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; @@ -118,7 +118,7 @@ impl DescriptorRing { )?; let low = object.alias_in( alloc.uat.kernel_lower_vm(), - dynamic::LOWER, + alloc.uat.kernel_lower_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, )?; @@ -267,15 +267,16 @@ impl Graph { let descriptors = self.descriptor_alias(vm)?; let prot = if self.m5 { mmu::PROT_GPU_SHARED_RW } else { mmu::PROT_GPU_FW_SHARED_RW }; let page = mmu::UAT_PGSZ as u64; + let private = mmu::client_private_range(self.m5); let preempt = self .preempt - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; + let usage = self.usage.alias_in(vm, private.clone(), page, prot)?; let operand = self .operand - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + .alias_in(vm, private.clone(), page, prot)?; let queue_context = self.context_low.as_ref() - .map(|context| context.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)).transpose()?; + .map(|context| context.alias_in(vm, private.clone(), page, prot)).transpose()?; Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 124f49d467c945..2c9443b2977691 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -369,6 +369,16 @@ impl Drop for MappingMutation { } } +/// Where client-private aliases are placed: [`layout::CLIENT_PRIVATE_RANGE`] on M5, +/// outside every Mesa window; the A18 Pro's arena otherwise. +pub(crate) fn client_private_range(m5: bool) -> Range { + if m5 { + layout::CLIENT_PRIVATE_RANGE + } else { + crate::hw::t8140::dynamic::CLIENT_LOWER + } +} + /// Inner data for a Vm instance. struct VmInner { dev: driver::AsahiDevRef, @@ -3682,6 +3692,13 @@ impl Uat { crate::hw::t8140::dynamic::LOWER } } + + /// Ranges an M5 client VM keeps from userspace binds, even while they hold + /// nothing: the canonical arena its queues alias and the client-private one. + pub(crate) fn m5_client_reserved_ranges(&self) -> [Range; 2] { + [layout::CANONICAL_RANGE, layout::CLIENT_PRIVATE_RANGE] + } + /// Returns the address space geometry of this UAT. pub(crate) fn geometry(&self) -> UatGeometry { self.geometry diff --git a/drivers/gpu/drm/asahi/mmu/layout.rs b/drivers/gpu/drm/asahi/mmu/layout.rs index 66c49f1189a308..6bd1a17df97a89 100644 --- a/drivers/gpu/drm/asahi/mmu/layout.rs +++ b/drivers/gpu/drm/asahi/mmu/layout.rs @@ -7,3 +7,8 @@ use core::ops::Range; /// Canonical queue backing has its own sparse arena. It must not overlap /// the client-private arena or retain the A18 Pro's sub-2-GiB limit. pub(super) const CANONICAL_RANGE: Range = 0x80_0000_0000..0x90_0000_0000; +/// Client-private aliases of M5 queues, placed per client VM. Like the +/// canonical arena it lies above Mesa's M5 windows (driver reservation, data +/// heap 0x40.., its read-only shadow 0x60..0x80, which Mesa binds for every +/// allocation, and the USC window 0x100..), so no user mapping can need it. +pub(super) const CLIENT_PRIVATE_RANGE: Range = 0x90_0000_0000..0xa0_0000_0000; From 27da4ea5cfd224e8fff418445fbe46c625ed8df3 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:37:46 -0800 Subject: [PATCH 109/171] drm/asahi: g17: Fix every compute job timing out on the M5 Ultra On the M5 Ultra Mac Studio (J775D) every Vulkan and GL compute job timed out and the firmware killed its context, so no compute work ran. Compute jobs now complete. DRM compute kicks used the J714 tuple (preemption-state offset, dispatch control base and auxiliary control). On the M5 Ultra every such kick timed out and the firmware killed its context, while the in-kernel qualification, which uses a different tuple, passed every dispatch. Name both tuples once in fw/m5/compute.rs and select the qualified one for DRM compute with the G17D forms (Board::g17d_forms); J714 keeps its tuple. The qualification's hunk is not part of this port. Signed-off-by: Jeff Queisser (cherry picked from commit e3df3cb6e8da9a5e60cebe3f6a9e041f24596fbf in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/compute.rs | 23 ++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 12 ++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs index 61a355a45b47c2..d44b8b06c2bf92 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/compute.rs @@ -74,6 +74,29 @@ pub(crate) fn kick_event_mask(board: &Board, context_id: u32) -> Result<[u64; 4] Ok(mask) } +/// The descriptor words of a DRM compute kick that follow the part, not the command: the +/// preemption-state offset, the dispatch control base (before [`dispatch_control`]'s topology +/// term) and the auxiliary control's first write. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct KickTuple { + pub(crate) preempt_state_offset: u32, + pub(crate) dispatch_base: u64, + pub(crate) auxiliary_control: u64, +} + +/// The tuple of J714 compute. +pub(crate) const NORMAL_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1b80, dispatch_base: 0x14c0_8a60, auxiliary_control: 1 }; +/// The tuple the arithmetic qualification runs with on every part. The G17D runs DRM compute with +/// it: with [`NORMAL_TUPLE`] its kicks never complete. +pub(crate) const QUALIFIED_TUPLE: KickTuple = + KickTuple { preempt_state_offset: 0x1480, dispatch_base: 0x10c0_8860, auxiliary_control: 0 }; + +/// The DRM compute tuple of `board` ([`Board::g17d_forms`]). +pub(crate) fn kick_tuple(board: &Board) -> KickTuple { + if board.g17d_forms() { QUALIFIED_TUPLE } else { NORMAL_TUPLE } +} + const _: () = { assert!(topology_dispatch_control(0x14c0_8a60, 8) == 0x14c0_8afc); assert!(topology_dispatch_control(0x10c0_8860, 8) == 0x10c0_88fc); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index d2cdbc7c5c0b74..7c30ef0e06a922 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2433,7 +2433,9 @@ impl Queue { let predecessor = KickPredecessor::new(qid, kick_parent)?; let matrix_ops = self.graph.m5 && *requested_matrix_ops; let m5_board = if self.graph.m5 { Some(self.graph.board.ok_or(EINVAL)?) } else { None }; - let m5_args = m5_board.map(|board| crate::g17::fw::m5::compute::DescriptorArgs { + let m5_args = m5_board.map(|board| { + let tuple = crate::g17::fw::m5::compute::kick_tuple(board); + crate::g17::fw::m5::compute::DescriptorArgs { registers: crate::g17::fw::m5::compute::RegisterArgs { matrix_ops, scratch: m5_scratch, @@ -2444,13 +2446,13 @@ impl Queue { context_id: args.context_id, work_state: work_word, object_ids: 0x0000_0001_0000_0001, resource_policy: plan.map_or(0xf5_0040_0020, |p| u64::from(p.pressure) << 32 | 0x0040_0020), - preempt_state_offset: 0x1b80, + preempt_state_offset: tuple.preempt_state_offset, // The board's core layout sizes the operand state, as on G17P. operand_region_bytes: self.graph.operand_state.bank_bytes(), operand_aux_offset: self.graph.operand_state.tail_offset(), operand_bytes: self.graph.operand_state.size, resource_control: plan.map_or(0x54b, |p| p.resource_flags), - dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, 0x14c08a60), - auxiliary_control: 1, + dispatch_control: crate::g17::fw::m5::compute::dispatch_control(board, tuple.dispatch_base), + auxiliary_control: tuple.auxiliary_control, }, descriptor_va: descriptor_low, submit_sequence: 0, work_state_va: work_node, cdm_end_va: *cdm_end_va, stamp_va: [args.stamp_va, args.aux_stamp_va], @@ -2460,7 +2462,7 @@ impl Queue { status_va: args.status_va, timestamp_va: [timestamp_va, timestamp_va + 8], shared_control_va: self.pool.control_va(), event_metadata: [0;16], usage_fw_va: self.graph.usage.gpu_va(), completion_identity: 1, - }); + }}); // Explicit dependencies keep the pipelined class; the implicit parent, last, takes its // own ([`crate::g17::fw::m5::compute::parent_dependency`]). let mut m5_dependency_types = [crate::g17::fw::m5::compute::COMPUTE_PIPELINED_DEPENDENCY; From c026391cf7a1327aa27191f5dc557f9a4853cbd3 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:41:53 -0800 Subject: [PATCH 110/171] drm/asahi: g17: Fix compute scratch leaking across processes on queue reuse On G17 GPUs, including the M5 Ultra Mac Studio (J775D), a process that took over a compute queue another process had used could read that process's GPU stack and scratch memory. A reused queue's memory is now tied to one process's VM, or replaced with fresh, zeroed memory. A retained compute queue is bound to a new owner by bind_reusable, and under QID pressure after share_idle hands an open owner's idle channel over. When the new owner's VM differed from the last one, bind_owner mapped the queue's populated pool backing into the new VM as it was. The pool's blocks hold the stack and scratch memory of the work that ran on the queue, so the new process could read the previous process's private GPU memory. The queue's preemption buffer, usage page and operand state were mapped likewise. Tie a compute pool's backing to one VM. ComputePool records the VM of its first client view and refuses a view in another VM (EPERM); giving the backing back clears the record, and fresh backing starts without one. bind_reusable binds only a queue whose pool serves the owner's VM, then offers a vacant queue for re-arming with fresh, zeroed backing, and otherwise a new QID is reserved. A queue whose last VM exited is marked for fresh backing when its binding is detached, so its backing is vacated and the queue re-armed for the next VM. Under QID pressure, a released queue serving another VM is marked for fresh backing first; an open owner's idle channel is shared preferring one that serves the requester's VM, and one serving another VM is marked for fresh backing. Nothing is shared while a release for fresh backing is in flight, so the requester waits for the vacant queue. A binding in another VM clears the preemption buffer, usage page and operand state before mapping them. RenderPool gets the same rule: ensure_backing refuses a VM other than the one its pool was created for. Render pools are per context and reused storage is already cleared before it is mapped, so this changes no current path. Signed-off-by: Jeff Queisser (cherry picked from commit c6a52ca75ac0852ad0d7279c2ead85540777fe14 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/freelist.rs | 37 ++++++++++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 27 +++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 65 +++++++++++++++++----- 3 files changed, 112 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 897407af5e609c..ab5f9e77a0f458 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -558,6 +558,8 @@ pub(crate) struct RenderPool { epoch: AtomicU64, /// Residency of the client VM the backing is aliased into. client_residency: Option>, + /// The one VM (`mmu::Vm::id`) this pool's backing is aliased into. + client_vm: u64, control: KernelObject, state: KernelObject, served: AtomicBool, @@ -610,6 +612,7 @@ impl RenderPool { let serial = RENDER_POOL_SERIAL.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)).map_err(|_| EOVERFLOW)? + 1; Ok(Self { storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), control, state, served: AtomicBool::new(true), serial, id, layout: PageLayout::M5(geometry) }) } @@ -660,6 +663,7 @@ impl RenderPool { storage, epoch: AtomicU64::new(0), client_residency: client.residency_gate(), + client_vm: client.id(), control, state, served: AtomicBool::new(false), @@ -722,13 +726,17 @@ impl RenderPool { drop(pages); } /// Re-arms a vacant pool with fresh backing built here, without the device - /// mutex. `client` is the VM of the context that owns this pool. + /// mutex. `client` is the VM of the context that owns this pool; EPERM + /// for any other VM. pub(crate) fn ensure_backing( &self, alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { + if client.id() != self.client_vm { + return Err(EPERM); + } let geometry = match self.layout { PageLayout::M5(geometry) => geometry, PageLayout::Neo { blocks, .. } => { @@ -979,8 +987,16 @@ impl Submitted<'_> { /// Idle retained queues may give their USC backing back (`pages` is `None`) /// once the firmware consumed the pool's release; the control and state /// objects the queue record names stay until processor stop. +/// +/// A backing serves one VM: its blocks hold the stack and scratch memory of +/// that VM's work, so a view of it in another VM is refused +/// ([`Self::map_client`]). Another VM gets the queue only after the backing +/// was given back and fresh, zeroed backing installed ([`Self::rearm`]). pub(crate) struct ComputePool { pages: Option, + /// The VM (`mmu::Vm::id`) the backing serves, from its first client view + /// until it is given back; zero while no client has had a view of it. + client_vm: u64, control: KernelObject, state: KernelObject, id: u16, @@ -1017,7 +1033,7 @@ impl ComputePool { )?; Self::initialize_control(&mut control, &pages, id, state.gpu_va(), [0; 8])?; fence(Ordering::SeqCst); - Ok(Self { pages: Some(pages), control, state, id, m5 }) + Ok(Self { pages: Some(pages), client_vm: 0, control, state, id, m5 }) } fn initialize_control(control: &mut KernelObject, pages: &Pages, id: u16, submitted: u64, completed: [u8; 8]) -> Result { @@ -1069,6 +1085,10 @@ impl ComputePool { pub(crate) fn populated(&self) -> bool { self.pages.is_some() } + /// Whether a view of the backing may be given to `vm`. + pub(crate) fn serves(&self, vm: &mmu::Vm) -> bool { + self.client_vm == 0 || self.client_vm == vm.id() + } /// The release record for this pool's buffer slot under the free-list /// generation the queue last published. pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { @@ -1077,6 +1097,7 @@ impl ComputePool { /// The caller has observed the firmware consume this pool's release and /// frees the returned backing after releasing the device mutex. pub(crate) fn take_pages(&mut self) -> Option { + self.client_vm = 0; self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then @@ -1093,6 +1114,8 @@ impl ComputePool { } /// Preserve both sets of pages on failure. The caller owns the consumed /// release and returns old pages for destruction outside the device mutex. + /// The grown backing's alias is in the VM the old backing served, which it + /// keeps serving. pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; core::mem::swap(&mut self.pages, pages); @@ -1106,12 +1129,18 @@ impl ComputePool { Ok(()) } + /// EPERM for a VM other than the one the backing serves. pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + if !self.serves(vm) { + return Err(EPERM); + } let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); - pages + let alias = pages .backing - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW)?; + self.client_vm = vm.id(); + Ok(alias) } pub(crate) fn reserve_submission(&self) -> Result { let counter = self.state.dword(0)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 7c30ef0e06a922..c5823382bc758b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -345,6 +345,15 @@ impl Graph { /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed /// descriptors at this point. + /// Clears the client-visible state the last owner's work left (preemption + /// buffer, usage page, operand state) before another VM is given views of + /// it. The caller proved the queue idle. + fn clear_client_state(&mut self) { + self.preempt.bytes_mut().fill(0); + self.usage.bytes_mut().fill(0); + self.operand.bytes_mut().fill(0); + fence(Ordering::SeqCst); + } fn vacate_ring(&mut self) -> Option { self.ring.take() } @@ -1677,8 +1686,10 @@ impl Queue { publication: None, }); } else { + // Another VM: the pool must be fresh backing (EPERM otherwise). let kick_alias = self.kick.map_client(context.vm())?; let pool_alias = self.pool.map_client(context.vm())?; + self.graph.clear_client_state(); let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; self.graph.set_owner(&context)?; let old_binding = self.binding.replace(binding); @@ -1762,6 +1773,8 @@ impl Queue { } let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); + // The backing served the exited VM only: give it back for fresh backing. + self.needs_fresh_backing = true; Some(binding) } /// An idle retained queue whose USC backing may be given back. @@ -1786,6 +1799,20 @@ impl Queue { pub(crate) fn needs_fresh_backing(&self) -> bool { self.needs_fresh_backing } + /// Whether this queue's pool backing may be bound for `vm`. + pub(crate) fn pool_serves(&self, vm: &mmu::Vm) -> bool { + self.pool.serves(vm) + } + /// Gives a released queue's backing back for fresh backing before reuse. + pub(crate) fn require_fresh_backing(&mut self) { + self.needs_fresh_backing = true; + } + /// A pool release is in flight, or a queue waits for one, that ends with a + /// vacant queue ready for re-arming. + pub(crate) fn vacancy_pending(&self) -> bool { + !self.growing && (self.pool_release_pending() + || (self.recycled_pending() && self.pool_phase == PoolPhase::Populated)) + } /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) /// and every killed command was settled. The QID stays killed in the firmware /// until a queue configuration with its QoS update set binds it to a scheduler diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0dbb529bd8ba8e..3faf038ee55ad6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -267,7 +267,11 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only - /// when no populated idle queue exists. + /// when no populated idle queue of the owner's VM exists. A populated pool + /// serves one VM: a queue whose backing served another VM is not bound. + /// An open owner's idle channel serving another VM is shared for fresh + /// backing; while any queue waits for such a release, none is shared, and + /// the requester waits for the vacant queue. pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; if let Some(queue) = self.compute(owner) { @@ -286,7 +290,35 @@ impl Registry { if self.qids.compute_fits(self.admission.queues() as usize) { return Ok(Rebind::Fresh); } + if self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.vacancy_pending()) + { + return Err(ENOSPC); + } let teardown = &self.teardown; + let vm = context.vm(); + // A released channel whose backing served another VM goes first: vacate it, then + // re-arm it fresh for this one. + let foreign = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.reusable() && !queue.pool_vacant() && !queue.pool_serves(vm)) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| (queue.released_at(), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + if let Some(qid) = foreign { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .require_fresh_backing(); + return Err(ENOSPC); + } let idle = self .compute .iter() @@ -294,21 +326,28 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.shareable()) .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + // A channel serving the requester's VM first, then the best fit, then the longest idle. + .map(|queue| ((!queue.pool_serves(vm), !queue.pool_fits(required), queue.last_published()), queue.qid())) .min_by_key(|(key, _)| *key) .map(|(_, qid)| qid); let Some(qid) = idle else { return Err(ENOSPC); }; admission.check()?; - let shared = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EIO)? - .share_idle()?; - if !shared { - return Err(ENOSPC); + { + let mut queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)?; + if !queue.share_idle()? { + return Err(ENOSPC); + } + if !queue.pool_serves(vm) { + // Its backing holds another VM's scratch: vacate it, then re-arm fresh. + queue.require_fresh_backing(); + return Err(ENOSPC); + } } Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) } @@ -368,7 +407,7 @@ impl Registry { vacant.get_or_insert(queue.qid()); continue; } - if !queue.pool_fits(required) { continue; } + if !queue.pool_fits(required) || !queue.pool_serves(context.vm()) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Some(Rebind::Done)); } From 7c7b892a28baecb676d9e60917723a795d133e14 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:28:08 -0800 Subject: [PATCH 111/171] drm/asahi: g17: M5: Give M5 Ultra renders the Ultra's 16 GiB render pool On the M5 Ultra (J775D) renders drew on the M5 Pro's render pool, 4 MiB blocks in a 4 GiB inventory. They now get the Ultra's own, 16 MiB blocks in 16 GiB. The J775D has rendered from the M5 Pro's pool: 4 MiB render blocks in a 4 GiB inventory. Its board now names its own, 16 MiB blocks in a 16 GiB inventory, selected at probe as on the other boards, and the build-time check of the minimum parameter buffer covers it. Signed-off-by: Jeff Queisser (cherry picked from commit dee13f16daf325e9b82cc6f1e483288f26f8ed99 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index abf2605aa862e9..4d2035a5d9ec85 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1857,7 +1857,8 @@ pub(crate) const J775D: Board = Board { register_windows: &J775D_REGISTER_WINDOWS, // The minimum parameter buffer for eight clusters. min_parameter_buffer: 0x0050_0000, - render_pool: (0x40_0000, 1 << 32), + // 16 MiB render blocks in a 16 GiB inventory. + render_pool: (0x100_0000, 1 << 34), // The power model's synthetic leakage inputs hold for every J775D: no calibration words. calibration: None, controller_defaults: Some(encode_m5_policy_controller_defaults_j775d), @@ -1875,6 +1876,7 @@ const _: () = { use super::target::min_parameter_buffer_pages; assert!(J714.min_parameter_buffer as u64 == min_parameter_buffer_pages(J714.clusters) << 15); assert!(J716C.min_parameter_buffer as u64 == min_parameter_buffer_pages(J716C.clusters) << 15); + assert!(J775D.min_parameter_buffer as u64 == min_parameter_buffer_pages(J775D.clusters) << 15); }; /// Populate the enabled-USC count from the device's sampled core masks. From 822c573049418f2a2571f5e02ed83145689f0c34 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:30:02 -0800 Subject: [PATCH 112/171] drm/asahi: g17: Reduce M5 job completion latency On the M5 Ultra (J775D) every job's completion was signalled only after the driver had drained the trace and statistics reports the firmware queued ahead of it, about five per compute completion. Completions are now signalled first. An event pass drained both roles' trace and statistics rings before it read the event rings, so every report the firmware had queued since the last pass delayed the completions behind it. On the M5 Ultra a compute kick's completion pass carries about five such records (the secondary's power-state reports and activity marks) at about 1.3 us each. On M5, read the event rings first, retire and signal what they complete, and drain the trace and statistics rings in the worker's next locked step, before recovery and replay. The reports are not scheduling inputs. G17P keeps its order. Signed-off-by: Jeff Queisser (cherry picked from commit 063ce92ee6abd2227eafddb9344b8a8f87d31145 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/event.rs | 17 +++++++++++----- drivers/gpu/drm/asahi/g17/runtime/events.rs | 22 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index a175e583923792..50df8107bbac98 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -2,6 +2,9 @@ //! Event service ordering under the device mutex. A restart ends its role's batch; a worker //! joins preparation leases and services it on a separate turn before continuing the rings. +//! On M5 only the event rings are consumed here: the firmware's trace and statistics reports +//! are drained after the completions this service retires have been signalled +//! (`Firmware::drain_reports`), so no report queued ahead of a completion delays its fence. use super::{ fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, @@ -15,6 +18,8 @@ pub(super) trait Transport { fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); + /// The reports are drained after the pass's completions are signalled (M5). + fn reports_after_signal(&self) -> bool; fn drain( &mut self, role: Role, @@ -91,11 +96,13 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for &role in host.roles() { - host.trace(role); - } - for &role in host.roles() { - host.statistics(role); + if !host.reports_after_signal() { + for &role in host.roles() { + host.trace(role); + } + for &role in host.roles() { + host.statistics(role); + } } let mut masks = [0; 2]; if !host.recovery().pending() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3a16ed7733ea88..5e0e7d8b4b7dad 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -30,6 +30,9 @@ impl event::Transport for Service<'_> { fn roles(&self) -> &'static [Role] { self.firmware.roles() } + fn reports_after_signal(&self) -> bool { + self.firmware.init.is_m5() + } fn trace(&mut self, role: Role) { let firmware = &mut *self.firmware; firmware @@ -302,6 +305,21 @@ impl Firmware { Ok(()) } + /// Drains every role's trace ring, then their statistics rings. On M5 the event worker calls + /// this after it has signalled the completions of its pass: the reports are not scheduling + /// inputs. + fn drain_reports(&mut self) { + let dev = &self.primary.state.shared.dev; + for &role in self.roles() { + self.rings.drain_trace(dev, &self.init, role); + } + for &role in self.roles() { + self.observe_statistics_power(); + self.rings.drain_statistics(&self.primary.state.shared.dev, &self.init, role); + self.observe_statistics_power(); + } + } + pub(super) fn service_runtime_events( &mut self, deferred: &mut DeferredBatch, @@ -441,6 +459,10 @@ impl crate::g17::Shared { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } + if firmware.init.is_m5() { + // The completions above are signalled; the reports that arrived with them come after. + firmware.drain_reports(); + } if recover { firmware.service_runtime_recovery(deferred)?; if !firmware.recovery.pending() { From ff8924ed8b5dba1b19e9c9feb027aff403ca0970 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:32:45 -0800 Subject: [PATCH 113/171] drm/asahi: g17: Fix M5 Ultra faults when a tiling stage is resumed On the M5 Ultra (J775D) a render whose tiling stage was stored and resumed faulted: the tiler context store, laid out for two clusters, overran its regions and the stage status page at eight. The store is now laid out from the cluster count. Every render pass has a tiler context store whose regions are sized from the GPU's cluster count: per-cluster control words, an auxiliary sub-buffer at the next 64-byte boundary, and the state stream after it, which also grows with each cluster. The fixed layout this driver used is that construction at two clusters. On the eight-cluster G17D the control words run over the auxiliary sub-buffer, the state stream lies inside the auxiliary region, and the store runs over the stage status page. A stored and resumed tiling stage then faults. Lay the store out from the cluster count (ContextStore, const-checked against the eight-cluster layout and the existing two-cluster one), and put the stage status page and the auxiliary framebuffer on the pages above its extent. The register program, the TA command and the per-pass layout take their offsets from it. The G17D forms select the cluster count; the M5 Pro keeps the two-cluster store, byte for byte. The board topology comes from an interim selection (fw/m5/topology.rs) until the compute topology lands. Signed-off-by: Jeff Queisser (cherry picked from commit 78884e19e3ce2743951a6a363f17dab1ace6624d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 1 + .../drm/asahi/g17/fw/m5/render_registers.rs | 49 +++++++++++-- drivers/gpu/drm/asahi/g17/fw/m5/topology.rs | 32 ++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 1 + .../drm/asahi/g17/queue/render/generation.rs | 73 +++++++++++++------ 5 files changed, 126 insertions(+), 30 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/topology.rs diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index 5c3eef8c54e22a..f8d4c9b5c6cd54 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -35,3 +35,4 @@ pub(crate) mod render_registers; pub(crate) mod render_assembly; pub(crate) mod target; +pub(crate) mod topology; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index 1400a687bbf593..fe83673980ac88 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -4,8 +4,40 @@ use kernel::prelude::*; use super::super::{kick::RegisterWrite, render::{RenderArgs, RenderPass, M5RegisterPrograms as RegisterPrograms}}; use super::render::M5RenderCommandInputs; -const DEFLAKE_STATE_OFFSET: u64 = 0x980; -const DEFLAKE_AUX_OFFSET: u64 = 0xc0; + +/// A pass's tiler context store for N clusters: 0x20 + 0x40 x N bytes of per-cluster control +/// words, the auxiliary sub-buffer at the next 64-byte boundary, the state stream +/// align64((N + 1) x 0x2e0) after it, and N x 0x680 + 0x800 bytes of the stream's storage +/// beyond it. Offsets from the store's base. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ContextStore { + pub(crate) auxiliary: usize, + pub(crate) stream: usize, + pub(crate) extent: usize, +} +const fn align64(bytes: usize) -> usize { + (bytes + 63) & !63 +} +impl ContextStore { + pub(crate) const fn of(clusters: usize) -> Self { + let auxiliary = align64(0x20 + 0x40 * clusters); + let stream = auxiliary + align64(0x2e0 * (clusters + 1)); + Self { auxiliary, stream, extent: stream + 0x680 * clusters + 0x800 } + } + + /// The selected board's store: laid out for the G17D's own cluster count with its forms; + /// other boards keep the two-cluster store. + pub(crate) fn selected() -> Self { + // STUB(G): super::compute::{g17d_forms, clusters}. + Self::of(if super::topology::g17d_forms() { super::topology::clusters() as usize } else { 2 }) + } +} +const _: () = { + let two = ContextStore::of(2); + assert!(two.auxiliary == 0xc0 && two.stream == 0x980 && two.extent == 0x1e80); + let eight = ContextStore::of(8); + assert!(eight.auxiliary == 0x240 && eight.stream == 0x1c40 && eight.extent == 0x5840); +}; fn utile_config(p: &RenderPass) -> u64 { (u64::from(p.utile_width / 16) << 12) | (u64::from(p.utile_height / 16) << 14) @@ -436,16 +468,17 @@ impl Workspace { self.common.write_m5(args)?; let r = args.resources; let p = args.pass; - // The TA's two deflake sub-buffers sit at fixed offsets from the - // deflake base (DEFLAKE_STATE_OFFSET, DEFLAKE_AUX_OFFSET). + // The two sub-buffers sit at the context store's offsets from its base. let base = r.deflake_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?; + let store = ContextStore::selected(); + let (state_offset, aux_offset) = (store.stream as u64, store.auxiliary as u64); let mut vdm_seen = false; for reg in &mut self.common.ta { let register = reg.reg; reg.value = match register { - 0x1c880 if vdm_seen => base.checked_add(DEFLAKE_STATE_OFFSET).ok_or(EINVAL)?, - 0x10111 | 0x1c9b0 => base.checked_add(DEFLAKE_STATE_OFFSET).ok_or(EINVAL)?, - 0x10119 | 0x1c9b8 => base.checked_add(DEFLAKE_AUX_OFFSET).ok_or(EINVAL)?, + 0x1c880 if vdm_seen => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10111 | 0x1c9b0 => base.checked_add(state_offset).ok_or(EINVAL)?, + 0x10119 | 0x1c9b8 => base.checked_add(aux_offset).ok_or(EINVAL)?, 0x1c079 | 0x1c9d8 if p.layers == 1 => r.heapmeta_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?, _ => reg.value, }; @@ -540,7 +573,7 @@ pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, let bg=usc(p.bg.offset)?;let pbg=usc(p.partial_bg.offset)?; let eot=usc(p.eot_offset)?;let peot=usc(p.partial_eot_offset)?; let deflake=r.deflake_va.checked_sub(0x10_0000_0000) - .and_then(|offset|offset.checked_add(DEFLAKE_STATE_OFFSET)).ok_or(EINVAL)?; + .and_then(|offset|offset.checked_add(ContextStore::selected().stream as u64)).ok_or(EINVAL)?; i.context_id=u32::from(r.context_id); i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; let f=&mut i.fragment; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs b/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs new file mode 100644 index 00000000000000..ffd926441754ba --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! STUB(G): the selected board's topology. Our #28/#30 port (group G) provides these as +//! `super::compute::{select_topology, clusters, g17d_forms, dispatch_control}`; this file and +//! its probe call go when that lands. + +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use super::initdata::{Board, J714, J775D}; + +static CLUSTERS: AtomicU32 = AtomicU32::new(J714.clusters); +static G17D_FORMS: AtomicBool = AtomicBool::new(false); + +/// Selects the board's topology at probe, before any command is encoded. +pub(crate) fn select_topology(board: &Board) { + CLUSTERS.store(board.clusters, Ordering::Relaxed); + G17D_FORMS.store(core::ptr::eq(board, &J775D), Ordering::Relaxed); +} + +/// The selected board's cluster count. +pub(crate) fn clusters() -> u32 { + CLUSTERS.load(Ordering::Relaxed) +} + +/// Whether the board takes the G17D host encodings measured on the M5 Ultra. +pub(crate) fn g17d_forms() -> bool { + G17D_FORMS.load(Ordering::Relaxed) +} + +/// A dispatch control word for the selected board's cluster count. +pub(crate) fn dispatch_control(base: u64) -> u64 { + if clusters() > 4 { base | 0x9c } else { base } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 02960b00ba7b29..9c7c5b20539dcf 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1423,6 +1423,7 @@ impl Gpu { return Err(ENODEV); } fw::m5::pool::select_render_pool(board); + fw::m5::topology::select_topology(board); thermal::select_board(board); } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs index e067bcb85a3a9b..5d5169752df999 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -5,7 +5,7 @@ use crate::g17::{ buffer::{self, BlockGeometry}, - fw::render::{storage, RenderPass}, + fw::{m5::render_registers::ContextStore, render::{storage, RenderPass}}, object::{Allocator, CpuMap, KernelObject, Placement}, }; use crate::{hw::t8140::{dynamic, resources as cfg}, mmu}; @@ -18,36 +18,64 @@ pub(in crate::g17) struct Layout { pub(in crate::g17) deflake: usize, pub(in crate::g17) status: usize, pub(in crate::g17) auxiliary: usize, - pub(in crate::g17) viewport: usize, + /// The viewport words the host writes before a pass, if it writes them. + pub(in crate::g17) viewport: Option, pub(in crate::g17) viewport_control: u32, } impl Layout { pub(in crate::g17) const NEO: Self = Self { bytes: storage::STATE_SIZE, deflake: storage::DEFLAKE, status: storage::TA_STATUS, auxiliary: storage::AUXILIARY, - viewport: 0x900, viewport_control: 0xc00, - }; - pub(in crate::g17) const M5: Self = Self { - bytes: 0x10000, deflake: 0, status: 0x4000, auxiliary: 0x8000, - viewport: 0x9c0, viewport_control: 0x3000, + viewport: Some(0x900), viewport_control: 0xc00, }; + /// M5: the context store at +0 ([`ContextStore`]), then the stage status page and the + /// auxiliary framebuffer, each on its own page above the store's extent. With the + /// two-cluster store the three end at 64 KiB; with the G17D's eight-cluster store at + /// 80 KiB. The host writes the viewport words into the state + /// stream. + const fn m5_for(store: ContextStore, host_viewport: bool) -> Self { + let page = mmu::UAT_PGSZ; + let status = store.extent.div_ceil(page) * page; + let auxiliary = status + page; + Self { + bytes: auxiliary + storage::AUXILIARY_SIZE, deflake: 0, status, auxiliary, + viewport: if host_viewport { Some(store.stream + 0x40) } else { None }, + viewport_control: 0x3000, + } + } + + /// The selected board's M5 layout. + pub(in crate::g17) fn m5() -> Self { + Self::m5_for(ContextStore::selected(), true) + } + pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { if state.len() < self.bytes { return Err(EINVAL); } - let viewport = storage::Viewport::new(pass.width, pass.height)? - .with_control(self.viewport_control); - let offset = self.deflake + self.viewport; - let view = state.get_mut(offset..offset + size_of::()) - .ok_or(EINVAL)?.as_mut_ptr().cast::(); - if view.align_offset(align_of::()) != 0 { return Err(EINVAL); } - // SAFETY: The complete integer-only view is aligned and in bounds. - // The caller exclusively owns this unpublished command generation. - unsafe { viewport.apply(&mut *view) }; + if let Some(viewport_offset) = self.viewport { + let viewport = storage::Viewport::new(pass.width, pass.height)? + .with_control(self.viewport_control); + let offset = self.deflake + viewport_offset; + let view = state.get_mut(offset..offset + size_of::()) + .ok_or(EINVAL)?.as_mut_ptr().cast::(); + if view.align_offset(align_of::()) != 0 { return Err(EINVAL); } + // SAFETY: The complete integer-only view is aligned and in bounds. + // The caller exclusively owns this unpublished command generation. + unsafe { viewport.apply(&mut *view) }; + } state[self.auxiliary..self.auxiliary + storage::AUXILIARY_SIZE].fill(0); Ok(()) } } +const _: () = { + let two = Layout::m5_for(ContextStore::of(2), true); + assert!(two.bytes == 0x10000 && two.status == 0x4000 && two.auxiliary == 0x8000); + assert!(matches!(two.viewport, Some(0x9c0))); + let eight = Layout::m5_for(ContextStore::of(8), true); + assert!(eight.bytes == 0x14000 && eight.status == 0x8000 && eight.auxiliary == 0xc000); +}; + enum Views { Neo { deflake: mmu::KernelMapping, @@ -66,7 +94,7 @@ pub(in crate::g17) struct Generation { impl Generation { pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { let (range, layout) = if alloc.uat.is_m5() { - (BlockGeometry::M5_COMPACT, Layout::M5) + (BlockGeometry::M5_COMPACT, Layout::m5()) } else { (dynamic::CLIENT_LOWER, Layout::NEO) }; KernelObject::new(alloc.dev, vm, Placement::In(range, cfg::TVB_PAGE), layout.bytes, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined) @@ -75,7 +103,7 @@ impl Generation { #[inline(never)] pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { let views = if alloc.uat.is_m5() { - let layout = Layout::M5; + let layout = Layout::m5(); let status_fw = state.map_range(alloc.uat.kernel_vm(), layout.status..layout.status + mmu::UAT_PGSZ, alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, @@ -105,7 +133,7 @@ impl Generation { Layout::NEO } Views::M5 { .. } => { - let layout = Layout::M5; + let layout = Layout::m5(); self.state.bytes_mut()[layout.status..layout.status + mmu::UAT_PGSZ].fill(0); layout } @@ -123,9 +151,10 @@ impl Generation { ], Views::M5 { status_fw } => { let base = self.state.gpu_va(); - [base + Layout::M5.deflake as u64, base + Layout::M5.status as u64, - base + Layout::M5.status as u64 + 8, status_fw.iova() + 8, - base + Layout::M5.auxiliary as u64] + let layout = Layout::m5(); + [base + layout.deflake as u64, base + layout.status as u64, + base + layout.status as u64 + 8, status_fw.iova() + 8, + base + layout.auxiliary as u64] } } } From 89fea8389aeff0b7831c3d8f3a032b0897c1ef36 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:33:20 -0800 Subject: [PATCH 114/171] drm/asahi: g17: Leave the M5 Ultra's GPU-owned tiler state to the GPU On the M5 Ultra (J775D) the driver wrote viewport words into tiler state that the GPU itself stores and resumes from; it now writes nothing there. No failure is recorded against the old write. Before each pass the host wrote viewport words into the context store's state stream. That stream is the hardware's: it overwrites the region when a pass is stored, and the hardware's resume reads it. On the G17D, leave the store to the GPU: with the G17D forms the host writes nothing into it. The M5 Pro keeps the write. Signed-off-by: Jeff Queisser (cherry picked from commit 9c33dc5bbfcfc889dd233ef4ee762c9f75857b97 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render/generation.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs index 5d5169752df999..2c5aadddf877ae 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -33,7 +33,8 @@ impl Layout { /// auxiliary framebuffer, each on its own page above the store's extent. With the /// two-cluster store the three end at 64 KiB; with the G17D's eight-cluster store at /// 80 KiB. The host writes the viewport words into the state - /// stream. + /// stream, except with the G17D forms: there the stream is the hardware's, which overwrites it + /// when a pass is stored, and the host leaves the store alone. const fn m5_for(store: ContextStore, host_viewport: bool) -> Self { let page = mmu::UAT_PGSZ; let status = store.extent.div_ceil(page) * page; @@ -47,7 +48,8 @@ impl Layout { /// The selected board's M5 layout. pub(in crate::g17) fn m5() -> Self { - Self::m5_for(ContextStore::selected(), true) + // STUB(G): super::compute::g17d_forms. + Self::m5_for(ContextStore::selected(), !crate::g17::fw::m5::topology::g17d_forms()) } pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { @@ -72,7 +74,7 @@ const _: () = { let two = Layout::m5_for(ContextStore::of(2), true); assert!(two.bytes == 0x10000 && two.status == 0x4000 && two.auxiliary == 0x8000); assert!(matches!(two.viewport, Some(0x9c0))); - let eight = Layout::m5_for(ContextStore::of(8), true); + let eight = Layout::m5_for(ContextStore::of(8), false); assert!(eight.bytes == 0x14000 && eight.status == 0x8000 && eight.auxiliary == 0xc000); }; From 2147fc70e4857ba85e57db1e29c9f5fb27d5656b Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:34:24 -0800 Subject: [PATCH 115/171] drm/asahi: g17: Match the M5 Ultra's tiling queue seed on rebinding No known user-visible change: on the M5 Ultra (J775D) a tiling queue rebound to a later pass carried the previous pass's payload in its completion seed, where the firmware's form is 0, and nothing is known to read it. A render pair's binding configures each queue with a completion seed (a valid flag and the seed, in the queue's configuration) that the firmware imports into its completion table: the predecessor's timestamp in bits 39:0 and a scratch payload in bits 47:40. On the G17D a tiling queue's seed carries payload 0 and a fragment queue's the low byte of the pass count. This driver seeded both stages with the previous pass's payload. On a context's first binding (payload 1) both give 0; on a later rebinding the tiling seed carried a nonzero payload. No consumer of the tiling queue's own seeded payload is known, so this is not a known fix. With the G17D forms the tiling seed is the predecessor's timestamp with payload 0; the fragment seed is unchanged, and the M5 Pro keeps its seeds. Signed-off-by: Jeff Queisser (cherry picked from commit 0809c2d267e26849f5481978272356cccf04f99e in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 1c9856080ca07e..b4bfac9817ea34 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -799,9 +799,18 @@ impl Pair { completion_seed: (covered[stage] && (!m5 || !self.installed[stage] || self.context_update)) .then(|| { - kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)) + // The predecessor's timestamp with the stage's scratch payload: the + // previous pass's on fragment; with the G17D forms 0 on tiling, as the + // G17D's tiling queues are seeded (its fragment checks read the paired + // fragment queue's payload, not this one). + // STUB(G): super::compute::g17d_forms. + let tiling_zero = + stage == 0 && m5 && crate::g17::fw::m5::topology::g17d_forms(); + kicks[stage].prev().completion_value(if tiling_zero { + 0 + } else { + payload.wrapping_sub(1) + }) }), context_id: context.id().try_into()?, free_list: FreeListBinding { From f2dee13064011407be3710e11704574dc8e57f89 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:35:06 -0800 Subject: [PATCH 116/171] drm/asahi: g17: Fix M5 Ultra fragment kicks missing an ordering dependency On the M5 Ultra (J775D) a fragment job named only the tiling barrier and its predecessor, where the firmware's form also names the fragment job before that. Later fragment jobs now carry all three. A G17D fragment kick names its implicit parent after the explicit barriers, in class 3 for an ordinary render command. After that parent it carries one more dependency while the kick queue remembers a previous kick: the same fragment queue at the remembered timestamp minus one, in class 1. So a pair's first fragment kick carries the tiling barrier and the self-parent, and every later one the tiling barrier, the previous fragment kick and the one before it. Ours stopped at the first two on every kick. The kick builder's dependency list, barriers followed by one parent, takes an optional ancestor after the parent: kick::Queue::ancestor() is the remembered previous kick's predecessor, present only while the queue has history (the first kick's synthetic parent has none, and recovery clears it), and prepare_entry refuses an ancestor that is not the queue's. With the G17D forms the render pair names it on the fragment kick, in class 1. Tiling kicks, compute kicks and the M5 Pro name none. Signed-off-by: Jeff Queisser (cherry picked from commit 5581697cd7ee6d9c0bc894a8ad078d2be802ed94 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/kick.rs | 19 +++++++++++++------ drivers/gpu/drm/asahi/g17/kick.rs | 7 +++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 + drivers/gpu/drm/asahi/g17/queue/render.rs | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index cd5e53683724da..8dd18ac550a469 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,10 +369,15 @@ pub(crate) struct KickArgs<'a> { /// queue. A queue's first kick names `timestamp.prev()` on render queues /// and [`KickTimestamp::ZERO`] on compute queues. pub(crate) parent: KickTimestamp, + /// The parent's own predecessor on the same queue, a further implicit dependency after the + /// parent: with the G17D forms a fragment kick names it while its queue remembers a previous + /// kick ([`super::super::kick::Queue::ancestor`]); `None` elsewhere. + pub(crate) ancestor: Option, /// Kicks of other queues this kick must wait for. pub(crate) barriers: &'a [KickDependency], /// Two-bit completion classes for each explicit dependency followed by - /// the implicit parent. Interfaces without typed dependencies use `None`. + /// the implicit parent and the ancestor if any. Interfaces without typed + /// dependencies use `None`. pub(crate) dependency_types: Option<&'a [u8]>, /// GPU address of the work descriptor (firmware alias). pub(crate) descriptor_va: u64, @@ -469,8 +474,10 @@ impl KickEntry { /// Builds the entry described by `args`. pub(crate) fn new(args: &KickArgs<'_>) -> Result { let barriers = args.barriers.len(); + let count = barriers + 1 + usize::from(args.ancestor.is_some()); if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX + || count > KICK_DEPENDENCIES || args.qos.class > Self::QOS_CLASS_MAX || args.priority >= Policy::PRIORITIES { @@ -498,12 +505,12 @@ impl KickEntry { for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { *slot = barrier.word(); } - dependencies[barriers] = KickDependency { - qid: args.qid, - timestamp: args.parent, + // The implicit dependencies on this queue follow the barriers: the parent, then the + // ancestor if any (`count` fits the array). + let implicit = [Some(args.parent), args.ancestor].into_iter().flatten(); + for (slot, timestamp) in dependencies[barriers..].iter_mut().zip(implicit) { + *slot = KickDependency { qid: args.qid, timestamp }.word(); } - .word(); - let count = barriers + 1; let mut dependency_types = 0; if let Some(types) = args.dependency_types { if types.len() != count || types.iter().any(|kind| *kind > 3) { diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index b74012bdd37542..46a1d8973774cf 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -721,10 +721,17 @@ impl Queue { _ => self.timestamp.prev(), }) } + /// The parent's predecessor, while this queue remembers a previous kick: a G17D fragment + /// kick names it after its pipelined self-parent. Never on the first kick's synthetic + /// parent; recovery clears the history, and with it this. + pub(crate) fn ancestor(&self) -> Option { + self.previous.map(KickTimestamp::prev) + } pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { if args.qid != self.id.qid || args.timestamp != self.timestamp || args.parent != self.parent() + || args.ancestor.is_some_and(|ancestor| Some(ancestor) != self.ancestor()) { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c5823382bc758b..c4ae8b5916b686 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2501,6 +2501,7 @@ impl Queue { qid, timestamp: kick_timestamp, parent: kick_parent, + ancestor: None, barriers: dependencies, dependency_types: self.graph.m5.then_some(&m5_dependency_types[..dependencies.len() + 1]), descriptor_va: descriptor_high, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index b4bfac9817ea34..934e4da5b8157f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -892,9 +892,22 @@ impl Pair { let barriers = render_barriers(pass.fragment_barriers, pass.stage_prefixes, dependencies, own_tiling); let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; + // With the G17D forms a fragment kick with history also names its parent's + // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). + // STUB(G): super::compute::g17d_forms. + let ancestor = if m5 && crate::g17::fw::m5::topology::g17d_forms() { + self.kicks[1].ancestor() + } else { + None + }; + let mut fragment_types = [0u8; 5 + crate::g17::hazard::PUBLICATION_BARRIERS]; + fragment_types[..barriers.fragment_count + 1] + .copy_from_slice(&barriers.fragment_types[..barriers.fragment_count + 1]); + fragment_types[barriers.fragment_count + 1] = 1; + let fragment_typed = barriers.fragment_count + 1 + usize::from(ancestor.is_some()); let barrier_types = [ &barriers.tiling_types[..barriers.tiling_count + 1], - &barriers.fragment_types[..barriers.fragment_count + 1], + &fragment_types[..fragment_typed], ]; let prepared = self.prepared.as_ref().ok_or(EIO)?; let m5_kick = match (prepared.register_arrays, prepared.kick_flags) { @@ -906,6 +919,7 @@ impl Pair { qid: qids[stage], timestamp: kicks[stage], parent: parents[stage], + ancestor: if stage == 1 { ancestor } else { None }, barriers, dependency_types: m5.then_some(barrier_types[stage]), descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), From c974ddb085978e4b46c54dfc67fa83c1c1907092 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:36:22 -0800 Subject: [PATCH 117/171] drm/asahi: g17: Fix M5 renders with many attachments being refused On M5 GPUs a render with more attachments than a list holds (16 ranges) could not be submitted: merging them into covering ranges was refused because a merged range spans unmapped gaps. Such renders are now accepted, on the M5 Ultra (J775D) as elsewhere on M5. Render validation requires every attachment range of a render to be mapped writable in the client's VM, as the G17P firmware reads the lists (at most 16 ranges each). The M5 firmware never sees them: a pass's tile stores are encoded from its surfaces, and the lists only give the host's footprint of the render (Vm::render_footprint), which orders renders that touch the same memory. A render with more attachments than a list holds could therefore not be described on M5 without dropping some: userspace that merges them into covering ranges was refused, because a merged range spans the unmapped gaps between memory objects. Add an AttachmentRule chosen with the M5 command set: on M5, accept any non-empty range that does not wrap, since a coarser footprint only orders more. G17P keeps the mapped-writable check. The limit stays 16 ranges per list. A hint reaches no further than the client's own VM. Its VA span is keyed by that VM's id, so it orders only against renders in the same VM. It reaches another client only through a shared object that is mapped in this VM inside the range, which the client could name exactly anyway. render_footprint's cost does not depend on the range size: it compares each of the list's ranges with the VM's shared bindings, and a footprint holds a fixed number of spans. A range over the whole VA cannot stall the submit path. Signed-off-by: Jeff Queisser (cherry picked from commit 0eadbd629b8bba7faf558666fe1b0b1d6d7a2b13 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/command.rs | 71 +++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index e3f42aaaa7b560..8da58c933b5c31 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -91,6 +91,35 @@ impl Attachments { } Ok(()) } + + /// An M5 render's lists: the firmware never sees them (the pass's tile stores are encoded from + /// its surfaces), so each range is only the host's ordering hint, its footprint + /// (`Vm::render_footprint`). A range may then cover several mappings and the gaps between them: + /// a coarser footprint only orders more. It must still be a range. + fn validate_hint(&self) -> Result { + for range in self.as_slice() { + if range.size == 0 || range.va.checked_add(range.size).is_none() { + return Err(EINVAL); + } + } + Ok(()) + } + + fn validate_as(&self, space: &impl AddressSpace, rule: AttachmentRule) -> Result { + match rule { + AttachmentRule::Mapped => self.validate(space), + AttachmentRule::Hint => self.validate_hint(), + } + } +} + +/// What a render's attachment lists must prove. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AttachmentRule { + /// Every range is mapped writable: the firmware reads the lists. + Mapped, + /// Every range is a non-empty range that does not wrap: the lists only order renders (M5). + Hint, } /// Where a render command's stage-prefix fields start: after the ray-query stages. @@ -444,6 +473,7 @@ impl Payload { // Everything M5 accepts beyond the G17P command set is gated on this. let m5 = variant.m5_commands; let max_tile_buffer = if m5 { M5_MAX_TILE_BUFFER } else { MAX_TILE_BUFFER }; + let attachment_rule = if m5 { AttachmentRule::Hint } else { AttachmentRule::Mapped }; match self { Self::Render(cmd, attachments, ray_query, resources) => { if cmd.flags & !(if m5 { RENDER_FLAGS } else { RENDER_FLAGS & !M5_RENDER_FLAGS }) != 0 @@ -500,7 +530,7 @@ impl Payload { window.program(space, program.usc)?; } for list in attachments { - list.validate(space)?; + list.validate_as(space, attachment_rule)?; } let pass = RenderPass { width: cmd.width_px.into(), @@ -664,3 +694,42 @@ fn refuse_ray_traversal() -> Error { kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) } + +#[kunit_tests(asahi_g17_attachments)] +mod attachment_tests { + use super::*; + + /// A VM with nothing mapped. + struct Unmapped; + + impl AddressSpace for Unmapped { + fn covers(&self, _address: u64, _size: u64, _access: Access) -> bool { + false + } + } + + fn list(ranges: &[(u64, u64)]) -> Attachments { + let mut list = Attachments::EMPTY; + for (slot, &(va, size)) in list.ranges.iter_mut().zip(ranges) { + *slot = WriteRange { va, size }; + list.count += 1; + } + list + } + + #[test] + fn an_m5_hint_may_span_unmapped_gaps() { + // One range over two attachments and the gap between them, as userspace merges more + // attachments than a list holds. + let merged = list(&[(0x1_0000_0000, 0x40_0000), (0x2_0000_0000, 0x1_0000_0000)]); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Hint), Ok(())); + assert_eq!(merged.validate_as(&Unmapped, AttachmentRule::Mapped), Err(EINVAL)); + } + + #[test] + fn an_m5_hint_is_still_a_range() { + assert_eq!(list(&[(0x1_0000_0000, 0)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[(u64::MAX - 0xfff, 0x2000)]).validate_hint(), Err(EINVAL)); + assert_eq!(list(&[]).validate_hint(), Ok(())); + } +} From a97dbd9e5c3d05ead2a39ec24a202f1e365cb7d3 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:44:19 -0800 Subject: [PATCH 118/171] drm/asahi: g17: Prepare M5 render encodings for per-board selection No change in what is written on the M5 Ultra (J775D). Its render encodings now come from the device's board rather than from a selected global, so each one follows the board it is built for. The M5 Ultra's render encodings (its tiler context store layout, the tiling completion seed and the fragment kick's ancestor) were chosen from a selected-topology static. Pass the board instead: Shared keeps the M5 board of an M5 profile, render pairs hand it to their memory, and the render controls carry it, so every encoding is a function of the board it is built for. Signed-off-by: Jeff Queisser (cherry picked from commit 9ef3df34ed8c5040b243cc237a854c9c0570cb1d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/mod.rs | 1 - .../drm/asahi/g17/fw/m5/render_registers.rs | 15 ++++---- drivers/gpu/drm/asahi/g17/fw/m5/topology.rs | 32 ----------------- drivers/gpu/drm/asahi/g17/mod.rs | 1 - drivers/gpu/drm/asahi/g17/queue/render.rs | 16 ++++++--- .../drm/asahi/g17/queue/render/generation.rs | 35 +++++++++++-------- .../g17/queue/render/m5_render_submission.rs | 6 +++- .../gpu/drm/asahi/g17/queue/render/memory.rs | 25 ++++++++----- .../gpu/drm/asahi/g17/queue/render/parked.rs | 4 +++ .../drm/asahi/g17/runtime/render/prepare.rs | 1 + 10 files changed, 65 insertions(+), 71 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g17/fw/m5/topology.rs diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs index f8d4c9b5c6cd54..5c3eef8c54e22a 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/mod.rs @@ -35,4 +35,3 @@ pub(crate) mod render_registers; pub(crate) mod render_assembly; pub(crate) mod target; -pub(crate) mod topology; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index fe83673980ac88..9d4111f2e628e3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -25,11 +25,10 @@ impl ContextStore { Self { auxiliary, stream, extent: stream + 0x680 * clusters + 0x800 } } - /// The selected board's store: laid out for the G17D's own cluster count with its forms; - /// other boards keep the two-cluster store. - pub(crate) fn selected() -> Self { - // STUB(G): super::compute::{g17d_forms, clusters}. - Self::of(if super::topology::g17d_forms() { super::topology::clusters() as usize } else { 2 }) + /// `board`'s store: laid out for the G17D's own cluster count with its forms; other boards + /// keep the two-cluster store. + pub(crate) const fn selected(board: &super::initdata::Board) -> Self { + Self::of(if board.g17d_forms() { board.clusters as usize } else { 2 }) } } const _: () = { @@ -57,6 +56,8 @@ pub(crate) struct M5RenderStageControls { #[derive(Copy, Clone)] pub(crate) struct M5RenderControls { + /// The board the render's encodings are for. + pub(crate) board: &'static super::initdata::Board, /// Encoded event-command state for the initial fragment register list. pub(crate) fragment_event_control: u64, pub(crate) ta: M5RenderStageControls, @@ -470,7 +471,7 @@ impl Workspace { let p = args.pass; // The two sub-buffers sit at the context store's offsets from its base. let base = r.deflake_va.checked_sub(0x10_0000_0000).ok_or(EINVAL)?; - let store = ContextStore::selected(); + let store = ContextStore::selected(controls.board); let (state_offset, aux_offset) = (store.stream as u64, store.auxiliary as u64); let mut vdm_seen = false; for reg in &mut self.common.ta { @@ -573,7 +574,7 @@ pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, let bg=usc(p.bg.offset)?;let pbg=usc(p.partial_bg.offset)?; let eot=usc(p.eot_offset)?;let peot=usc(p.partial_eot_offset)?; let deflake=r.deflake_va.checked_sub(0x10_0000_0000) - .and_then(|offset|offset.checked_add(ContextStore::selected().stream as u64)).ok_or(EINVAL)?; + .and_then(|offset|offset.checked_add(ContextStore::selected(c.board).stream as u64)).ok_or(EINVAL)?; i.context_id=u32::from(r.context_id); i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; let f=&mut i.fragment; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs b/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs deleted file mode 100644 index ffd926441754ba..00000000000000 --- a/drivers/gpu/drm/asahi/g17/fw/m5/topology.rs +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! STUB(G): the selected board's topology. Our #28/#30 port (group G) provides these as -//! `super::compute::{select_topology, clusters, g17d_forms, dispatch_control}`; this file and -//! its probe call go when that lands. - -use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; - -use super::initdata::{Board, J714, J775D}; - -static CLUSTERS: AtomicU32 = AtomicU32::new(J714.clusters); -static G17D_FORMS: AtomicBool = AtomicBool::new(false); - -/// Selects the board's topology at probe, before any command is encoded. -pub(crate) fn select_topology(board: &Board) { - CLUSTERS.store(board.clusters, Ordering::Relaxed); - G17D_FORMS.store(core::ptr::eq(board, &J775D), Ordering::Relaxed); -} - -/// The selected board's cluster count. -pub(crate) fn clusters() -> u32 { - CLUSTERS.load(Ordering::Relaxed) -} - -/// Whether the board takes the G17D host encodings measured on the M5 Ultra. -pub(crate) fn g17d_forms() -> bool { - G17D_FORMS.load(Ordering::Relaxed) -} - -/// A dispatch control word for the selected board's cluster count. -pub(crate) fn dispatch_control(base: u64) -> u64 { - if clusters() > 4 { base | 0x9c } else { base } -} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9c7c5b20539dcf..02960b00ba7b29 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1423,7 +1423,6 @@ impl Gpu { return Err(ENODEV); } fw::m5::pool::select_render_pool(board); - fw::m5::topology::select_topology(board); thermal::select_board(board); } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 934e4da5b8157f..ae2d1b2683334b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -204,6 +204,7 @@ pub(crate) struct Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -271,6 +272,7 @@ impl Pair { deflake: abi::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, ) -> Result> { let pool_pin = free_list.pin_id()?; if alloc.uat.is_m5() { @@ -312,6 +314,7 @@ impl Pair { 0, word, variant, + board, )?; storage.write_init(kernel::try_init!(Self { pool_pin, @@ -331,6 +334,7 @@ impl Pair { deflake, descriptor_flags, variant, + board, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -375,6 +379,10 @@ impl Pair { pub(crate) fn buffer_id(&self) -> u8 { self.memory.manager.buffer_id() } + /// Whether the pair's M5 board takes the G17D forms. + fn g17d_forms(&self) -> bool { + self.board.is_some_and(|board| board.g17d_forms()) + } /// The parameter-buffer state an M5 PB token is bound to. pub(crate) fn m5_pb_state_va(&self) -> Option { self.memory.graph.is_m5().then(|| self.memory.manager.state_va()) @@ -803,9 +811,7 @@ impl Pair { // previous pass's on fragment; with the G17D forms 0 on tiling, as the // G17D's tiling queues are seeded (its fragment checks read the paired // fragment queue's payload, not this one). - // STUB(G): super::compute::g17d_forms. - let tiling_zero = - stage == 0 && m5 && crate::g17::fw::m5::topology::g17d_forms(); + let tiling_zero = stage == 0 && m5 && self.g17d_forms(); kicks[stage].prev().completion_value(if tiling_zero { 0 } else { @@ -894,8 +900,7 @@ impl Pair { let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; // With the G17D forms a fragment kick with history also names its parent's // predecessor, in class 1, after the self-parent (classes [1, 3, 1] for a lone pass). - // STUB(G): super::compute::g17d_forms. - let ancestor = if m5 && crate::g17::fw::m5::topology::g17d_forms() { + let ancestor = if m5 && self.g17d_forms() { self.kicks[1].ancestor() } else { None @@ -2829,6 +2834,7 @@ impl Pair { self.ordinal, first.completion.work_state()?.word(), self.variant, + self.board, )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs index 2c5aadddf877ae..d03857cc10a560 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/generation.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/generation.rs @@ -5,7 +5,10 @@ use crate::g17::{ buffer::{self, BlockGeometry}, - fw::{m5::render_registers::ContextStore, render::{storage, RenderPass}}, + fw::{ + m5::{initdata::Board, render_registers::ContextStore}, + render::{storage, RenderPass}, + }, object::{Allocator, CpuMap, KernelObject, Placement}, }; use crate::{hw::t8140::{dynamic, resources as cfg}, mmu}; @@ -46,10 +49,9 @@ impl Layout { } } - /// The selected board's M5 layout. - pub(in crate::g17) fn m5() -> Self { - // STUB(G): super::compute::g17d_forms. - Self::m5_for(ContextStore::selected(), !crate::g17::fw::m5::topology::g17d_forms()) + /// `board`'s M5 layout. + pub(in crate::g17) const fn m5(board: &Board) -> Self { + Self::m5_for(ContextStore::selected(board), !board.g17d_forms()) } pub(in crate::g17) fn prepare(self, state: &mut [u8], pass: &RenderPass) -> Result { @@ -85,7 +87,7 @@ enum Views { fragment_client: mmu::KernelMapping, fragment: KernelObject, }, - M5 { status_fw: mmu::KernelMapping }, + M5 { status_fw: mmu::KernelMapping, board: &'static Board }, } pub(in crate::g17) struct Generation { @@ -94,23 +96,26 @@ pub(in crate::g17) struct Generation { pub(super) ordinal: Option, } impl Generation { - pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + pub(in crate::g17) fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, + board: Option<&'static Board>) -> Result { let (range, layout) = if alloc.uat.is_m5() { - (BlockGeometry::M5_COMPACT, Layout::m5()) + (BlockGeometry::M5_COMPACT, Layout::m5(board.ok_or(EINVAL)?)) } else { (dynamic::CLIENT_LOWER, Layout::NEO) }; KernelObject::new(alloc.dev, vm, Placement::In(range, cfg::TVB_PAGE), layout.bytes, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined) } #[inline(never)] - pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { + pub(in crate::g17) fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result> { let views = if alloc.uat.is_m5() { - let layout = Layout::m5(); + let board = board.ok_or(EINVAL)?; + let layout = Layout::m5(board); let status_fw = state.map_range(alloc.uat.kernel_vm(), layout.status..layout.status + mmu::UAT_PGSZ, alloc.uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, 0)?; - Views::M5 { status_fw } + Views::M5 { status_fw, board } } else { let deflake = buffer::compact_alias(&state, vm, storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, @@ -134,8 +139,8 @@ impl Generation { fragment.bytes_mut().fill(0); Layout::NEO } - Views::M5 { .. } => { - let layout = Layout::m5(); + Views::M5 { board, .. } => { + let layout = Layout::m5(board); self.state.bytes_mut()[layout.status..layout.status + mmu::UAT_PGSZ].fill(0); layout } @@ -151,9 +156,9 @@ impl Generation { deflake.iova(), self.state.gpu_va() + storage::TA_STATUS as u64, fragment_client.iova(), fragment.gpu_va(), auxiliary.iova(), ], - Views::M5 { status_fw } => { + Views::M5 { status_fw, board } => { let base = self.state.gpu_va(); - let layout = Layout::m5(); + let layout = Layout::m5(board); [base + layout.deflake as u64, base + layout.status as u64, base + layout.status as u64 + 8, status_fw.iova() + 8, base + layout.auxiliary as u64] diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs index 9f40046215d11c..935e62593f0163 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -141,13 +141,17 @@ pub(crate) fn layout(command: &Validated, clusters: u32, programs: RegisterProgr layout.with_m5_slices(pass, clusters, groups, 64, bytes) } -pub(crate) fn controls(pass: &RenderPass) -> Result { +pub(crate) fn controls( + pass: &RenderPass, + board: &'static crate::g17::fw::m5::initdata::Board, +) -> Result { validate(pass)?; let stage = |resource_flags| regs::M5RenderStageControls { scs: crate::g17::render_resources::StageScratch::NONE, resource_flags, resource_control: 0x200000001, ptc_max: 96, work_key: 0, }; Ok(regs::M5RenderControls { + board, fragment_event_control: regs::fragment_event_control(true, 1), ta: stage(0x4054b), fragment: stage(0x104054b), ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 189b67010ec5c1..b129f521cc34ca 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -19,6 +19,7 @@ use crate::{ }; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; +use crate::g17::fw::m5::initdata::Board; pub(super) const RECORDS: usize = 32; pub(super) const TA_BYTES: usize = 0x14000; @@ -347,16 +348,18 @@ pub(super) enum Generation { M5(KBox), } impl Generation { - fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm, board: Option<&'static Board>) + -> Result { if alloc.uat.is_m5() { - super::generation::Generation::allocate_state(alloc, vm) + super::generation::Generation::allocate_state(alloc, vm, board) } else { NeoGeneration::allocate_state(alloc, vm) } } - fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result { + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject, + board: Option<&'static Board>) -> Result { Ok(if alloc.uat.is_m5() { - Self::M5(super::generation::Generation::new(alloc, vm, state)?) + Self::M5(super::generation::Generation::new(alloc, vm, state, board)?) } else { Self::Neo(NeoGeneration::new(alloc, vm, state)?) }) @@ -511,6 +514,8 @@ pub(super) struct Memory { m5_workspace: Option>, pub(super) context: Arc, pub(super) installed_pool: Option, + /// The M5 board ([`crate::g17::Shared::m5_board`]); `None` off M5. + pub(super) board: Option<&'static Board>, } impl Memory { #[inline(never)] @@ -530,6 +535,7 @@ impl Memory { ordinal: u64, word: u32, variant: &'static fw::Variant, + board: Option<&'static Board>, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); @@ -538,7 +544,7 @@ impl Memory { let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; - let state = Generation::allocate_state(alloc, vm)?; + let state = Generation::allocate_state(alloc, vm, board)?; let (tvb, mappings, scratch, discard) = if m5 { let geometry = super::m5_render_submission::parameter_geometry(layout.tvb_blocks)?; let bytes = fw::m5::pool::parameter_scratch_stride(geometry.capacity, 0) @@ -561,11 +567,11 @@ impl Memory { (ordinal ^ 1) as usize % count }; let mut generations = core::array::from_fn(|_| None); - generations[initial] = Some(Generation::new(alloc, vm, state)?); + generations[initial] = Some(Generation::new(alloc, vm, state, board)?); for offset in 1..count { let index = (initial + offset) % count; - let state = Generation::allocate_state(alloc, vm)?; - generations[index] = Some(Generation::new(alloc, vm, state)?); + let state = Generation::allocate_state(alloc, vm, board)?; + generations[index] = Some(Generation::new(alloc, vm, state, board)?); } let timestamps = alloc.kernel( mmu::UAT_PGSZ, @@ -641,6 +647,7 @@ impl Memory { m5_workspace, context, installed_pool: None, + board, }))?, mappings, )) @@ -780,7 +787,7 @@ impl Memory { if !self.graph.m5 { return Err(EINVAL); } - let mut controls = render_submission::controls(args.pass)?; + let mut controls = render_submission::controls(args.pass, self.board.ok_or(EINVAL)?)?; if let Some(stages) = plan.and_then(|p| p.stages) { controls.ta.scs = stages[0].scs; controls.fragment.scs = stages[1].scs; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 3c18c58f646cb0..73186d9077303f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -27,6 +27,7 @@ pub(crate) struct ParkedPair { deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + board: Option<&'static crate::g17::fw::m5::initdata::Board>, installed: [bool; 2], recovery_generation: u64, registered: Option, @@ -95,6 +96,7 @@ impl Pair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, @@ -178,6 +180,7 @@ impl ParkedPair { self.ordinal, word, self.variant, + self.board, )?; Ok(Prepared { pool_pin, @@ -245,6 +248,7 @@ impl ParkedPair { deflake: self.deflake, descriptor_flags: self.descriptor_flags, variant: self.variant, + board: self.board, installed: self.installed, configs: [None; 2], cursors: [0; 2], diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 863e7a59e118e3..40d1e6fec4dc5e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -475,6 +475,7 @@ impl super::super::Backend { resources.config.deflake, resources.config.descriptor_flags, self.shared.variant, + self.shared.m5_board, ) })(); let (mut built, error) = match built { From 7c5b6ca1662d4c132da5b5114606c2aed1d6101b Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:44:40 -0800 Subject: [PATCH 119/171] drm/asahi: g17: Fix GPU firmware restarts during M5 Ultra renders On the M5 Ultra Mac Studio (J775D) render passes made the GPU firmware restart: the tiler and the fragment stage disagreed on the parameter-buffer records, and the fragment stage read record fields as addresses. Renders now run without restarts; J714 and J716c are unchanged. Above four clusters compute's dispatch control carries the cluster-topology term, but the render path wrote the tiler's and the fragment stage's dispatch controls without it. On the eight-cluster G17D the tiler and its fragment-side readers then disagree about the parameter-buffer records: the fragment fetchers read fields of the GPU's own records as addresses and the firmware restarts. Both render words now take the term through dispatch_control, so one rule covers both engines. It is a no-op at four clusters or fewer, so J714 and J716c are unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 40af808326ac0c38450ede88c95d2c2d1ecc78bc in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index 9d4111f2e628e3..35bd2cfed38bb3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -166,7 +166,10 @@ pub(crate) fn build_ta_registers_into( out.push(0x0d499, controls.ta.resource_flags)?; out.push(0x0d419, controls.ta.resource_control)?; out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; - register.value = controls.ta_tile_control; + // Above four clusters the tiler's control word takes the topology term, as + // compute's dispatch control does; the fragment side's must agree with it. + register.value = + super::compute::dispatch_control(controls.board, controls.ta_tile_control); } 0x10171 => { for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] @@ -231,7 +234,10 @@ pub(crate) fn build_fragment_registers_into( 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, 0x100b1 => register.reg = 0x16469, 0x100d9 => register.reg = 0x16471, - 0x100b8 => register.reg = 0x16138, + 0x100b8 => { + register.reg = 0x16138; + register.value = super::compute::dispatch_control(controls.board, register.value); + } 0x10791 => { let scratch = controls.fragment.scs.scratch; register.value = if scratch.enabled() { From ac6699949bb1744429dac2952f358628f0f4e606 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:44:42 -0800 Subject: [PATCH 120/171] drm/asahi: g17: Fix M5 Ultra render passes flagged as recycled passes On the M5 Ultra (J775D) every render pass reached the firmware with its pass flag clear, the value for a recycled pass, because nothing set it. Ordinary passes are now flagged as such. The G17D's fragment command has a pass flag that is 1 in every ordinary pass and 0 only in a recycled pass. It follows the render context, not the attachment count, the load/store actions, the sample count, layers or the tile size, and the fragment command is its only consumer. This driver emitted the flag from a descriptor byte that nothing set, so it was always 0. With the G17D forms (Board::g17d_forms), set it for every render pass. J714 and J716c keep 0. Signed-off-by: Jeff Queisser (cherry picked from commit ceb2d8e714164eca4ff8c110419a6f088f19a1cf in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index 35bd2cfed38bb3..cf7aaf25a00cb7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -585,6 +585,10 @@ pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; let f=&mut i.fragment; f.empty_tile_flags[0]=u8::from(p.process_empty_tiles); + // The fragment command's pass flag: 1 in every ordinary render pass with the G17D forms. It + // follows the render context, not the attachments, sample count, layers or tile size. J714 + // keeps 0. + f.fragment_pass_flag = u8::from(c.board.g17d_forms()); // The admitted ordinary single-sample profile has one stored colour // attachment. Its clear-intent flag follows empty-tile intent; a LOAD // pass uses zero here, while partial-store ZLS addresses can From 6c820643bc3d1e39bea1aba024083d67137b2289 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Wed, 4 Nov 2026 23:44:44 -0800 Subject: [PATCH 121/171] drm/asahi: g17: M5: Support parameter-buffer growth on the M5 Ultra On the M5 Ultra (J775D) a render pass that overflowed its parameter buffer was always rendered in parts. The firmware now asks the driver for more blocks at the overflow and continues the pass with them. A large M5 render pass can overflow its pair's parameter buffer. The firmware then renders the pass partially, unless the TA command asks it to request blocks from the host instead. With that bit set, it sends a growth request at the overflow, waits for the reply and takes the newly committed blocks into the live ring. The shared growth worker already serves M5 pairs: they are render entries, and grow_owner matches them by buffer and VM. Their target slices are already sized for the whole parameter-buffer inventory (m5_render_submission::layout), so growth during a pass never outgrows the storage the pass names. Only the request bit was left clear. Set it with the G17D forms. J714 and J716c keep their form, which renders partially. Signed-off-by: Jeff Queisser (cherry picked from commit 5e4a6ade2fe5832657bbcf0718f4f111d94a2558 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs index cf7aaf25a00cb7..15e1887cee2727 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/render_registers.rs @@ -583,6 +583,10 @@ pub(crate) fn apply_pass(args: &RenderArgs<'_>, c: &M5RenderControls, .and_then(|offset|offset.checked_add(ContextStore::selected(c.board).stream as u64)).ok_or(EINVAL)?; i.context_id=u32::from(r.context_id); i.ta.tpc_va=r.tpc_va;i.ta.tpc_size=r.tpc_size;i.ta.deflake_state_va=deflake; + // With the G17D forms the tiler asks the host for parameter-buffer blocks when the buffer + // runs out (a growth request, served by the shared growth worker) instead of rendering + // partially. J714 keeps 0. + i.ta.pb_growth_request = u8::from(c.board.g17d_forms()); let f=&mut i.fragment; f.empty_tile_flags[0]=u8::from(p.process_empty_tiles); // The fragment command's pass flag: 1 in every ordinary render pass with the G17D forms. It From 2b6b2badd85315d75c286623f86f8feb0fc15e9b Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:17:17 -0800 Subject: [PATCH 122/171] drm/asahi: g17: M5: Fix M5 misreading firmware event 7 as a compute fault On the M5 Ultra (J775D) the driver decoded firmware event type 7 with another firmware's compute-fault layout and applied it to M5 queues, so a misread record could fault the wrong work or let a held queue's memory go early. M5 now treats it as an unknown event. Firmware event type 7 is a compute fault witness only where its layout was qualified. M5's is not, so M5 consumes it as an unknown event, as 25G83 does. Signed-off-by: Jeff Queisser [Eduard Moldovan: conflict resolved by taking aurora-silicon's version of g17/fw/channels.rs] (cherry picked from commit 387b238ebfce4a6479b40a4bdcb1422ac5593e65 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/channel.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 079bc954d6f24f..6fbc75c4835502 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -281,7 +281,7 @@ impl Events { let words = init.status_read_words(role, offset)?; super::trace::report(role as u8, 2, &words); let record = EventRecord::from_words(words); - let event = record.decode_for(self.graph); + let event = record.decode_for(self.graph, init.is_m5()); fence(Ordering::SeqCst); consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 81fee87f406213..2d145f4511a2d7 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -165,6 +165,16 @@ impl EventRecord { unsafe { self.words[0] } } + /// Type 7 is a retirement witness only on the firmware whose layout and + /// visibility semantics are qualified. 25G83 and M5 must keep consuming it as an + /// unknown event until their own fault contract is established. + pub(crate) fn decode_for(&self, graph: super::variant::InitGraph, m5: bool) -> Result { + if (m5 || graph == super::variant::InitGraph::V25G83) && self.tag() == Self::COMPUTE_FAULT { + return Ok(Event::Unknown(Self::COMPUTE_FAULT)); + } + self.decode() + } + /// Validates the record's fields without interpreting resource ownership. /// A malformed known event is an error, unlike an unknown tag. pub(crate) fn decode(&self) -> Result { @@ -564,6 +574,18 @@ mod tests { assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); } + #[test] + fn v25g83_does_not_accept_an_unqualified_fault_witness() { + use super::super::variant::InitGraph; + let fault = record(&[7, 1, 2, 0x39, 0x300]); + assert_eq!(fault.decode_for(InitGraph::G17P, false), + Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); + assert_eq!(fault.decode_for(InitGraph::G17P, true), Ok(Event::Unknown(7))); + assert_eq!(fault.decode_for(InitGraph::V25G83, false), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83, false), + Ok(Event::Unknown(7))); + } + #[test] fn an_unrecognized_type_7_layout_stays_unknown() { assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7))); From 3cdd1e07377a9744e74540a2a0375b6881e7d7d2 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:17:19 -0800 Subject: [PATCH 123/171] drm/asahi: g17: M5: Fix spurious and delayed kills of M5 contexts On the M5 Ultra (J775D) the driver also killed closed render-only contexts and delayed a failed context's kill while others' compute ran, two behaviours measured only on Neo. M5 now kills a context only when its compute queue exits, at once. Two context-kill behaviours measured on Neo also ran on M5: a context kill for a closed render-only context, and the wait before a failed context's kill while other contexts' compute runs. M5 issues a context kill only from a compute queue's exit, and publishes it at once. Signed-off-by: Jeff Queisser (cherry picked from commit fee6428bc43f5b9c8a26fc7d7aaa1d19ce6a8558 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 9e12842f41c02c..a354ead64dcf95 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -326,8 +326,11 @@ impl Firmware { /// retired on the firmware's fault record, so no compute queue is left to carry a kill, /// yet those pairs are released only after one: without this the context, its pairs and /// its memory stay until the processors stop. + /// + /// M5 has none: it issues a context kill only from a compute queue's exit, so a + /// render-only M5 context is never killed. fn render_kill_candidate(&self, index: usize, context: &Arc, age: u64) -> bool { - render_kill_eligible( + !self.init.is_m5() && render_kill_eligible( age, self.queues.teardown.render_kill_published(index) || context.kill_published(), self.queues.render.executed_slots(context) != 0, @@ -428,8 +431,11 @@ impl Firmware { /// minutes of a five-lane load with 67 faults), even with nothing freed before the kill's /// acknowledgement; such a kill takes the next moment without foreign compute instead. /// Render work is not waited for. A GPU that retired nothing for [`LIVENESS_NS`] has - /// stopped. + /// stopped. The wait is Neo's; M5 publishes its kills at once. fn foreign_compute_running(&self, context: &Arc, now: u64) -> bool { + if self.init.is_m5() { + return false; + } let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); !stalled && self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { From b74b4fa3e5a19087a5cd45c9cc7a34267051e294 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:35:46 -0800 Subject: [PATCH 124/171] drm/asahi: g17: Fix M5 renders failing after their render pool grows On the M5 Ultra (J775D) a render that needed a bigger pool tore down its own live context: its later submissions failed and the firmware was told to release a scheduler page installed queues still used. The current Mesa never grows a pool this way, so it was not seen in testing. A render that needs a bigger pool rebinds its pair to the same context. The rebind displaced the pair's context unconditionally, so a live context was marked draining: its graphs were detached and its pool dropped while it kept submitting, and the firmware was told to release a scheduler page that installed queues still named. Only another context is displaced now. Signed-off-by: Jeff Queisser (cherry picked from commit b191921aef74d5eb34635f0c9dc1357c4103a0ea in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../gpu/drm/asahi/g17/runtime/render/prepare.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 40d1e6fec4dc5e..85cb8903ad7d92 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -714,11 +714,15 @@ impl super::super::Backend { { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; + // A rebind of the same context (a render that needs a bigger pool) keeps its + // live scheduler state; only another context's is displaced. + if !pair.matches(self.owner, &self.context)? { + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + } pair.bind_owner( prepared.as_mut().ok_or(EIO)?, alloc, From fa1c8f9e404c3f887d0b48a7257531fafd5b6e9f Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:36:02 -0800 Subject: [PATCH 125/171] drm/asahi: g17: M5: Fix freeing memory the M5 GPU may still write at unbind On the M5 Ultra (J775D), unbinding or shutting down the GPU driver freed the firmware session's memory once the GPU processors stopped, though nothing shows its engines have; an engine still running could write into memory the system had reused. The session's memory is now kept. A stopped firmware session was freed once the processors stopped. Nothing on M5 proves its GPU engines retired by then, and engines still running can write into the freed pages. Without that witness (InitData::has_engine_retirement_witness, false on M5), the session is kept. Signed-off-by: Jeff Queisser (cherry picked from commit 82e60be68ab3ddd6b79cf9702d07adf6b05c88d1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6c383b6e35be4e..006f04c31d0071 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -422,6 +422,12 @@ impl InitData { Ok(()) } + /// Whether a stopped session can prove its GPU engines retired, so that its pages may be + /// freed. M5 has no engine-retirement witness yet. + pub(super) fn has_engine_retirement_witness(&self) -> bool { + !self.m5 + } + /// The statistics record tags the family's firmware emits. pub(super) fn statistics_types(&self) -> u64 { if self.m5 { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 3faf038ee55ad6..74d764909c0ae0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2388,6 +2388,12 @@ impl super::Shared { state.take() }; if let Some(mut firmware) = firmware { + if !firmware.init.has_engine_retirement_witness() { + // Engines may still write into the session's pages after the processors + // stop, and nothing here proves they have stopped: keep the session. + core::mem::forget(firmware); + return; + } firmware.finish_stopped_queues(); // RTKit destruction still dereferences the mailbox provider. drop(firmware); From e8cff84ce3ce27d82ac330a963d41bd303cbad45 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:36:03 -0800 Subject: [PATCH 126/171] drm/asahi: g17: M5: Fix compute queues wedging on a full work ring On the M5 Ultra (J775D), if the work ring compute shares with render was full when a compute queue was installed, the queue never installed and every job on it failed at its deadline for the life of the queue. A full ring is now refused before anything changes. M5 compute takes its slot on the work ring M5 render shares, and nothing checked that ring before a publication changed state. On a full ring the queue's install record was appended and the install marked, then the ring refused the slot; every retry then waited for an install the firmware was never told of, and each job failed at its deadline. The ring's free credits (InitData::work_ready) are checked first now, under the same mutex as the publication. Signed-off-by: Jeff Queisser (cherry picked from commit f9596124552931d7c2fa2bb283e4f3cfe205a9c0 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/runtime.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 74d764909c0ae0..bd29bd9f8b688a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1629,9 +1629,18 @@ impl super::Firmware { } // No descriptor, accounting or inner producer changes precede this // reservation; dropping it unused returns the work-ring slot. M5 compute takes its - // credit from the shared ring under this mutex instead (InitData::publish_work). - let outer = if self.init.is_m5() { None } - else { Some(self.port.reserve_compute(packet.context.policy().priority())?) }; + // slot on the work ring M5 render shares, under this mutex (InitData::publish_work): + // a full ring is refused here, before a queue install or kick changes any state. + let outer = if self.init.is_m5() { + if !self.init.work_ready(packet.context.policy().priority(), + super::fw::queue::DataMaster::Compute)? + { + return Err(EAGAIN); + } + None + } else { + Some(self.port.reserve_compute(packet.context.policy().priority())?) + }; let queues = &mut self.queues; let mut queue = queues .compute From c2de5985dd3d5da0b95fc2a9d50f6ad9af655f64 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:36:14 -0800 Subject: [PATCH 127/171] drm/asahi: g17: M5: Fix M5 renders silently dropping their ray queries On the M5 Ultra (J775D) the driver reported render ray-query support and accepted renders with ray-query stages, then encoded them with no ray-query state, so the ray queries were silently dropped. The feature is no longer reported and such renders are refused. The M5 firmware variant inherited ray traversal from G17P, so M5 reported the render ray-query feature and admitted renders with ray-query stages, while the M5 render encoder writes no ray-query state and dropped them. A variant now says whether renders may carry ray-query stages; M5's does not, so the feature is not reported and such renders are refused. Signed-off-by: Jeff Queisser (cherry picked from commit 78723c7af94021e190d53ca3908b04f4b2cc7d5d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/command.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/variant.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/mod.rs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 8da58c933b5c31..3583d89eb2f89c 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -584,7 +584,7 @@ impl Payload { }; for stage in ray_query { if (stage.intersection.enabled() || stage.scratch.enabled()) - && !variant.ray_traversal + && !(variant.ray_traversal && variant.render_ray_query) { return Err(refuse_ray_traversal()); } diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 83fd6515a1bfca..95a052fc5f5d9e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -64,6 +64,9 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Renders may carry ray-query stages. The M5 command set has none: it encodes no render + /// ray-query state. + pub(crate) render_ray_query: bool, /// Tiling descriptors report whether the fragment queue still had earlier kicks /// outstanding (`fragment_busy`). pub(crate) fragment_busy: bool, @@ -121,6 +124,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + render_ray_query: true, fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, @@ -152,6 +156,7 @@ pub(crate) const V25G83: Variant = Variant { // A fragment stage with the few-primitives flag hangs. few_primitives: false, ray_traversal: true, + render_ray_query: true, // When a tiling pass runs out of memory with this set, the firmware resumes it without // adding memory and the tiler stops silently; macOS always writes 0. fragment_busy: false, @@ -169,6 +174,7 @@ pub(crate) const V25G83: Variant = Variant { /// command set). pub(crate) const V26A428_M5: Variant = Variant { m5_commands: true, + render_ray_query: false, registration_power_hold: false, ..V26A428 }; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 02960b00ba7b29..9e19b853bcd1d5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2106,7 +2106,7 @@ impl gpu::Gpu for Gpu { if variant.few_primitives { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } - if variant.ray_traversal { + if variant.render_ray_query { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); } From b2275b198d2ea8d21f240dd85a90bfb457f8bf75 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:02:01 -0800 Subject: [PATCH 128/171] drm/asahi: g17: Prevent M5 firmware crashes from InitData objects M5 lacks On the M5 Ultra (J775D), a shared G17 path that used a completion ring or render list the M5 graph does not have wrote into the firmware's private data instead; during the port this crashed the GPU firmware at the first compute job. Such a use is now an error. The completion rings and the render page and run lists are set only by the families that have them: M5 sets four completion rings and no render lists, G17P two rings and both lists. An unset handle defaulted to 0 and named object 0, the private cluster on M5. Each is now an Option, so a use without the object is an error. Signed-off-by: Jeff Queisser (cherry picked from commit 601ed2953cfa411c725f00977954ff3e1f07ea2a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 30 +++++++++++++----------- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 2 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 006f04c31d0071..f246256ea28ff1 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -205,10 +205,12 @@ pub(super) struct InitData { compute_scheduler: Option, compute_active: core::cell::Cell, event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], - completion_rings: [usize; 4], + /// Completion-ring objects; a family sets only the rings it has. + completion_rings: [Option; 4], stamps: [usize; 2], - render_pages: usize, - render_runs: usize, + /// The G17P render page and run lists (client aliases). M5 has none. + render_pages: Option, + render_runs: Option, render_blocks: core::ops::Range, } @@ -306,10 +308,10 @@ impl InitData { compute_scheduler: None, compute_active: core::cell::Cell::new(false), event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], - completion_rings: [0; 4], + completion_rings: [None; 4], stamps: [0; 2], - render_pages: 0, - render_runs: 0, + render_pages: None, + render_runs: None, render_blocks: 0..0, } } @@ -555,7 +557,7 @@ impl InitData { } for slot in 0..2 { mappings.push( - self.object(self.completion_rings[slot])?.map_alias( + self.object(self.completion_rings[slot].ok_or(EINVAL)?)?.map_alias( vm, low::COMPLETION_RINGS[slot], mmu::PROT_GPU_SHARED_RW, @@ -985,7 +987,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -1011,7 +1013,7 @@ impl InitData { seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { - let object = self.object(self.completion_rings[0])?; + let object = self.object(self.completion_rings[0].ok_or(EINVAL)?)?; if seen.len() != COMPUTE_COMPLETION_SLOTS { return Err(EINVAL); } @@ -1247,8 +1249,8 @@ impl InitData { WB, )?)?; } - self.render_pages = pages; - self.render_runs = runs; + self.render_pages = Some(pages); + self.render_runs = Some(runs); self.render_blocks = first_block..self.objects.len(); self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; @@ -1348,7 +1350,7 @@ impl InitData { for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; - self.completion_rings[slot] = ring; + self.completion_rings[slot] = Some(ring); self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], @@ -1530,7 +1532,7 @@ impl InitData { ) -> Result> { let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; mappings.push( - self.object_mut(self.render_pages)?.map_alias( + self.object_mut(self.render_pages.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_PAGE_LIST, mmu::PROT_GPU_SHARED_RW, @@ -1538,7 +1540,7 @@ impl InitData { GFP_KERNEL, )?; mappings.push( - self.object_mut(self.render_runs)?.map_alias( + self.object_mut(self.render_runs.ok_or(EINVAL)?)?.map_alias( vm, low::RENDER_RUN_LIST, mmu::PROT_GPU_SHARED_RW, diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index d7d28ee3f6fbee..d446b1b25616e8 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -127,7 +127,7 @@ impl InitData { fixed(pair.high, m::M5_25G76_COMPLETION_BACKING_SIZE, RW)? }; let object = self.add(backing)?; - self.completion_rings[index] = object; + *self.completion_rings.get_mut(index).ok_or(EINVAL)? = Some(object); self.alias(object, uat.kernel_lower_vm(), pair.low, GPU)?; } m::encode_common26_aon_ptd(board, &mut self.object_mut(aon)?.bytes_mut()[..m::M5_AON_PTD_SIZE]) From b02044b787c98c97001f5a5de0d52268732c1eb8 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:17:20 -0800 Subject: [PATCH 129/171] drm/asahi: g17: M5: Fix M5 power timeouts under a longer idle delay On the M5 Ultra (J775D), when the device tree or sysfs asked for a longer GPU idle delay than the firmware's, the driver held the cores powered itself, and the firmware's core-power witness then read unpowered until the hold ended, so waits on it timed out. M5 now takes no host hold. The host idle hold keeps the cores powered after work when the tree or sysfs asks for a longer idle delay than the firmware's. The M5 firmware powers the cores for work by its own policy, and under a host hold J775D's core-power witness reads unpowered until the hold is released. M5 takes no host hold. Signed-off-by: Jeff Queisser (cherry picked from commit 26db14b92bc73626e3312b1cea87ef7309e3bcce in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9e19b853bcd1d5..8cc4536d8fad6d 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -932,6 +932,12 @@ impl Firmware { /// refused control (recovery in progress) is retried on a later check. fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { const RETRY_NS: u64 = 1_000_000; + // The M5 firmware powers the cores for work by its own policy, and under a host + // hold J775D's core-power witness reads unpowered until the hold is released. M5 + // takes no host hold, whatever delay the tree or sysfs asks for. + if self.init.is_m5() { + return None; + } let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { match self From 4696f20d0a72d3487a36f4b429c4591b3ed9743c Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 03:42:52 -0800 Subject: [PATCH 130/171] drm/asahi: g17: Fix GPU jobs that never retire after out-of-order submission On G17 GPUs, including the M5 Ultra Mac Studio (J775D), a job could stay unretired behind a newer job of its own context and then time out, as Chromium hit on the M5 Ultra before this port. Work-state nodes are now taken strictly in submission order. A context's ioctls take their work-state nodes in submission order, but an ioctl could take its node while up to eight older ones still waited for theirs. The node's word is drawn when the node is taken, so such an ioctl holds a lower word than an older ioctl taking its node later. The firmware sorts a context's nodes by word and flushes them from the head: the older ioctl's completion stamp then waits behind a node submitted after it, and its kick never retires. Take a node only at the ioctl's turn, once every older ioctl of the context has taken one or left without one, so the words follow submission order. A new ioctl more than 1024 past its context's oldest waiting one fails with EBUSY instead of falling back to unordered admission, so there is one ordering rule. Signed-off-by: Jeff Queisser (cherry picked from commit c4f1341770867aef61321292c88634b4ab9819f1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/context.rs | 15 ++- drivers/gpu/drm/asahi/g17/context/identity.rs | 123 +++++++++--------- drivers/gpu/drm/asahi/g17/fence.rs | 2 +- 3 files changed, 72 insertions(+), 68 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 96e1804d9b573e..3147a726967656 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -527,17 +527,20 @@ pub(crate) struct WorkStateLease { } impl WorkStateLease { pub(super) fn status(&self) -> &Arc { self.context.status() } - pub(crate) fn deferred(context: Arc) -> Self { - let ordinal = context.order.lock().assign(); - Self { + /// The lease of a new ioctl, ordered after every earlier one of its context. EBUSY while + /// too many of the context's ioctls are queued behind its oldest waiting one + /// (see `identity::Order`). + pub(crate) fn deferred(context: Arc) -> Result { + let ordinal = context.order.lock().assign().ok_or(EBUSY)?; + Ok(Self { context, word: AtomicU32::new(FREE), engine: AtomicU8::new(0), ordinal, - } + }) } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many - /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: an older + /// ioctl still waits for its node (see `identity::Order`), or no node may be taken now /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7646637bd6d0fa..0fe045d6f780c0 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -77,12 +77,10 @@ impl WorkStates { } } -/// Ordinals past the turn remembered for ioctls that took a node or left early. +/// How far past the oldest waiting ioctl a new ioctl may be ordered: the ordinals past the +/// turn remembered for ioctls that left early. pub(super) const ORDER_WINDOW: u64 = 1024; -/// How far past the oldest waiting ioctl an ioctl may take its node. -pub(super) const RUN_AHEAD: u64 = 8; - /// The order in which a context's ioctls take work-state nodes. /// /// Each engine admits a context's ioctls in submission order, but the engines advance @@ -93,20 +91,21 @@ pub(super) const RUN_AHEAD: u64 = 8; /// engines, at most 36 between scheduling and completion, and the same load runs there without /// failure. /// -/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have -/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; -/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler -/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that -/// without letting one engine get far ahead of the other. +/// So an ioctl takes its node only at its turn, once every older ioctl of its context has taken +/// one or left without one. The firmware sorts a context's nodes by their words and flushes +/// from the head, and a node's word is drawn when the node is taken +/// ([`WorkStates::allocate`]): an ioctl taking its node ahead of an older one would hold a lower +/// word than that ioctl, whose completion stamp would then wait behind a node submitted after +/// it. Taking nodes strictly in turn keeps the words in submission order. Each engine admits +/// from its own scheduler, so the turn costs a scheduler round trip whenever the other engine's +/// ioctl is ready first. pub(super) struct Order { /// The next ordinal to hand out. next: u64, /// The oldest ordinal that has neither taken a node nor left. turn: u64, - /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + /// Ordinals past the turn that left, as bits at ordinal % ORDER_WINDOW. done: [u64; (ORDER_WINDOW / 64) as usize], - /// Set once an ioctl left too far past the turn to remember; admission is unordered after. - unordered: bool, /// An ioctl was refused since the turn last moved: its publisher waits for the next move. refused: bool, } @@ -116,24 +115,28 @@ impl Order { next: 0, turn: 0, done: [0; (ORDER_WINDOW / 64) as usize], - unordered: false, refused: false, } } - /// The ordinal of a new ioctl, in submission order. - pub(super) fn assign(&mut self) -> u64 { + /// The ordinal of a new ioctl, in submission order. None while [`ORDER_WINDOW`] ioctls of + /// the context are past the oldest one still waiting for its node. + pub(super) fn assign(&mut self) -> Option { + if self.next - self.turn >= ORDER_WINDOW { + return None; + } let ordinal = self.next; self.next += 1; - ordinal + Some(ordinal) } - /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the - /// turn next moves, so that move wakes the refused publisher. + /// Whether the ioctl `ordinal` may take its node now: only at its turn. A refusal is + /// remembered until the turn next moves, so that move wakes the refused publisher. pub(super) fn admits(&mut self, ordinal: u64) -> bool { - let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + let admits = ordinal == self.turn; self.refused |= !admits; admits } - /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + /// The ioctl `ordinal` took its node at its turn. Returns whether a refused ioctl may now be + /// admitted. pub(super) fn admit(&mut self, ordinal: u64) -> bool { self.finish(ordinal) } @@ -143,17 +146,14 @@ impl Order { self.finish(ordinal) } fn finish(&mut self, ordinal: u64) -> bool { - if self.unordered || ordinal < self.turn { + if ordinal < self.turn { return false; } if ordinal == self.turn { self.pass(); return core::mem::take(&mut self.refused); } - if ordinal - self.turn >= ORDER_WINDOW { - self.unordered = true; - return core::mem::take(&mut self.refused); - } + // assign() keeps every ordinal within the window past the turn. let bit = ordinal % ORDER_WINDOW; self.done[(bit / 64) as usize] |= 1 << (bit % 64); false @@ -313,65 +313,66 @@ mod tests { } #[test] - fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + fn ioctls_take_nodes_only_at_their_turn() { let mut order = Order::new(); - let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); - let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); - // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. - for &ordinal in &ordinals[..=RUN_AHEAD as usize] { - assert!(order.admits(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // Ioctls ahead of the turn taking their nodes do not move it. - for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { - assert!(!order.admit(ordinal)); + let ordinals: [u64; 4] = core::array::from_fn(|_| order.assign().unwrap()); + // Only the oldest waiting ioctl may take its node; the others are refused. + for &ordinal in &ordinals[1..] { + assert!(!order.admits(ordinal)); } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // The oldest one moves the turn past everything already admitted, and wakes the refused. - assert!(order.admit(oldest)); - assert!(order.admits(last)); - let next = order.assign(); - assert!(order.admits(next)); - assert!(!order.admits(next + RUN_AHEAD + 1)); + assert!(order.admits(ordinals[0])); + // Its admission moves the turn and wakes the refused. + assert!(order.admit(ordinals[0])); + assert!(order.admits(ordinals[1])); + // Nobody was refused since, so the next admission wakes nobody. + assert!(!order.admit(ordinals[1])); + assert!(!order.admits(ordinals[3])); + assert!(order.admits(ordinals[2])); + assert!(order.admit(ordinals[2])); + assert!(order.admits(ordinals[3])); } #[test] fn ioctls_leaving_without_a_node_pass_the_turn() { let mut order = Order::new(); - let (a, b, c) = (order.assign(), order.assign(), order.assign()); - let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); - let beyond = later[RUN_AHEAD as usize - 1]; - // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + let (a, b, c, d) = ( + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + order.assign().unwrap(), + ); + // b and c fail before their turn; while a waits, d waits too. assert!(!order.leave(b)); assert!(!order.leave(c)); - assert!(!order.admits(beyond)); + assert!(!order.admits(d)); // Once a takes its node, the turn passes b and c as well, and the refused one may retry. assert!(order.admit(a)); - assert!(order.admits(beyond)); - // With nobody refused since, moving the turn wakes nobody; an ordinal already passed - // leaves without effect. - assert!(!order.leave(later[0])); + assert!(order.admits(d)); + // An ordinal already passed leaves without effect. assert!(!order.leave(a)); } #[test] - fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + fn no_ioctl_is_ordered_past_the_window() { let mut order = Order::new(); - let first = order.assign(); + let first = order.assign().unwrap(); let mut last = first; - for _ in 0..ORDER_WINDOW { - last = order.assign(); + for _ in 1..ORDER_WINDOW { + last = order.assign().unwrap(); } - assert!(!order.admits(last)); - assert!(order.leave(last)); - assert!(order.admits(last - 1)); + assert!(order.assign().is_none()); + // An ioctl leaving far ahead is remembered within the window. + assert!(!order.leave(last)); + assert!(order.admits(first)); + assert!(!order.admit(first)); + assert!(order.assign().is_some()); } #[test] fn single_engine_admission_wakes_nobody() { let mut order = Order::new(); - for _ in 0..3 * RUN_AHEAD { - let ordinal = order.assign(); + for _ in 0..24 { + let ordinal = order.assign().unwrap(); assert!(order.admits(ordinal)); assert!(!order.admit(ordinal)); } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 47e481e6880d26..1ef72a62dc35d1 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -199,7 +199,7 @@ pub(crate) struct Submission { impl Submission { fn new(context: Arc, firmware_source: bool) -> Result> { - let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + let work = Arc::new(WorkStateLease::deferred(context)?, GFP_KERNEL)?; Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { From 42a61cd69e26b7c830a9e7e2d78ef2de06d610f3 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:13:40 -0800 Subject: [PATCH 131/171] drm/asahi: g17: Fix M5 submissions stalling when the shared work ring fills On the M5 Ultra (J775D), concurrent clients could each prepare a job for room on the work ring tiling, fragment and compute share, and find it taken at publication. Room is now claimed from preparation to publication. On M5, tiling, fragment and compute share each priority's compute work ring. A compute or render publication only checked the ring's unclaimed room at publication time, so concurrent publishers could find it taken after preparing, and the credit domain that tracks claims was never populated. Give each owner one claim per engine on that ring. Compute preparation reserves the queue's slots (two for a cold install, else one) and waits on the credit domain when none are free; render preparation reserves the pair's two before preparing it off-lock. Publication lends the claim to the ring under the device mutex, and the claim is returned after the run, a cancellation or close. The event worker polls the ring consumers while a claim waits, since the firmware can consume slots without a completion interrupt, and a compute replay checks the shared ring rather than the classic one. Signed-off-by: Jeff Queisser (cherry picked from commit 2672b37fb2532508a84de594e3e0c92de92f59ec in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 15 +- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 3 + .../drm/asahi/g17/initdata/outer_credit.rs | 8 ++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 - drivers/gpu/drm/asahi/g17/runtime.rs | 136 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + .../drm/asahi/g17/runtime/compute_replay.rs | 9 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 8 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 17 +++ .../drm/asahi/g17/runtime/render/submit.rs | 13 +- 10 files changed, 187 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index f246256ea28ff1..e5167685e54593 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -623,20 +623,17 @@ impl InitData { Ok(()) } - /// Resource-specific wait domain; register only after dropping the device - /// mutex. A successful claim allocates no waiter fence. - pub(crate) fn outer_available(&mut self) -> Result> { - if self.outer_credits.is_none() { - self.outer_credits = Some(outer_credit::Credits::new()?); - } - Ok(self.outer_credits.as_ref().ok_or(EIO)?.available()) + /// The domain outer-ring waiters sleep on: unposted claims returned, or firmware + /// consumption seen by `poll_outer_progress`. Register only after dropping the device + /// mutex; a successful claim allocates no waiter fence. ENODEV off M5. + pub(crate) fn outer_available(&self) -> Result> { + Ok(self.outer_credits.as_ref().ok_or(ENODEV)?.available()) } /// M5 TA, fragment and compute share the compute outer ring per priority. /// The physical queue claim prevents reassignment while this credit lives. - pub(crate) fn reserve_outer(&mut self, priority: u8, count: u32) -> Result { + pub(crate) fn reserve_outer(&self, priority: u8, count: u32) -> Result { if !self.m5 { return Err(EINVAL); } - self.outer_available()?; let cursors = self.work_cursors(priority, super::fw::queue::DataMaster::Compute)?; self.outer_credits.as_ref().ok_or(EIO)?.reserve(priority, count, cursors) } diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index d446b1b25616e8..9e76ca5a4f8f94 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -86,6 +86,9 @@ impl InitData { use mmu::{PROT_FW_SHARED_RW as RW, PROT_FW_SHARED_RO as RO, PROT_GPU_SHARED_RW as GPU}; self.m5 = true; + // TA, fragment and compute share the compute work ring per priority; every + // preparation's claim on it is counted from the first publication on. + self.outer_credits = Some(super::outer_credit::Credits::new()?); let fixed = |address, size, protection| KernelObject::new(dev, uat.kernel_vm(), Placement::At(address), size, protection, CpuMap::WriteCombined); let fixed_controller = |address, size| { diff --git a/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs index 988cbe28fb5f6e..08804ff1fc4989 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/outer_credit.rs @@ -110,6 +110,14 @@ impl Drop for OuterScope<'_> { } } +impl OuterClaim { + /// Credits not yet consumed by a publication. A claim with none left can be dropped + /// anywhere: only returned credits notify waiters. + pub(crate) fn remaining(&self) -> u32 { + self.remaining + } +} + impl Drop for OuterClaim { /// As for physical queue claims, drop outside the device mutex because /// availability notification can run DMA-fence callbacks. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c4ae8b5916b686..a1974432143a13 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -809,7 +809,6 @@ impl Queue { } /// Cold M5 installation publishes one configuration before the first kick. /// A prepared claim prevents reassignment while these credits are owned. - #[allow(dead_code)] // Read by M5 lazy admission, a follow-up. pub(crate) fn outer_slots(&self) -> u32 { if self.graph.m5 && self.m5_install.is_none() { 2 } else { 1 } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index bd29bd9f8b688a..8bfd1ee7ef86db 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -659,6 +659,8 @@ pub(super) enum Readiness { Channel, /// Retirement-bound capacity or reopened publication. Progress, + /// M5: no unclaimed room on the shared work ring. + Outer, } /// Outcome of offering an owner a retained compute queue. @@ -718,6 +720,13 @@ pub(super) struct Backend { /// channel from being shared away between prepare_job and run_job. #[pin] claimed: kernel::sync::Mutex>>, + /// M5: this owner's claims on the work ring M5 TA, fragment and compute share, one per + /// engine, taken while preparing a publication and consumed by it + /// ([`Backend::claim_outer`], [`Backend::publish_claimed`]). + #[pin] + outer_compute: kernel::sync::Mutex>, + #[pin] + outer_render: kernel::sync::Mutex>, /// Render pair admission, shared by every logical queue of the session. admission: Arc, /// The render pair this owner last published to without the device mutex @@ -825,6 +834,8 @@ impl Backend { closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + outer_compute <- kernel::new_mutex!(None, "g17::Backend::outer_compute"), + outer_render <- kernel::new_mutex!(None, "g17::Backend::outer_render"), admission, direct_slot: core::sync::atomic::AtomicU32::new(0), }), @@ -1957,23 +1968,94 @@ impl Backend { return None; } let required = self.compute_capacity(&packet.command).ok()?; + let outer = self.outer_domain(); loop { let channels = self.shared.channels.observe(); let progress = self.shared.progress.observe(); - let (domain, observed) = match self.compute_readiness(packet, required) { - Readiness::Ready => return None, - Readiness::Channel => (&self.shared.channels, channels), - Readiness::Progress => (&self.shared.progress, progress), + let claimed = outer.as_ref().map(|domain| domain.observe()); + let (domain, observed) = match (self.compute_readiness(packet, required), &outer, claimed) { + (Readiness::Ready, ..) => return None, + (Readiness::Channel, ..) => (&self.shared.channels, channels), + (Readiness::Progress, ..) => (&self.shared.progress, progress), + (Readiness::Outer, Some(domain), Some(observed)) => (domain, observed), + (Readiness::Outer, ..) => return None, }; - self.shared.queue_events(); match domain.wait(observed) { - Ok(Some(fence)) => return Some(fence), + Ok(Some(fence)) => { + // Registered first: the event worker polls outer consumption only for waiters. + self.shared.queue_events(); + return Some(fence); + } Ok(None) => continue, Err(_) => return None, } } } + /// M5: the domain outer-ring claims wait on. None off M5 or without firmware. + fn outer_domain(&self) -> Option> { + self.shared.state.lock().as_deref().and_then(|firmware| firmware.init.outer_available().ok()) + } + + /// M5: holds this owner's claim on the shared work ring for the next compute or render + /// publication, reserving one under the device mutex unless one with credits left is + /// held. False: no unclaimed room; wait on [`Self::outer_domain`] from a sample taken + /// before this call. Always true off M5. + fn claim_outer(&self, firmware: &super::Firmware, engine: crate::g17::job::Engine) -> Result { + if !firmware.init.is_m5() { + return Ok(true); + } + let (held, count) = match engine { + crate::g17::job::Engine::Compute => ( + &self.outer_compute, + firmware.queues.compute(self.owner).map_or(2, |queue| queue.outer_slots()), + ), + // Tiling and fragment: one slot each. + crate::g17::job::Engine::Render => (&self.outer_render, 2), + }; + let mut held = held.lock(); + if held.as_ref().is_some_and(|claim| claim.remaining() != 0) { + return Ok(true); + } + match firmware.init.reserve_outer(self.context.policy().priority(), count) { + Ok(claim) => { + // Replaces only a spent claim, whose drop notifies nobody. + *held = Some(claim); + Ok(true) + } + Err(error) if error == EBUSY => Ok(false), + Err(error) => Err(error), + } + } + + /// Publishes into this owner's compute channel, lending the held outer claim's credits. + fn publish_claimed( + &self, + firmware: &mut super::Firmware, + packet: &Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + let mut held = self.outer_compute.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; + firmware.publish_compute(self.owner, packet.clone(), dependencies, deferred) + } + + /// Returns an engine's unspent outer claim after its publication, cancellation or close. + /// Outside the device mutex: returned credits notify waiters. + pub(super) fn release_outer(&self, engine: crate::g17::job::Engine) { + let held = match engine { + crate::g17::job::Engine::Compute => &self.outer_compute, + crate::g17::job::Engine::Render => &self.outer_render, + }; + let claim = held.lock().take(); + drop(claim); + } + fn compute_readiness(&self, packet: &Arc, required: super::freelist::ComputeCapacity) -> Readiness { match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { Err(error) if error == EBUSY => return Readiness::Progress, @@ -1995,7 +2077,15 @@ impl Backend { }); if bound { self.hold_claim(Some(cell)); - return Readiness::Ready; + if !self.context.is_m5() { + return Readiness::Ready; + } + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + return match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }; } } } @@ -2017,7 +2107,13 @@ impl Backend { { return Readiness::Progress; } - firmware.queues.compute_readiness(self.owner, Some((&self.context, required))) + match firmware.queues.compute_readiness(self.owner, Some((&self.context, required))) { + Readiness::Ready => match self.claim_outer(firmware, crate::g17::job::Engine::Compute) { + Ok(false) => Readiness::Outer, + _ => Readiness::Ready, + }, + other => other, + } } /// Replaces the prepared packet's channel claim; None releases it after publication, @@ -2143,8 +2239,11 @@ impl Backend { { return Err(render::FALLBACK); } + if !self.claim_outer(firmware, crate::g17::job::Engine::Compute)? { + return Err(render::FALLBACK); + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + match self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred) { Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { Err(render::FALLBACK) } @@ -2206,9 +2305,11 @@ impl Backend { }; let admission = Admission::new(deadline, &check); let progress = &self.shared.progress; + let outer = self.outer_domain(); loop { admission.check()?; let observed = progress.observe(); + let claimed = outer.as_ref().map(|domain| domain.observe()); if !self.sources_published(packet)? { let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { firmware.recovery.pending() @@ -2274,14 +2375,19 @@ impl Backend { drop(lease); continue; } + let held = self.claim_outer(firmware, crate::g17::job::Engine::Compute)?; + if let (false, Some(domain), Some(claimed)) = (held, &outer, claimed) { + // The shared work ring has no unclaimed room: consumption or returned claims + // free it. + drop(state); + drop(lease); + self.shared.queue_events(); + self.wait_release(domain, claimed, &admission)?; + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; admission.check()?; - let result = firmware.publish_compute( - self.owner, - packet.clone(), - dependencies.as_slice(), - deferred, - ); + let result = self.publish_claimed(firmware, packet, dependencies.as_slice(), deferred); if !packet.is_published() && (matches!(result, Err(EAGAIN)) || (matches!(result, Err(EBUSY)) diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 5482f2b96b0f37..5b5ad7c1f11817 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -94,6 +94,8 @@ impl Backend { // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); self.hold_claim(None); + self.release_outer(Engine::Compute); + self.release_outer(Engine::Render); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -281,6 +283,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -312,6 +315,7 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { self.hold_claim(None); } + self.release_outer(packet.engine()); if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d16b22e35b896f..0aca0ec0c8ffee 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -94,7 +94,14 @@ impl Firmware { .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)? .clear_consumer_wait(); - let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; + // M5 compute takes its slot on the work ring M5 render shares, whose unclaimed + // room the publication checks (InitData::work_ready). + let priority = packet.context.policy().priority(); + let outer_ready = if self.init.is_m5() { + self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Compute)? + } else { + self.port.compute_ready(priority)? + }; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 5e0e7d8b4b7dad..24821c1d804a75 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -541,8 +541,12 @@ impl crate::g17::Shared { self.queue_reclaims(); } } + // The firmware may consume M5 work-ring slots without a completion interrupt: while + // a preparation waits for unclaimed room, every pass polls the consumers. + let outer = firmware.init.poll_outer_progress()?; let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.init.outer_waiting() || firmware.queues.replay_outer_wait.is_some() || firmware.queues.replay_consumer_wait.is_some() || firmware.queues.render.replay_polling() @@ -594,6 +598,10 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Outside the device mutex: the notification can run DMA-fence callbacks. + if let Some(domain) = outer { + domain.notify(); + } // Firmware ring room, consumed releases and teardown steps have no // event of their own: recheck render publishers' conditions every pass. if self.renders.has_waiters() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 85cb8903ad7d92..1704139a9f97f1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -844,7 +844,9 @@ impl super::super::Backend { packet: &Arc, deferred: &mut super::super::DeferredBatch, sleep: bool, + outer_wait: &mut Option<(Arc, u64)>, ) -> Result { + *outer_wait = None; let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { @@ -867,6 +869,15 @@ impl super::super::Backend { firmware.init.stamp_addresses(qids[1])?, ], }; + // M5: the pair's two slots on the shared work ring are claimed before preparing + // and lent to its publication. Sampled first: a miss waits for the next return. + if let Ok(domain) = firmware.init.outer_available() { + let observed = domain.observe(); + if !self.claim_outer(firmware, crate::g17::job::Engine::Render)? { + *outer_wait = Some((domain, observed)); + return Err(EBUSY); + } + } let pair = firmware .queues .render @@ -956,6 +967,12 @@ impl super::super::Backend { if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { return Err(ECANCELED); } + let mut held = self.outer_render.lock(); + let _scope = (*held) + .as_mut() + .filter(|claim| claim.remaining() != 0) + .map(|claim| firmware.init.outer_scope(claim)) + .transpose()?; firmware.publish_render(self.owner, slot, packet, deferred) })(); let result = (|| { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 24fea49f807d13..7dfac5e539f180 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -404,6 +404,7 @@ impl Backend { let mut recovery_since = None; let mut seen = None; let mut states = None; + let mut outer_wait = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -471,7 +472,7 @@ impl Backend { } result => result?, } - match self.prepare_render(slot, packet, deferred, sleep) { + match self.prepare_render(slot, packet, deferred, sleep, &mut outer_wait) { Err(error) if !packet.is_published() && error == ENOSPC => { self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; @@ -498,6 +499,15 @@ impl Backend { self.shared .preparations .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; + } else if let Some((domain, observed)) = outer_wait.take() { + // No unclaimed room on the shared work ring: consumption or a + // returned claim frees it. + if !sleep { + return Err(FALLBACK); + } + let deadline = self.wait_deadline(&mut busy, super::super::WORK_RING_WAIT_NS)?; + self.shared.queue_events(); + self.wait_domain(&domain, observed, deadline, packet)?; } else if error == EBUSY { self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { @@ -544,6 +554,7 @@ impl Backend { packet.abandon_publication(); return false; } + self.release_outer(crate::g17::job::Engine::Render); let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); From 354fb5c40941893641e9b1016ba72bf8a780c149 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:13:43 -0800 Subject: [PATCH 132/171] drm/asahi: g17: Fix restart-cancelled M5 compute jobs reporting success On the M5 Ultra Mac Studio (J775D), a compute job in flight when the GPU firmware restarted was completed as if it had run, though the restart had cancelled it, so its application got results that were never written. It is now treated as not executed; with the recovery-record fix the compute bystander of a render-spin recovery is exact in 4 of 4 runs, where 2 of 4 were wrong. A compute command in flight across a firmware restart is retired with a record whose end is at most one tick after its start. Retirement only treated a zero-duration record as unexecuted, so such a command was completed as if it had run. Use the same rule as recovery's spare check for commands that crossed a recovery. Signed-off-by: Jeff Queisser (cherry picked from commit b88d110849029811e2130901529db73586c478f8 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 902e7ff03ef04b..90736f77a336a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -153,7 +153,8 @@ pub(crate) fn unexecuted(record: Observation, crossed: bool) -> bool { start == end || (crossed && start != 0 && end >= start && end - start <= CANCELLED_TICKS) } -/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// A record showing the command never executed ([`unexecuted`]) fails before visibility +/// polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has /// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery @@ -174,10 +175,11 @@ pub(crate) fn poll( next_end: Option, record_aged: bool, ) -> Result> { - if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + let crossed = matches!(recordless, Recordless::CrossedRecovery); + if observed.is_some_and(|record| unexecuted(record, crossed)) { return Err(ENODATA); } - let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let same_generation = !crossed; let completed = stamp_covers(stamp, ticket.kick) || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { From 048c8f568f91254a3a3dc0d820be3a8aba554d4e Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:13:45 -0800 Subject: [PATCH 133/171] drm/asahi: g17: Fix M5 compute timeouts judged by an early record On the M5 Ultra (J775D) the watchdog took a compute queue's head as finished once its completion record appeared, which can precede the job's writes, so it could excuse or blame the wrong job at a timeout. It now uses the stamp, as retirement does. The watchdog took the head of a compute queue as finished once its completion record was seen. On M5 the record can precede the command's writes, and retirement waits for the stamp; let the timeout classification use the same proof. Signed-off-by: Jeff Queisser (cherry picked from commit 5f0dca3f1492c1b9e3e12f702dbe2d5f7ec7893b in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index a1974432143a13..745b91f8bf92ef 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -830,7 +830,14 @@ impl Queue { self.active.front().map(|active| crate::g17::timeout::Head { vm: active.packet.context.vm().id(), published: active.published, - finished: active.observed.is_some(), + // An M5 completion record can precede the command's writes: only the stamp + // proves it finished. + finished: if self.graph.m5 { + active.ticket.as_ref().is_some_and(|ticket| self.graph.stamp() + .is_ok_and(|stamp| retirement::stamp_covers(stamp, ticket.kick))) + } else { + active.observed.is_some() + }, failed: self.quarantined, }) } From 3b454f0fe4f72ce6a1e00cd81f30ad0e4d87f8b9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:13:47 -0800 Subject: [PATCH 134/171] drm/asahi: g17: Fix freeing render state the M5 firmware still links On the M5 Ultra (J775D), when an application closed with renders that had run without a stamp, teardown could release its scheduler, slot and pool before the context's kill was published, while the firmware still linked that work. A scenario sweep rated this a crash or hang; it was not seen on the Studio. Teardown treated a render pair whose commands all ran without a stamp as idle for its context whether or not a kill had been published. Until the kill, the firmware still links that work, so the scheduler must stay held; count such a pair only once the context's kill is published. Integration's exception for a pair with no stamped command stays, but applies only once the kill is published. Signed-off-by: Jeff Queisser (cherry picked from commit 4acee56c4a5931d74a995a28cd260ee8b05e3de4 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index bbb2f9f1d08139..c700f9e1020248 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -38,7 +38,11 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { + // Commands that ran unstamped leave nothing for the firmware to run only once a + // kill of the context has been published; until then they still hold it. + if !pair.idle_for_context(context)? + && !(context.kill_published() && pair.executed_for_kill(context)) + { return Ok(false); } } From 1c7e729bb8bf940d4e6f17a268d4a98dfbe3132e Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:13:49 -0800 Subject: [PATCH 135/171] drm/asahi: g17: Fix skipped or repeated GPU effort updates on M5 On the M5 Ultra (J775D) the driver compared a new effort request with the last request, not with the value it had given the firmware, so it could skip a needed update or resend one, and the opt-in governor judged activity from the request. No symptom was observed. The effort state recorded the requested value while M5 is given its own submission value, so a repeated request was skipped or republished by the wrong comparison, and the governor's activity check read the request. Record and compare the published value; the performance-controller demand still follows the request. Signed-off-by: Jeff Queisser (cherry picked from commit 8996e80269f66564c4425d326981c5b1e9a6b634 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 8cc4536d8fad6d..f1e3a813e3f62e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -869,16 +869,16 @@ impl Firmware { port: &port::Port, effort: u32, ) -> Result { - // The state machine and the performance-controller demand follow the requested effort; - // the firmware is given the M5 value (InitData::submission_effort). + // The performance-controller demand follows the requested effort; the firmware is + // given, and the state records, the M5 value (InitData::submission_effort). let published = init.submission_effort(effort, state.waited()); - if state.applied() == effort && init.effort()? == published { + port.set_effort_full(effort == power::FULL_EFFORT); + if state.applied() == published && init.effort()? == published { return Ok(()); } init.publish_effort(published)?; trace::message(trace::EFFORT, u64::from(published)); - state.did_apply(effort); - port.set_effort_full(effort == power::FULL_EFFORT); + state.did_apply(published); Ok(()) } From 3636a244b51fd187523b45503274e5ea0e32a8b7 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:31:58 -0800 Subject: [PATCH 136/171] drm/asahi: g17: Fix M5 compute jobs failing with EBUSY when clients contend On the M5 Ultra Mac Studio (J775D), with more compute clients than hardware queues, a job that waited over 2 s for a queue failed with EBUSY although nothing was wrong: 11 clients of a 46-process burst failed this way. A job now waits up to 30 s. A compute publication bounded every wait by the 2 s work-ring bound, including the wait for a physical compute queue. With more compute owners than queues, an owner waits for others' work to retire or go idle, and after 2 s its command failed with EBUSY although nothing was wrong; a 46-client burst ran into it. A compute publication now has no 2 s overall bound: its admission starts at the 30 s queue bound, so it waits for a queue, and for room on the shared work ring, for up to 30 s before giving up. Signed-off-by: Jeff Queisser (cherry picked from commit 2a2648eba8470dd026650d8c7359a6ca295e5e2a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/preparation.rs | 4 ++++ drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index c7d1761699a24b..85994ab8e6c73c 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -30,6 +30,10 @@ pub(super) struct Admission<'a> { check: &'a dyn Fn() -> Result, } +/// Bound for a compute publication's waits: a physical compute queue, or room on the shared +/// work ring, frees only as other owners' work retires or goes idle. +pub(super) const QUEUE_WAIT_NS: u64 = 30_000_000_000; + impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { Self { deadline: Cell::new(deadline), started: super::now_ns(), check } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8bfd1ee7ef86db..23c3e31f5b873d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2296,7 +2296,7 @@ impl Backend { // publication after recovery both signal `progress`. let mut bound = true; let mut demand = None; - let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let deadline = now_ns().saturating_add(super::preparation::QUEUE_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } self.admission_open()?; From 4a3b9e3a3a86dedfbbec11c7bf6f9128c707c3f5 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:33:49 -0800 Subject: [PATCH 137/171] drm/asahi: g17: Fix out-of-memory kills when many M5 compute clients start On the M5 Ultra Mac Studio (J775D), when many compute clients started at once, every waiting client built a full pool for the same free queue and the system ran out of memory: in a 46-process burst 27 clients got ENOMEM and the OOM killer ran. With this and the 30 s wait the burst passes 46 of 46 with no OOM. Every owner waiting for a compute queue was offered the same vacant queue to re-arm, and each built a full M5 pool for it without the device mutex before finding it taken. A 46-client burst then allocated dozens of pools at once and ran the system out of memory (27 clients saw ENOMEM and the OOM killer ran). Reserve the vacant queue for the owner it is offered to: others wait for a channel, and the reservation ends when the owner binds the queue or gives up. Signed-off-by: Jeff Queisser (cherry picked from commit 51fde336cf91b89e854858878053d4ed420d45be in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 17 ++++ drivers/gpu/drm/asahi/g17/runtime.rs | 113 ++++++++++++++------- 2 files changed, 96 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 745b91f8bf92ef..6fd44f0b7e2db6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -623,6 +623,9 @@ pub(crate) struct Queue { /// and whether that wait was reported. kill_wait: Option<(u64, bool)>, installed: bool, + /// The owner re-arming this vacant queue: only it builds the fresh backing, so concurrent + /// waiters do not each build a pool for the same queue. + rearm_for: Option, configuration: Option, m5_install: Option, /// Bit `p` set once a work slot of priority class `p` named this queue. @@ -704,6 +707,7 @@ impl Queue { exit: None, kill_wait: None, installed: false, + rearm_for: None, configuration: None, m5_install: None, outer_started: 0, @@ -1717,6 +1721,7 @@ impl Queue { if same_vm { 2 } else { 1 }, ); self.owner = Some(owner); + self.rearm_for = None; self.unpublished_binding = true; self.released = false; self.retirement_ready = false; @@ -1906,6 +1911,18 @@ impl Queue { pub(crate) fn pool_vacant(&self) -> bool { self.pool_phase == PoolPhase::Vacant } + /// No other owner is re-arming this queue. + pub(crate) fn rearm_open(&self, owner: u64) -> bool { + self.rearm_for.is_none_or(|reserved| reserved == owner) + } + pub(crate) fn reserve_rearm(&mut self, owner: u64) { + self.rearm_for = Some(owner); + } + pub(crate) fn release_rearm(&mut self, owner: u64) { + if self.rearm_for == Some(owner) { + self.rearm_for = None; + } + } /// Publishes the release of this idle queue's free list. The caller holds the /// device mutex and rings the control doorbell afterwards. pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 23c3e31f5b873d..1c9aa00b6a95df 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -404,14 +404,24 @@ impl Registry { continue; } if queue.pool_vacant() { - vacant.get_or_insert(queue.qid()); + if queue.rearm_open(owner) { + vacant.get_or_insert(queue.qid()); + } continue; } if !queue.pool_fits(required) || !queue.pool_serves(context.vm()) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Some(Rebind::Done)); } - Ok(vacant.map(Rebind::Rearm)) + let Some(qid) = vacant else { return Ok(None) }; + // Reserved for this owner until it binds the queue or gives up: other owners wait for + // a channel instead of each building a pool for the same queue. + self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .reserve_rearm(owner); + Ok(Some(Rebind::Rearm(qid))) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -432,6 +442,16 @@ impl Registry { queue.rearm(pages, ring) } + /// Ends `owner`'s re-arm reservation of `qid` when it did not bind the queue. + pub(super) fn release_rearm(&mut self, qid: u8, owner: u64) { + if let Some(mut queue) = self.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + { + queue.release_rearm(owner); + } + } + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; let mut queue = self @@ -1076,6 +1096,51 @@ impl Backend { result } + /// Re-arms the vacant queue `qid` reserved for this owner with fresh zeroed backing, built + /// without the device mutex, and binds it. Some(true): bound; Some(false): retry (the + /// preparation epoch changed or the queue is no longer re-armable); None: no memory for + /// the backing now. + fn rearm_vacant( + &self, + qid: u8, + required: super::freelist::ComputeCapacity, + alloc: &super::object::Allocator<'_>, + objects: &Arc, + lease: &super::preparation::Lease, + admission: &Admission<'_>, + ) -> Result> { + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::for_compute(alloc, required)?; + Ok((pages, compute::DescriptorRing::new(alloc, objects)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => return Ok(None), + result => result?, + }; + let mut state = self.shared.state.lock(); + admission.check()?; + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Ok(Some(false)); + } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // Leave rejected backing local so it is dropped after the device guard, not in + // rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm() && queue.rearm_open(self.owner)); + if !ready { drop(state); return Ok(Some(false)); } + let descriptor = match firmware.queues.rearm_compute(qid, pages, ring) { + Err(error) if error == EAGAIN => return Ok(Some(false)), + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.install_compute_pool(descriptor)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + Ok(Some(true)) + } + fn build_compute(&self, required: super::freelist::ComputeCapacity, admission: &Admission<'_>) -> Result { admission.check()?; self.admission_open()?; @@ -1153,43 +1218,23 @@ impl Backend { let reservation = match reservation { Ok(reservation) => reservation, Err(qid) => { - // Fresh zeroed backing is built without the device mutex; the - // queue may have been taken or re-armed by another owner since. - let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::for_compute(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) - })(); - let (pages, ring) = match built { - Err(error) if error == ENOSPC => { + // bind_reusable reserved `qid` for this owner: any outcome but binding it + // ends the reservation, so another owner may re-arm it. + let result = self.rearm_vacant(qid, required, &alloc, &objects, &lease, admission); + if !matches!(result, Ok(Some(true))) { + if let Some(firmware) = self.shared.state.lock().as_deref_mut() { + firmware.queues.release_rearm(qid, self.owner); + } + } + match result? { + Some(true) => return Ok(()), + Some(false) => continue, + None => { drop(lease); self.wait_channels(observed, admission, true)?; continue; } - result => result?, - }; - let mut state = self.shared.state.lock(); - admission.check()?; - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - continue; } - if let Err(error) = self.admission_open() { drop(state); return Err(error); } - // A competing ready owner may have taken/rearmed it while - // backing was built off-lock. Leave rejected backing local - // so it is dropped after the device guard, not in rearm(). - let ready = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.qid() == qid && queue.can_rearm()); - if !ready { drop(state); continue; } - let descriptor = match firmware.queues.rearm_compute(qid, pages, ring) { - Err(error) if error == EAGAIN => continue, - result => result?, - }; - // The firmware consumed this slot's release before the backing - // was dropped; the row is rewritten before the queue is kicked. - firmware.init.install_compute_pool(descriptor)?; - firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; - return Ok(()); } }; let built = (|| { From 59f20548acc2e6d06bfcececa70c5cd945bd5b6b Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 04:56:28 -0800 Subject: [PATCH 138/171] drm/asahi: g17: Fix M5 renders refused while their context changes pair On the M5 Ultra (J775D), a render submitted while its context's render channel was moving to a new pair was refused instead of waiting for the old pair's work to retire. With one render pair per client today this path was not reached in testing. A context's render channel moves to a new pair only once its publications on the old pair have retired (Context::render_channel). Until then render preparation returns RENDER_CHANNEL_PENDING, and the submission was refused. Wait for that retirement instead, as long as for a compute queue. Signed-off-by: Jeff Queisser (cherry picked from commit 63a3dd52e195e87b32c4a210fe5f219d7e9d9952 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../drm/asahi/g17/runtime/render/submit.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 7dfac5e539f180..eef06526c49379 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -258,11 +258,23 @@ impl Backend { observed: &mut Option, sleep: bool, packet: &Packet, + ) -> Result { + self.wait_render_retirement(clock, observed, sleep, packet, super::super::WORK_RING_WAIT_NS) + } + + /// [`Self::wait_render_busy`] bounded at `bound_ns`. + fn wait_render_retirement( + &self, + clock: &mut WaitClock, + observed: &mut Option, + sleep: bool, + packet: &Packet, + bound_ns: u64, ) -> Result { if !sleep { return Err(FALLBACK); } - let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; + let deadline = self.wait_deadline(clock, bound_ns)?; // Sample, then retry once: only a refusal after the sample waits. let Some(sample) = observed.take() else { *observed = Some(self.shared.renders.observe()); @@ -405,6 +417,7 @@ impl Backend { let mut seen = None; let mut states = None; let mut outer_wait = None; + let mut channel = WaitClock::default(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -473,6 +486,15 @@ impl Backend { result => result?, } match self.prepare_render(slot, packet, deferred, sleep, &mut outer_wait) { + Err(error) + if !packet.is_published() && error == crate::g17::context::RENDER_CHANNEL_PENDING => + { + // Both of the context's render channels serve other pairs: one moves here + // once its publications retire. Like a compute queue, it frees only as this + // context's earlier work retires, so wait as long. + self.wait_render_retirement(&mut channel, &mut seen, sleep, packet, + crate::g17::preparation::QUEUE_WAIT_NS)?; + } Err(error) if !packet.is_published() && error == ENOSPC => { self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; From 84b302ea436f6605ae1961d1ace43e904f5208e9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 05:03:43 -0800 Subject: [PATCH 139/171] drm/asahi: g17: Fix M5 GPU recovery failing or completing the wrong job On the M5 Ultra Mac Studio (J775D), when one application's hung render forced a GPU recovery, another application's compute job could come back wrong: the recovery matched commands by another firmware's stamp form, failing one job and completing the one the restart cancelled (2 of 4 runs). M5 recovery now only classifies, and with the cancellation-record fix the bystander is exact in 4 of 4. The recovery table walk stopped at the first invalid record, but the table need not be packed, so commands listed after a gap were never seen. Walk all records and skip invalid ones. On M5 the table's stamp form differs, so its records only classify the recovery and no command is marked interrupted from them; M5 retirement already decides from the ring records. Signed-off-by: Jeff Queisser (cherry picked from commit 4c62107745e7b1ec6bfa5c938ee8066d7c4400b1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/initdata.rs | 3 +++ drivers/gpu/drm/asahi/g17/recovery.rs | 18 ++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 +++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index e5167685e54593..08d72402d4115b 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1874,6 +1874,9 @@ impl InitData { } impl super::recovery::Memory for InitData { + fn is_m5(&self) -> bool { + self.m5 + } fn fault_sources(&self) -> Result { let mask_offset = if self.m5 { super::fw::m5::initdata::M5_PRIVATE_LAYOUT.status_b diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index dcd584f28c0caf..09a1f072b21507 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -50,8 +50,8 @@ pub(super) struct Entries { } impl Entries { - /// Returns false at the first invalid record: the table is a packed list and - /// records after it are left over from an earlier restart. + /// Counts a valid record. Returns false for an invalid one, which the walk skips: the + /// table need not be packed (see `read_entries`). pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { return false; @@ -132,6 +132,8 @@ pub(super) trait Memory { fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); + /// The M5 firmware's recovery table, whose stamp form is not the G17P one. + fn is_m5(&self) -> bool; } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -328,16 +330,19 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error, leftover }) } -/// Walks the packed recovery table up to its first invalid record and marks -/// every listed command interrupted. A failed read keeps what was read so far. +/// Walks every record of the recovery table: the table need not be packed, so an invalid +/// record is skipped rather than ending the walk. Every listed command is +/// marked interrupted where the stamp form is known (not on M5, whose table only classifies). +/// A failed read keeps what was read so far. fn read_entries(host: &mut H) -> Entries { let mut entries = Entries::default(); + let mark = !host.is_m5(); for index in 0..RECOVERY_RECORDS { let Ok((flags, stamp)) = host.entry(index) else { break; }; - if !entries.include(flags) { - break; + if !entries.include(flags) || !mark { + continue; } let qid = ((flags >> 4) & 0x7f) as u8; match host.note_interrupted(qid, stamp) { @@ -456,6 +461,7 @@ mod tests { } impl Memory for Halted { + fn is_m5(&self) -> bool { false } fn recovery_state(&self) -> Result { Ok(self.state.get()) } fn host_recovery(&self) -> Result { Ok(0) } fn fault_sources(&self) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 24821c1d804a75..b66f1a17444c8e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -133,6 +133,9 @@ impl recovery::Host for Service<'_> { } impl recovery::Memory for Service<'_> { + fn is_m5(&self) -> bool { + self.firmware.init.is_m5() + } fn recovery_state(&self) -> Result { self.firmware.init.recovery_state() } From 4f2b1ccfb9b0c98853d7d400e409ed5570df0a28 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 05:03:46 -0800 Subject: [PATCH 140/171] drm/asahi: g17: Fix M5 renders retiring before their writes land On the M5 Ultra (J775D) a render whose stamps lagged could be retired on its pass record alone, which the firmware can write before the render's writes land, so its buffers could go back to the application while still in use. Only the stamps retire an M5 render now. When a render group's stamps lagged its executed pass record, the observation scan retired it on that record after a fallback interval. On M5 the pass record can be written before the group's writes land, so retiring on it could return buffers still in use. As for M5 compute, only the stamps retire an M5 render group. Signed-off-by: Jeff Queisser (cherry picked from commit 36a446148c1529ae5dd15217b4bae244e052d3f7 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/render.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index ae2d1b2683334b..6c14fca21a8166 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1453,14 +1453,16 @@ impl Pair { /// Latches when the front group executed and the firmware consumed its ring entries while /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it - /// on its pass record instead of waiting for stamps no later event will move. + /// on its pass record instead of waiting for stamps no later event will move. Not on M5, + /// whose pass record can precede the group's writes: as for M5 compute, only the stamps + /// return the group's buffers. pub(crate) fn note_stamp_wait( &mut self, stamps: [u32; 2], state: u32, now: u64, ) -> Result { - if self.quarantined || self.prepared.is_some() || state != 0 { + if self.quarantined || self.prepared.is_some() || state != 0 || self.memory.graph.is_m5() { return Ok(false); } let Some(front) = self.active.front_mut() else { From d6a894d43948af4313220f1abcf211e118a33ca4 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 05:23:38 -0800 Subject: [PATCH 141/171] drm/asahi: g17: Fix the M5 recovery log claiming listed jobs fail On the M5 Ultra (J775D) a GPU recovery logged that the commands it listed would fail instead of completing, which M5 no longer does with them. The log now says each is retired from its own queue's records. Since the M5 recovery table only classifies, no listed command is marked interrupted on M5, yet the recovery still reported that the listed commands fail instead of completing. On M5, report that each is retired from its own queue's records. [ rebased onto integration/aurora: conflict in recovery.rs only; kept the third result of classify() (leftover) that "drm/asahi: g17: do not count a failed VM's leftover hangs as a restart storm" added ] Signed-off-by: Jeff Queisser (cherry picked from commit 92c1366896daf1902bbe7627d77c83d91e41397f in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/recovery.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 09a1f072b21507..8d083e755b741d 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -303,7 +303,14 @@ pub(super) fn service( let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame, leftover) = classify(host, entries, request.blamed, attribution); - if listed != 0 { + if listed != 0 && host.is_m5() { + // The M5 table only classifies (read_entries): each command is retired from its own + // queue's records, replayed when it never ran. + pr_warn!( + "G17: GPU recovery: {} commands were in flight; each is retired from its own queue's records\n", + listed + ); + } else if listed != 0 { pr_warn!( "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", listed, From 9d19a842d19a7a753913e0b851b1ec4df0b58e08 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 06:03:54 -0800 Subject: [PATCH 142/171] drm/asahi: g17: M5: Fix slow kills of hung jobs after other clients finish On the M5 Ultra Mac Studio (J775D), a hung compute job started just after other clients finished was killed only after 6 s instead of at its 2 s watchdog (2 of 4 runs). It now fails at its first watchdog expiry: 2 s in 4 of 4. A published job's deadline was renewed while any other command had retired within the last 5 s, a rule measured on G17P where clients share the GPU's cores. On M5 it also renewed a hung compute spin started just after other clients finished, so its kill came after three watchdog intervals (6 s) instead of one. M5 takes no such renewal: a job that has not finished at its watchdog fails then, and the compute kill follows. A timeout is only deferred to the fault attribution, with no shared or execution renewal. Signed-off-by: Jeff Queisser (cherry picked from commit 68ec774b95cd94119da991e8bd0c408834cc536d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/job.rs | 8 ++++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 54bd823eda094b..e42c0ce76f966c 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -329,6 +329,9 @@ pub(crate) trait Backend: Send + Sync { fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// A long job of another queue is running beside `packet`. fn long_job_beside(&self, _packet: &Arc) -> bool { false } + /// A published job's deadline may be renewed while other work retires + /// ([`super::timeout::renew_shared`]). + fn renews_shared(&self) -> bool { true } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -384,8 +387,9 @@ impl sched::JobImpl for Job { } // Other work retiring means the GPU is shared, not hung. So does another long job // running beside a long job, though neither retires until it ends. - let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) - || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); + let shared = job.backend.renews_shared() + && (super::timeout::retiring(super::recovery::since_retired(super::now_ns())) + || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet))); if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { return true; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 5b5ad7c1f11817..afee32d794e24a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -349,6 +349,13 @@ impl job::Backend for Backend { }; } + /// Not on M5: a job that does not finish within its watchdog interval fails then, however + /// recently other work retired. Its compute kill then follows the first expiry. + fn renews_shared(&self) -> bool { + let state = self.shared.state.lock(); + (*state).as_deref().is_none_or(|firmware| !firmware.init.is_m5()) + } + fn long_job_beside(&self, _packet: &Arc) -> bool { let state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref() else { From 02aed39508f6644aa4094310af4fcd83df4b22e9 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 11:21:06 -0800 Subject: [PATCH 143/171] drm/asahi: g17: M5: Fix M5 GPU completions being handled at a lowered CPU clock On the M5 Ultra Mac Studio (J775D), the thread that receives GPU completions keeps its default scheduling instead of having its minimum CPU utilization lowered while the GPU looks like the bottleneck. Completion wakeups then never wait on a lower CPU clock on that thread's cluster. Integration's port lowers the receive thread's utilization floor while the CPU refill detector reports the GPU as the bottleneck, and restores it otherwise, at every publication and completion. On M5 no utilization handle is attached (no rx_util_min handle is taken), so rx_update() returns at once and the thread keeps the default. G17P is unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 3df07ca113b7afb554ee0060b67e07ddedd9165b in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 2 ++ drivers/gpu/drm/asahi/g17/port.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f1e3a813e3f62e..054b477db4cbda 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1786,6 +1786,7 @@ impl Gpu { session.init.port_views(mailbox)?, session.bridge.regs.clone(), shared.variant.registration, + !shared.variant.m5_commands, ) .inspect_err(at("port"))?; session @@ -1943,6 +1944,7 @@ impl Gpu { session.init.port_views(mailbox)?, session.bridge.regs.clone(), shared.variant.registration, + !shared.variant.m5_commands, )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 5e759bb2b7d18a..079c93d063d447 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -188,13 +188,16 @@ impl Port { views: Views, regs: crate::regs::G17Resources, registration: super::fw::variant::QueueRegistration, + clamp_rx: bool, ) -> Result { // SAFETY: The session owns the mailbox and outlives every port user. let rx = unsafe { (*views.mailbox).rx_util_min_handle() }; + // Without `clamp_rx` (M5) no handle is attached and the receive thread keeps its + // default; rx_update() then returns at once. if self.views.populate(views) && self.regs.populate(regs) && self.registration.populate(registration) - && self.rx.populate(rx) + && (!clamp_rx || self.rx.populate(rx)) { Ok(()) } else { @@ -455,10 +458,10 @@ impl Port { Ok(()) } - /// Requests the receive thread's minimum utilization for the feed's current view of the GPU. - /// Called at every publication and completion: the feed's state lingers for a few - /// milliseconds, so after a pause the publication that ends it restores the default before - /// its completion arrives. + /// Requests the receive thread's minimum utilization for the feed's current view of the GPU, + /// where the port clamps it (not on M5, see attach()). Called at every publication and + /// completion: the feed's state lingers for a few milliseconds, so after a pause the + /// publication that ends it restores the default before its completion arrives. fn rx_update(&self) { let Some(rx) = self.rx.as_ref() else { return; From 2424dcaeb8a80c01e40c5dc39c5b4d28362f4326 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 13:30:58 -0800 Subject: [PATCH 144/171] rust: workqueue: Let drivers queue work near a CPU, for 15% faster M5 Ultra GPU Rust drivers could only queue work on the system queues or on an ordered queue of their own, so they had no way to keep deferred work near the CPU that needs its result. On multi-die parts that work could land on the other die and pay for moving its data across. On the M5 Ultra (J775D), the asahi driver uses this to run its GPU completion work on the submitting app's die, which alone makes a round of 32 GPU jobs about 15% faster (1.306 to 1.131 ms) for an app on the other die. The gain takes effect with that driver commit. Add Queue::enqueue_on(), which queues a work item for a given CPU: on a per-CPU queue the work runs there, on an unbound queue the CPU selects the affinity pod whose worker runs it. Add OwnedQueue::new_unbound() for an unbound driver queue, optionally with high-priority workers, OwnedQueue::new_ordered_reclaim() for an ordered queue on the memory reclaim path, and OwnedQueue::restrict(), which limits a queue's workers to a set of CPUs and keeps an ordered queue ordered and a high-priority one at its priority. Queue::as_raw() hands a queue to C interfaces, and Cpumask::cluster_of() returns the CPUs of a CPU's cluster from the architecture topology. These let a driver keep follow-up work near the CPU that submitted it instead of where an interrupt thread happened to run. Signed-off-by: Jeff Queisser (cherry picked from commit 72c297332070fc56dc1e59cde35c6eee45728e98 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- rust/helpers/cpumask.c | 7 +++ rust/helpers/workqueue.c | 24 +++++++++ rust/kernel/cpumask.rs | 8 +++ rust/kernel/workqueue.rs | 104 ++++++++++++++++++++++++++++++++++----- 4 files changed, 130 insertions(+), 13 deletions(-) diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index 5deced5b975e09..c3366c96822b48 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include __rust_helper void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) @@ -81,3 +82,9 @@ void rust_helper_free_cpumask_var(cpumask_var_t mask) free_cpumask_var(mask); } #endif + +__rust_helper +const struct cpumask *rust_helper_topology_cluster_cpumask(int cpu) +{ + return topology_cluster_cpumask(cpu); +} diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index 7cd3b000a5b6bf..657ceb2ccf6d35 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include __rust_helper void rust_helper_init_work_with_key(struct work_struct *work, @@ -20,3 +21,26 @@ rust_helper_alloc_ordered_workqueue(const char *name, unsigned int flags) { return alloc_ordered_workqueue("%s", flags, name); } + +__rust_helper struct workqueue_struct * +rust_helper_alloc_unbound_workqueue(const char *name, bool highpri) +{ + return alloc_workqueue("%s", WQ_UNBOUND | (highpri ? WQ_HIGHPRI : 0), 0, name); +} + +__rust_helper int rust_helper_workqueue_restrict(struct workqueue_struct *wq, + const struct cpumask *cpus, + bool ordered, bool highpri) +{ + struct workqueue_attrs *attrs = alloc_workqueue_attrs(); + int ret; + + if (!attrs) + return -ENOMEM; + cpumask_copy(attrs->cpumask, cpus); + attrs->nice = highpri ? MIN_NICE : 0; + attrs->ordered = ordered; + ret = apply_workqueue_attrs(wq, attrs); + free_workqueue_attrs(attrs); + return ret; +} diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 44bb36636ee323..3b82790eb03402 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -77,6 +77,14 @@ impl Cpumask { unsafe { &*ptr.cast() } } + /// The CPUs of `cpu`'s cluster as the architecture's topology describes it: `cpu` alone when + /// there is no cluster information. + pub fn cluster_of(cpu: CpuId) -> &'static Self { + // SAFETY: The topology keeps one mask per possible CPU for the kernel's lifetime, and `cpu` + // is a valid CPU ID by its type invariant. + unsafe { Self::from_raw(bindings::topology_cluster_cpumask(cpu.as_u32() as i32)) } + } + /// Obtain the raw `struct cpumask` pointer. pub fn as_raw(&self) -> *mut bindings::cpumask { let this: *const Self = self; diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 3124062c3cad94..0052a68a873bd0 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -298,6 +298,33 @@ impl Queue { } } + /// Returns the raw `struct workqueue_struct` pointer, for C interfaces that take a queue. + pub fn as_raw(&self) -> *mut bindings::workqueue_struct { + self.0.get() + } + + /// Enqueues a work item on `cpu`. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// On a per-CPU queue the work item runs on `cpu`. On an unbound queue `cpu` only selects the + /// affinity pod (for example the CPU's last-level cache) whose worker runs it, so the + /// scheduler may pick another CPU of that pod. + pub fn enqueue_on(&self, cpu: crate::cpu::CpuId, w: W) -> W::EnqueueOutput + where + W: RawWorkItem + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: As for `enqueue`. `cpu` is a valid CPU ID by its type invariant, which is what + // `queue_work_on` requires in place of `WORK_CPU_UNBOUND`. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on(cpu.as_u32() as ffi::c_int, queue_ptr, work_ptr) + }) + } + } + /// Enqueues a delayed work item. /// /// This may fail if the work item is already enqueued in a workqueue. @@ -374,17 +401,23 @@ impl WorkItem for ClosureWork { /// A work queue owned by the caller. /// -/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()`. It dereferences -/// to a [`Queue`], so work items are enqueued on it exactly like on the system queues. Dropping it -/// calls `destroy_workqueue()`, which waits for every pending work item to finish first. +/// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()` or, for an +/// unbound queue, `alloc_workqueue(WQ_UNBOUND)`. It dereferences to a [`Queue`], so work items +/// are enqueued on it exactly like on the system queues. Dropping it calls `destroy_workqueue()`, +/// which waits for every pending work item to finish first. /// -/// An owned queue gives a driver two things the system queues do not: its work items run one at a -/// time in queueing order, and the driver can wait for all of them at once (see [`drain`]) before -/// it tears down the state they use. +/// An ordered owned queue gives a driver two things the system queues do not: its work items run +/// one at a time in queueing order, and the driver can wait for all of them at once (see +/// [`drain`]) before it tears down the state they use. An unbound owned queue runs its work items +/// in the affinity pod of the CPU they are queued for (see [`Queue::enqueue_on`]), and either kind +/// can be restricted to a set of CPUs (see [`restrict`]). /// /// # Invariants /// -/// The pointer is valid and points at a workqueue allocated by `alloc_ordered_workqueue()`. +/// `queue` is valid and points at a workqueue allocated by `alloc_ordered_workqueue()` when +/// `ordered` is set, or by `alloc_workqueue(WQ_UNBOUND)` with `WQ_HIGHPRI` when `highpri` is set. +/// +/// [`restrict`]: OwnedQueue::restrict /// /// # Examples /// @@ -422,7 +455,11 @@ impl WorkItem for ClosureWork { /// ``` /// /// [`drain`]: OwnedQueue::drain -pub struct OwnedQueue(NonNull); +pub struct OwnedQueue { + queue: NonNull, + ordered: bool, + highpri: bool, +} impl OwnedQueue { /// Allocates an ordered workqueue named `name`. @@ -431,11 +468,52 @@ impl OwnedQueue { /// they were queued, by an unbound worker. The name is copied by the kernel and appears in the /// worker's thread name. pub fn new_ordered(name: &CStr) -> Result { + Self::ordered(name, 0) + } + + /// Allocates an ordered workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), such as a GPU scheduler's submission queue. + pub fn new_ordered_reclaim(name: &CStr) -> Result { + Self::ordered(name, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn ordered(name: &CStr, flags: u32) -> Result { // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. - let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), 0) }; + let ptr = unsafe { bindings::alloc_ordered_workqueue(name.as_char_ptr(), flags) }; // INVARIANT: `alloc_ordered_workqueue()` returns a valid workqueue or NULL. - Ok(Self(NonNull::new(ptr).ok_or(ENOMEM)?)) + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: true, + highpri: false, + }) + } + + /// Allocates an unbound workqueue named `name`, with high-priority workers if `highpri`. + /// + /// Its work items run concurrently, each in the affinity pod of the CPU it is queued for. + pub fn new_unbound(name: &CStr, highpri: bool) -> Result { + // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. + let ptr = unsafe { bindings::alloc_unbound_workqueue(name.as_char_ptr(), highpri) }; + + // INVARIANT: The helper returns a valid unbound workqueue, high priority if asked, or NULL. + Ok(Self { + queue: NonNull::new(ptr).ok_or(ENOMEM)?, + ordered: false, + highpri, + }) + } + + /// Restricts the queue's workers to the CPUs in `cpus`. + /// + /// An ordered queue stays ordered and a high-priority one keeps its priority. Allocates, so it + /// must be called from a context that may sleep. + pub fn restrict(&self, cpus: &crate::cpumask::Cpumask) -> Result { + // SAFETY: The queue is a valid unbound workqueue by the type invariant and `cpus` is a + // valid mask; the helper copies it. + crate::error::to_result(unsafe { + bindings::workqueue_restrict(self.queue.as_ptr(), cpus.as_raw(), self.ordered, self.highpri) + }) } /// Waits until the workqueue is empty. @@ -446,7 +524,7 @@ impl OwnedQueue { /// stop every other producer before draining. Must be called from a context that may sleep. pub fn drain(&self) { // SAFETY: The pointer is valid by the type invariant. - unsafe { bindings::drain_workqueue(self.0.as_ptr()) }; + unsafe { bindings::drain_workqueue(self.queue.as_ptr()) }; } } @@ -456,7 +534,7 @@ impl Deref for OwnedQueue { fn deref(&self) -> &Queue { // SAFETY: The pointer is valid by the type invariant and stays valid for as long as `self` // does, which bounds the returned reference. - unsafe { Queue::from_raw(self.0.as_ptr()) } + unsafe { Queue::from_raw(self.queue.as_ptr()) } } } @@ -464,7 +542,7 @@ impl Drop for OwnedQueue { fn drop(&mut self) { // SAFETY: The pointer is valid by the type invariant. `destroy_workqueue()` drains the // queue before freeing it, so no work item can run on a freed queue. - unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + unsafe { bindings::destroy_workqueue(self.queue.as_ptr()) }; } } From 99062a762c224d2890d2d56435a64c41cff28aa7 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 13:31:52 -0800 Subject: [PATCH 145/171] rust: drm: sched: Let drivers place scheduler work, for 4% faster M5 Ultra GPU A Rust DRM driver's scheduler always ran its run and free work on a queue drm_sched allocated itself, wherever an unbound worker happened to be, so the driver couldn't keep that work near the CPUs submitting jobs. On the M5 Ultra (J775D), the asahi driver uses this to keep each scheduler's work in its creating CPU's cluster, which saves about 45 us a round of 32 GPU jobs on every CPU, about 4%. The gain takes effect with that driver commit. Add Scheduler::new_with_submit_wq(), which passes a driver-owned queue to drm_sched_init() as the scheduler's submission workqueue instead of letting drm_sched allocate its own. The scheduler takes ownership of the queue and destroys it after drm_sched_fini(), so no run or free work can be left on a freed queue; a failed initialization drops it with the error. Like drm_sched's own, the queue must be ordered and usable for memory reclaim. A driver can then restrict where its scheduler's work runs. Signed-off-by: Jeff Queisser [Eduard Moldovan: the KUnit cases aurora-silicon adds to its rust_drm_sched test module are left out; integration/aurora has no such module] (cherry picked from commit 69dfb1dfbfbe3720617e8c5ffabf9f91e32e6e5a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- rust/kernel/drm/sched.rs | 68 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 5d0c785e103a65..ad98317711b5de 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -458,6 +458,9 @@ pub struct SchedulerInner { // SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, + /// The driver's submission queue, if it supplied one. Fields drop after `Drop::drop`, so the + /// queue is destroyed only after `drm_sched_fini` has stopped the scheduler's work on it. + _submit_wq: Option, } impl Drop for SchedulerOwner { @@ -506,8 +509,58 @@ impl Scheduler { timeout_ms: time::Msecs, name: &'static CStr, ) -> Result> { - let mut sched: UniqueArc>> = - UniqueArc::new_uninit(GFP_KERNEL)?; + Self::init( + UniqueArc::new_uninit(GFP_KERNEL)?, + device, + num_rqs, + credit_limit, + hang_limit, + timeout_ms, + name, + None, + ) + } + + /// Creates a new DRM Scheduler object whose run and free work goes to `submit_wq`. + /// + /// The scheduler owns the queue and destroys it after it has been finalized. Like the queue + /// drm_sched allocates itself, it must be ordered and usable for memory reclaim (see + /// [`OwnedQueue::new_ordered_reclaim`]), so that one scheduler's jobs run and are freed one at a + /// time. + /// + /// [`OwnedQueue::new_ordered_reclaim`]: crate::workqueue::OwnedQueue::new_ordered_reclaim + pub fn new_with_submit_wq( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + submit_wq: crate::workqueue::OwnedQueue, + ) -> Result> { + Self::init( + UniqueArc::new_uninit(GFP_KERNEL)?, + device, + num_rqs, + credit_limit, + hang_limit, + timeout_ms, + name, + Some(submit_wq), + ) + } + + /// Initializes a scheduler in `sched`, whatever that storage held before. + fn init( + mut sched: UniqueArc>>, + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + submit_wq: Option, + ) -> Result> { // Allocate the owner before starting C workers: no fallible allocation // may abandon an initialized scheduler without running drm_sched_fini. let owner: UniqueArc>> = @@ -521,7 +574,7 @@ impl Scheduler { let init_ops = bindings::drm_sched_init_args { ops: &Self::OPS, - submit_wq: core::ptr::null_mut(), + submit_wq: submit_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), timeout_wq: core::ptr::null_mut(), num_rqs, credit_limit, @@ -544,6 +597,13 @@ impl Scheduler { // SAFETY: C initialized sched and PhantomData has no stored state. // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); - Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) + Ok(Scheduler( + owner + .write(SchedulerOwner { + allocation, + _submit_wq: submit_wq, + }) + .into(), + )) } } From e277e03b94838e9f2b4b190378e32ca88ca7b6e2 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 13:45:08 -0800 Subject: [PATCH 146/171] drm/asahi: g17: M5: Make M5 Ultra GPU jobs complete up to 20% faster on either die On the M5 Ultra (J775D), a round of 32 back-to-back GPU jobs now takes 1.085-1.102 ms from every CPU of both dies, where an app on the other die from the GPU's completion path took about 1.306 ms: up to 20% faster, and no longer dependent on where the app or the boot landed. The GPU's mailbox receive IRQ is threaded, and the AIC cannot re-affine it, so its thread stays wherever it first ran. The event work was queued on the per-CPU high-priority queue from that thread and ran on its CPU, and each scheduler's run and free work ran wherever an unbound worker was. On M5, queue the event work on an unbound high-priority queue of the driver's, aimed at the CPU that last published work: it runs in that CPU's affinity pod, its L2 cluster, wherever the IRQ thread is. Each scheduler gets an ordered submission queue restricted to the cluster of the CPU that created it. No die or CPU is named, so a one-die part behaves the same. The queues are freed at unbind, after the event work has been disabled. Other G17 GPUs are unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 5bf870bd80d09618bcc1ca2dcccb5597d7eb37fb in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/job.rs | 16 +++++++++- drivers/gpu/drm/asahi/g17/mod.rs | 32 +++++++++++++++++++- drivers/gpu/drm/asahi/g17/runtime.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 3 ++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index e42c0ce76f966c..da94d77aacaa9f 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -7,6 +7,9 @@ use crate::driver::AsahiDrmDevice; use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ bindings, c_str, + cpu::CpuId, + cpumask::Cpumask, + workqueue, dma_fence::{Fence, RawDmaFence}, drm::sched, prelude::*, @@ -303,6 +306,8 @@ pub(crate) trait Backend: Send + Sync { /// The device accepts command submission (an M5 GPU before its queues take submission /// does not). fn supports_submission(&self) -> bool { true } + /// Whether this backend's scheduler workers run in their queue creator's cluster (M5). + fn creator_cluster_scheduling(&self) -> bool { false } fn supports_engine(&self, _engine: Engine) -> bool { true } /// Device policy applied to a validated command before it is ordered (the M5 resource /// plan). @@ -466,7 +471,16 @@ impl Scheduler { Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), }; - let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let device = dev.as_ref().as_ref(); + let scheduler = if backend.creator_cluster_scheduling() { + // The run and free work stays in the creating CPU's cluster, as the event work + // follows the submitter, instead of wherever an unbound worker last ran. + let queue = workqueue::OwnedQueue::new_ordered_reclaim(name)?; + queue.restrict(Cpumask::cluster_of(CpuId::current()))?; + sched::Scheduler::new_with_submit_wq(device, 1, credits, 0, TIMEOUT_MS, name, queue)? + } else { + sched::Scheduler::new(device, 1, credits, 0, TIMEOUT_MS, name)? + }; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; Ok(Self { entity, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 054b477db4cbda..f98ce67b0eeb86 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -811,6 +811,7 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.primary.state.shared.note_submitter(); self.port.note_gpu_activity(); self.primary.state.shared.note_gpu_activity(); self.effort.submit(); @@ -1075,6 +1076,12 @@ struct Shared { /// When work was last published while the governor is requested (0: none since its last /// step). Publishers without the device mutex set it too. governor_activity_ns: AtomicU64, + /// M5: the event work's queue. Unbound, so the event work runs in the affinity pod (the L2) + /// of the CPU that last published work, not on whichever CPU the completion IRQ thread first + /// ran on. On a two-die part that keeps completion processing on the submitter's die. + events: Option, + /// M5: the CPU of the last publication, `u32::MAX` before the first. + submitter: AtomicU32, #[pin] grow: Work, #[pin] @@ -1245,7 +1252,23 @@ impl Shared { return; } // Already queued if this fails, which is just as good. - let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + let _ = match (&self.events, self.submitter()) { + (Some(events), Some(cpu)) => { + events.enqueue_on::, EVENT_WORK>(cpu, self.clone()) + } + _ => workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()), + }; + } + + /// Records the publishing CPU for the event work's placement (M5). + fn note_submitter(&self) { + if self.events.is_some() { + self.submitter.store(kernel::cpu::CpuId::current().as_u32(), Ordering::Relaxed); + } + } + + fn submitter(&self) -> Option { + kernel::cpu::CpuId::from_u32(self.submitter.load(Ordering::Relaxed)) } fn queue_grow(self: &Arc) { @@ -1490,6 +1513,11 @@ impl Gpu { let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; // The bare-metal M5 boot runs both engines; other M5 profiles take no submission. let submits = !m5 || cfg.profile == FirmwareProfile::M5PublishedRoots; + let events = if m5 { + Some(workqueue::OwnedQueue::new_unbound(c"aurora_g17_events", true)?) + } else { + None + }; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1523,6 +1551,8 @@ impl Gpu { governor <- new_delayed_work!("g17::Shared::governor"), governor_armed: AtomicBool::new(false), governor_activity_ns: AtomicU64::new(0), + events, + submitter: AtomicU32::new(u32::MAX), grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 1c9aa00b6a95df..10557c655194cd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1536,6 +1536,7 @@ impl compute::Host for ComputeHost<'_> { self.port.notify(message) } fn note_submission(&mut self) -> Result { + self.shared.note_submitter(); self.shared.note_gpu_activity(); self.effort.submit(); self.effort.note_waiters(super::now_ns(), super::fence::waiters()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index afee32d794e24a..ee40f6a3c63c35 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -118,6 +118,9 @@ impl Backend { } impl job::Backend for Backend { + fn creator_cluster_scheduling(&self) -> bool { + self.shared.events.is_some() + } fn input_sources(&self, inputs: &[crate::file::SyncItem]) -> Option> { From e943aa8e58e68b89fe3d2907f22097ff369075c5 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 13:55:24 -0800 Subject: [PATCH 147/171] drm/asahi: g17: M5: Fix one client's binds being able to crash the GPU for every client On M5, a client could bind its own memory over the kernel's mappings in the compact window at 64 GiB of its VM, where the kernel places the parameter buffers' page lists, render targets, scratch and the metrics records the firmware reads. The bind replaced the kernel's page-table entries, so the GPU then used client memory in place of that state, and the later unmap of either mapping cleared the other's. An M5 client VM now reserves the compact window against VM_BIND from creation, as it already reserves the queue arenas. The bind check refuses only ranges registered as driver mappings, and the compact window was never registered in a client VM: its mappings were placed there at render or compute setup, after the VM existed. The kernel range a client names at VM_CREATE need not cover the window, so the client's own range check did not catch it either. Mesa's M5 layout keeps that window inside its own driver reservation, so no Mesa bind is refused. Signed-off-by: Jeff Queisser (cherry picked from commit 3acf8c75683d01fd70c802a1f8f906eef0d4e1ef in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/mod.rs | 12 +++++++----- drivers/gpu/drm/asahi/mmu.rs | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f98ce67b0eeb86..56752e1194db65 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2224,12 +2224,14 @@ impl gpu::Gpu for Gpu { .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; if self.cfg.profile.is_m5() { - // The canonical arena the client's queues alias and the client-private one stay - // reserved against userspace binds from creation, so no bind replaces a kernel alias - // the firmware reads. + // The canonical arena the client's queues alias, the client-private one and the + // compact window, where the kernel places the parameter buffers' page lists, render + // targets, scratch and metrics records, stay reserved against userspace binds from + // creation, so no bind replaces a kernel alias the firmware reads. The client's own + // kernel range need not cover the compact window. let ranges = firmware.uat.m5_client_reserved_ranges(); - let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; - for range in ranges { + let mut reserved = KVec::with_capacity(ranges.len() + 1, GFP_KERNEL)?; + for range in Iterator::chain(ranges.into_iter(), [buffer::BlockGeometry::M5_COMPACT]) { reserved.push(range, GFP_KERNEL)?; } vm.install_driver_mappings(KVec::new(), reserved)?; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2c9443b2977691..6be617824f3a8f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3693,8 +3693,9 @@ impl Uat { } } - /// Ranges an M5 client VM keeps from userspace binds, even while they hold - /// nothing: the canonical arena its queues alias and the client-private one. + /// The MMU's arenas an M5 client VM keeps from userspace binds, even while they + /// hold nothing: the canonical arena its queues alias and the client-private one. + /// The client VM also reserves the compact window (`BlockGeometry::M5_COMPACT`). pub(crate) fn m5_client_reserved_ranges(&self) -> [Range; 2] { [layout::CANONICAL_RANGE, layout::CLIENT_PRIVATE_RANGE] } From e746e4c539f1b831142ffa7dff76c830196c8064 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:02 -0800 Subject: [PATCH 148/171] drm/asahi: g17: M5: Fix hung GPU work running on with no time limit On M5, GPU work that never finished could hold its queue, queue ID, render pairs, pool backing and memory for as long as other clients kept the GPU busy, or for as long as its owner kept the queue open. Every such path is now bounded: - A job queued behind an older node of its own context was renewed by the watchdog at every poll, with no count, while any client kept the firmware's device-wide progress word moving. When the older node was failed work held until the firmware drains it, the renewal had no end. It now stops once the VM has failed work, mirroring the rule that a failed blocker of another VM excuses nothing, and it is bounded by the existing thirty-second shared budget counted on the job's deferrals. The older node's own job runs under the same allowances and was published earlier, so a healthy older node drains or fails within that window. - A compute command the watchdog timed out was never killed while its owner kept the queue open: a context kill was published only for a released queue or a closed context. A timed-out queue's context is now killed once the quarantine delay has passed; its VM has already failed, so the owner loses nothing it could still use. Spared work, which a recovery returns to service, is not killed, and a running ray traversal is still never killed, as before. - The kill of a closed context with a timed-out command was postponed while any other client had work in flight and the GPU retired something every five seconds, on the assumption that its commands retire like any others. A timed-out command now counts as failed compute for the kill, as a firmware-reported fault already does, and any closed context's kill waits at most the recovery bound after it closed. Signed-off-by: Jeff Queisser (cherry picked from commit c4f8ad4927dbf5a448dda97ba556220aeca447a5 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 6 +++ .../gpu/drm/asahi/g17/queue/compute/exit.rs | 31 ++++++++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 2 +- .../drm/asahi/g17/runtime/compute_control.rs | 12 +++-- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 44 +++++++++++++++---- drivers/gpu/drm/asahi/g17/timeout.rs | 33 +++++++++++--- 6 files changed, 108 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 6fd44f0b7e2db6..e2836df906ea16 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -601,6 +601,8 @@ pub(crate) struct Queue { needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, + /// Monotonic time the watchdog quarantined this queue, zero otherwise. + timed_out_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, /// `None` after an exited owner was detached; a new owner binds afresh. @@ -691,6 +693,7 @@ impl Queue { growth: None, needs_fresh_backing: false, released_at: 0, + timed_out_at: 0, free_list_generation: 0, binding: Some(binding), previous: None, @@ -1296,6 +1299,7 @@ impl Queue { if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; + self.timed_out_at = 0; self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; @@ -1885,6 +1889,7 @@ impl Queue { self.exit = None; self.quarantined = false; self.quarantine_error = None; + self.timed_out_at = 0; self.failure_status_pending = false; self.retired_by_teardown = false; self.spared_quarantine = false; @@ -2125,6 +2130,7 @@ impl Queue { let result = self.quarantine(ETIMEDOUT, defer); if fresh { self.spared_quarantine = return_spared; + self.timed_out_at = now_ns().max(1); } result } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index b731e8e24c38c9..ace49e373311ff 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -102,7 +102,7 @@ impl Queue { } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - && self.released + && (self.released || self.timed_out()) && self.active.len != 0 && self.exit.is_none() && (!self.active_traversal() || self.fault_pending()) @@ -114,6 +114,35 @@ impl Queue { CANCELLED_DELAY_NS } } + /// M5: the watchdog failed this queue's command, and nothing has killed it yet. Its + /// commands may never retire on their own (a dispatch that does not terminate), so a kill + /// of its context treats it as failed work, like a command the firmware reported failed. + pub(crate) fn timed_out(&self) -> bool { + self.graph.m5 + && self.quarantined + && self.quarantine_error == Some(ETIMEDOUT) + && !self.spared_quarantine + && self.exit.is_none() + } + /// M5: a timed-out queue whose owner keeps it open, due for a kill of its context, with + /// how long ago the watchdog failed it. The owner's VM has already failed, so it loses + /// nothing it could still use; without the kill a command that never terminates keeps + /// running, and keeps this queue, until the owner closes. + pub(crate) fn open_timeout_kill(&self, now: u64) -> Option<(Arc, u64)> { + if self.released || self.timed_out_at == 0 || !self.timed_out() { + return None; + } + let context = self.context()?.clone(); + let age = now.saturating_sub(self.timed_out_at); + self.exit_candidate(&context, age).then_some((context, age)) + } + /// An open timed-out queue's kill is due within the exit polling window. + pub(crate) fn open_timeout_polling(&self, now: u64) -> bool { + !self.released + && self.timed_out_at != 0 + && self.timed_out() + && now.saturating_sub(self.timed_out_at) < POLL_NS + } /// A kill of `context` was published through this queue. pub(crate) fn exit_published(&self, context: &Arc) -> bool { self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index ee40f6a3c63c35..c32bd5012d5661 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -386,7 +386,7 @@ impl job::Backend for Backend { if let Some(deferred) = deferred.as_mut() { firmware.scan_renders(deferred)?; } - if let Some((older, own)) = self.behind_own_head(packet, &firmware.init) { + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { if renewals == 0 { dev_info!(self.shared.dev.as_ref(), "Render watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 94d1e68197c06b..dfa343a9ac2753 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,7 +20,7 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - if let Some((older, own)) = self.behind_own_head(packet, &firmware.init) { + if let Some((older, own)) = self.behind_own_head(packet, &firmware.init, renewals) { if renewals == 0 { dev_info!(self.shared.dev.as_ref(), "Compute watchdog owner={} behind its context's older node {:#x} (own {:#x}); the firmware progresses: renewed\n", @@ -58,15 +58,19 @@ impl Backend { } /// The older node of the packet's context ahead of its own, with its own word, when the - /// firmware's progress word moved within the watchdog's interval + /// firmware's progress word moved within the watchdog's interval, the packet's VM has no + /// failed work and the job's deferrals are within budget /// ([`crate::g17::timeout::behind_own_head`]). The word is read at every poll. Firmware /// without the word renews nothing by this rule. pub(super) fn behind_own_head(&self, packet: &Arc, - init: &crate::g17::initdata::InitData) -> Option<(u32, u32)> { + init: &crate::g17::initdata::InitData, renewals: u8) -> Option<(u32, u32)> { let work = packet.completion.work_state().ok()?; let older = work.older_node(); let progressing = init.m5_progress().is_ok_and(crate::g17::timeout::note_progress); - crate::g17::timeout::behind_own_head(packet.is_published(), older, progressing) + let status = packet.completion.status(); + let failed = status.get() != 0 || status.reported_error() != 0; + crate::g17::timeout::behind_own_head(packet.is_published(), older, progressing, + failed, renewals) .then(|| (older.unwrap_or(0), work.word())) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index a354ead64dcf95..5b9044b9638b09 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,7 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, - timeout::LIVENESS_NS, + timeout::{LIVENESS_NS, RECOVERY_WAIT_MS}, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; @@ -201,6 +201,28 @@ impl Firmware { ); } } + // M5: a command the watchdog failed is killed once its quarantine delay has passed, + // also while its owner keeps the queue open (`Queue::open_timeout_kill`). + for index in 0..self.queues.compute.len() { + let due = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| queue.open_timeout_kill(now)); + let Some((context, age)) = due else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now, &mut pressed) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } Ok(()) } @@ -242,7 +264,7 @@ impl Firmware { } return Ok(()); } - } else if self.kill_unsafe(context, now) { + } else if self.kill_unsafe(context, age, now) { return Ok(()); } let (owner, target) = { @@ -444,13 +466,15 @@ impl Firmware { } /// A compute command of `context` failed and the firmware holds commands behind it that it - /// never runs. `publish_compute_exit` is only reached once `teardown_idle` found all of - /// the context's render pairs idle; the pairs behind such a failure become idle when + /// never runs, or, on M5, the watchdog failed one whose commands may never retire + /// (`Queue::timed_out`). `publish_compute_exit` is only reached once `teardown_idle` found + /// all of the context's render pairs idle; the pairs behind such a failure become idle when /// their commands have executed (`Pair::firmware_drained`), so by then nothing of the /// context runs any more. fn compute_failed(&self, context: &Arc) -> bool { self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).any(|queue| { - queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && queue.fault_pending() + queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + && (queue.fault_pending() || queue.timed_out()) }) } @@ -462,9 +486,13 @@ impl Firmware { /// kill proceeds. A context whose compute failed ([`Self::compute_failed`]) /// is not left alone, since its held commands never retire: its kill waits /// only for other contexts' compute ([`Self::foreign_compute_running`]). - fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + /// + /// On M5 the wait ends [`RECOVERY_WAIT_MS`] after the context closed (`age`): a closed + /// context's command that never terminates would otherwise hold its queue, pairs and + /// memory for as long as other clients keep the GPU busy. + fn kill_unsafe(&self, context: &Arc, age: u64, now: u64) -> bool { let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); - if stalled { + if stalled || (self.init.is_m5() && age >= u64::from(RECOVERY_WAIT_MS) * 1_000_000) { return false; } self.queues.render.foreign_in_flight(context) @@ -486,7 +514,7 @@ impl Firmware { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_polling(now)) + .any(|queue| queue.exit_polling(now) || queue.open_timeout_polling(now)) { return true; } diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index 077bf97567de8f..3da5e255408b5a 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -182,12 +182,20 @@ pub(crate) fn note_progress(word: u32) -> bool { /// Work whose context has an older node in use ahead of its own is not late: the firmware /// lists a context's nodes by their words and flushes from the head, so this work's stamps wait -/// behind the older node until it drains. That wait has no ceiling while the firmware works, so -/// a published job behind its context's older node is renewed, without a count, while the -/// firmware's progress word moves; a firmware that stops progressing for the interval expires it -/// as before, and an older node of another VM still goes through [`attribute`]. -pub(crate) fn behind_own_head(published: bool, older: Option, progressing: bool) -> bool { - published && older.is_some() && progressing +/// behind the older node until it drains. A published job behind its context's older node is +/// renewed while the firmware's progress word moves, with two limits: +/// +/// - Not once its VM has failed work (`failed`). The older node's holder may be that failed +/// work, held until the firmware drains it, and a failed blocker excuses nothing ([`decide`]). +/// - At most [`SHARED_RENEWALS`] times, counted with the job's other deferrals (`renewals`). +/// The older node's own job runs under the same allowances and started earlier, so a healthy +/// older node drains or fails within that budget; past it the job times out as any other. +/// +/// A firmware that stops progressing for the interval expires the job as before, and an older +/// node of another VM still goes through [`attribute`]. +pub(crate) fn behind_own_head(published: bool, older: Option, progressing: bool, + failed: bool, renewals: u8) -> bool { + published && older.is_some() && progressing && !failed && renewals < SHARED_RENEWALS } #[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] @@ -317,6 +325,19 @@ mod tests { assert!(!renew_shared(true, true, &mut renewals)); } #[test] + fn work_behind_its_own_context_is_renewed_only_while_healthy_and_bounded() { + let older = Some(7); + for renewals in 0..SHARED_RENEWALS { + assert!(behind_own_head(true, older, true, false, renewals)); + } + assert!(!behind_own_head(true, older, true, false, SHARED_RENEWALS)); + assert!(!behind_own_head(true, older, true, false, u8::MAX)); + assert!(!behind_own_head(true, older, true, true, 0)); + assert!(!behind_own_head(true, None, true, false, 0)); + assert!(!behind_own_head(true, older, false, false, 0)); + assert!(!behind_own_head(false, older, true, false, 0)); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; From b27413af73ce4ddb41d4617a03d08db3046c2630 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:04 -0800 Subject: [PATCH 149/171] drm/asahi: g17: Fix a process keeping access to a GPU queue after another process takes it Physical compute queues and render pairs move between processes, but the client-visible state behind them did not move cleanly with them. A previous owner could keep reading and writing the state the next owner's GPU work used, and the next owner could read what the previous owner's commands left. Each handover now gives the next owner state the previous owner can no longer write, and clears what it left: - Compute: when a queue moved to another VM, the previous VM kept read-write mappings of the queue's kick ring, preemption buffer, usage page and operand state until the new owner's first command retired, and the kick ring kept the previous owner's entries. The previous owner's mappings are now removed before the new owner's are created, and the kick ring is cleared with the other state pages. A move needs an idle queue whose pool release the firmware consumed, so the firmware no longer uses those views; the previous context's scheduler state, which the queue may still run until the new owner's first command, stays retained as before. If the new owner releases the queue before publishing anything, the queue is left detached, as for an exited owner, instead of restoring the removed views. - Render: a parked pair goes to whichever queue takes its render slot next, and storage of a ring that never wrote an entry returns to a stock any VM's next ring could take. The rings kept their contents, and stock storage handed to another VM stayed mapped read-write in the VM that created it. A parked pair's rings are now cleared when it goes to another VM's context, and stock storage is reused only by the VM that created it. A parked pair moved to another VM still leaves the previous VM's mapping of its rings in place, since that mapping lasts as long as the VM. The read-only client kick rings later in this series keep that VM from writing the new owner's entries. Signed-off-by: Jeff Queisser (cherry picked from commit b98a430aa530210c4d010dc0a1b079fa055a984a in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/kick.rs | 40 ++++++++++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 61 ++++++++++++++++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 18 ++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 1 + 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 46a1d8973774cf..860d07628ea410 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -403,6 +403,8 @@ struct RenderStorage { // The lower alias drops before its backing. low: mmu::KernelMapping, high: KernelObject, + /// The client VM the storage was created for, whose alias of it lasts as long as that VM. + vm: u64, } /// Render ring storage outlives its rings for the life of the device. @@ -442,9 +444,14 @@ impl RenderRings { ) } - fn take(&self) -> Option { + /// Reusable storage created for `vm`. Storage of another VM is never handed out: that VM + /// keeps its read-write alias of the pages, so it could read and write the new ring. + fn take(&self, vm: u64) -> Option { let mut stock = self.stock.lock(); - let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let index = stock + .retired + .iter() + .rposition(|(reusable, storage)| *reusable && storage.vm == vm)?; let last = stock.retired.len() - 1; stock.retired.swap(index, last); stock.retired.pop().map(|(_, storage)| storage) @@ -567,7 +574,8 @@ impl Queue { if id.engine == DataMaster::Compute { return Err(EINVAL); } - if let Some(storage) = rings.take() { + let vm = client.id(); + if let Some(storage) = rings.take(vm) { let mut ring = RenderRing { storage: ManuallyDrop::new(storage), rings: rings.clone(), @@ -621,7 +629,7 @@ impl Queue { Err(error) => return Err(error), } }; - let storage = RenderStorage { low, high }; + let storage = RenderStorage { low, high, vm }; // Count the storage only once it exists. Without room to return it, // nothing has aliased it but `client`, which drops first. if let Err(error) = rings.admit() { @@ -671,6 +679,30 @@ impl Queue { .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) } + /// Clears an idle compute ring before another VM is given a view of it, so the new owner + /// cannot read the entries the previous owner's commands left. The firmware has consumed + /// every entry; the host's position in the ring is kept. + pub(crate) fn clear_ring(&mut self) -> Result { + let Backing::Compute { low, .. } = &mut self.backing else { + return Err(EINVAL); + }; + low.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Clears an idle render ring before a context of another VM is given a view of it, so the + /// new owner cannot read the entries earlier owners' commands left. The firmware has + /// consumed every entry; the host's position in the ring is kept. + pub(crate) fn clear_render_ring(&mut self) -> Result { + let Backing::Render(ring) = &mut self.backing else { + return Err(EINVAL); + }; + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + Ok(()) + } + /// Reuse a permanent client view only when every page names this ring. pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e2836df906ea16..fa2ddbdfd72dff 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -508,6 +508,9 @@ struct Previous> { binding: Option, context: C, publication: Option<(u64, u64)>, + /// The queue moved to another VM. The previous owner's views were unmapped at the move, + /// so a rollback leaves the queue detached instead of restoring them. + moved: bool, } impl Previous { /// The exact drain witness covers the replacement publication even when @@ -1698,21 +1701,49 @@ impl Queue { binding: None, context: old.ok_or(EIO)?, publication: None, + moved: false, }); } else { - // Another VM: the pool must be fresh backing (EPERM otherwise). - let kick_alias = self.kick.map_client(context.vm())?; - let pool_alias = self.pool.map_client(context.vm())?; - self.graph.clear_client_state(); - let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + // Another VM: the pool must be fresh backing (EPERM otherwise). The queue's state + // pages belong to one VM at a time. The previous owner's views (kick ring, + // preemption, usage and operand state, descriptors, queue context) are unmapped + // first, so its GPU work can no longer read or write what the new owner's work + // leaves there; then the pages are cleared and mapped into the new VM. The queue + // is idle (`reusable`) and the previous owner's pool release was consumed (fresh + // backing), so the firmware no longer uses those views. The previous context, + // whose scheduler state the queue may still run until the new owner's first + // command, stays in `Previous`. + if !self.pool.serves(context.vm()) { + return Err(EPERM); + } + drop(self.binding.take()); + let moved = (|| -> Result { + self.graph.clear_client_state(); + self.kick.clear_ring()?; + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + self.graph.bind(context.clone(), kick_alias, pool_alias) + })(); + let binding = match moved { + Ok(binding) => binding, + Err(error) => { + // Left detached: the record still names the previous owner's pages. + if let Some(old) = old.as_ref() { + self.retained_owner = Some(old.retain_firmware_pages()); + } + self.needs_fresh_backing = true; + return Err(error); + } + }; self.graph.set_owner(&context)?; - let old_binding = self.binding.replace(binding); + self.binding = Some(binding); // A detached queue (exited owner already dropped) has nothing to supersede; // the new owner's record fields replace the retained pages' references. self.previous = old.map(|old| Previous { - binding: old_binding, + binding: None, context: old, publication: None, + moved: true, }); self.retained_owner = None; } @@ -1750,7 +1781,13 @@ impl Queue { return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; - if let Some(binding) = previous.binding.take() { + if previous.moved { + // The record names the previous owner again, whose views went at the + // move: the queue is left detached, as for an exited owner. + self.binding = None; + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { self.binding = Some(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; @@ -3042,6 +3079,7 @@ mod fault_ownership_tests { binding: Some(Lease::new(&used)), context: Lease::new(&used), publication, + moved: false, }); assert_eq!(used.load(Ordering::Relaxed), 2); Previous::retire_witnessed(&mut previous); @@ -3188,7 +3226,12 @@ impl Queue { self.graph.set_owner(&previous.context)?; } if let Some(mut previous) = self.previous.take() { - if let Some(binding) = previous.binding.take() { + if previous.moved { + // As in `release_owner`: left detached, the new owner's views returned. + retired = self.binding.take(); + self.retained_owner = Some(previous.context.retain_firmware_pages()); + self.needs_fresh_backing = true; + } else if let Some(binding) = previous.binding.take() { retired = self.binding.replace(binding); } else { self.binding.as_mut().ok_or(EIO)?.context = previous.context; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 73186d9077303f..83a4bcb2f3022c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -38,6 +38,8 @@ pub(crate) struct ParkedPair { /// owner) keeps it, so its next owner change still waits while that /// scheduler is released. executed: u64, + /// The VM of the context that last ran on these rings. + vm: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -84,6 +86,7 @@ impl Pair { /// under the same device mutex. Moving these owners performs no firmware operation. pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { let context = self.memory.context.clone(); + let vm = context.vm().id(); let scheduler = context.scheduler_owner(); let buffer_id = self.buffer_id(); let parked = ParkedPair { @@ -103,6 +106,7 @@ impl Pair { outer_started: self.outer_started, previous: self.previous, executed: self.executed, + vm, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -118,6 +122,20 @@ impl Pair { } impl ParkedPair { + /// Clears both kick rings when the pair goes to a context of another VM than the one + /// that last ran on it, so the new owner cannot read the entries the previous owner's + /// commands left. The pair is parked, so the firmware consumed every entry. + pub(crate) fn clear_for_vm(&mut self, vm: u64) -> Result { + if self.vm == vm { + return Ok(()); + } + for queue in self.kicks.as_mut().ok_or(EBUSY)?.iter_mut() { + queue.clear_render_ring()?; + } + self.vm = vm; + Ok(()) + } + /// The release of `scheduler` has been consumed: a later owner change no /// longer has to wait for it. pub(crate) fn release_consumed(&mut self, scheduler: u64) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 1704139a9f97f1..ff0c4904f80f99 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -537,6 +537,7 @@ impl super::super::Backend { first, } = *ctx; let built = (|| -> Result<_> { + parked.clear_for_vm(self.context.vm().id())?; let free_list = self.context.render_pool( alloc, &resources.ids, From 693a80ad792ad7da80b01b601c1206d05c874f33 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:06 -0800 Subject: [PATCH 150/171] drm/asahi: g17: Fix a released context's QoS entry pointing at memory another process reuses After a GPU context was released, the firmware's QoS table kept naming its scheduler page for that QoS ID until the ID's next owner published work. On M5 scheduler pages are pooled, so the page could meanwhile be zeroed and given to another process's context, and a firmware read of that row would see the other process's scheduler state. A released ID's scheduler and share rows are now cleared before a new context is created, and at each event turn. A released ID is recorded when its lease drops and forgotten again if a new context takes it, since that owner writes its own row at its first publication. Rows of IDs with work still in flight are left alone. Signed-off-by: Jeff Queisser (cherry picked from commit 33a290369dca55a2c661f53f06f9dd6dbf1e8257 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/context.rs | 24 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/port.rs | 10 ++++++++ drivers/gpu/drm/asahi/g17/qos.rs | 20 ++++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 +++ .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 3 +++ 5 files changed, 58 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 3147a726967656..ff0e12f3dcdacc 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -32,11 +32,18 @@ use kernel::{ pub(crate) struct QosIds { #[pin] used: Mutex, + /// IDs released since the firmware's QoS table last had their rows cleared + /// (`take_released`). Locked after `used`. + #[pin] + released: Mutex, } impl QosIds { pub(crate) fn new() -> Result> { Arc::pin_init( - pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + pin_init!(Self { + used <- new_mutex!(3, "G17 QoS IDs"), + released <- new_mutex!(0, "G17 released QoS IDs"), + }), GFP_KERNEL, ) } @@ -47,6 +54,8 @@ impl QosIds { return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; + // A new owner writes its own scheduler row at its first publication. + *self.released.lock() &= !(1u128 << id); Ok(QosLease { ids: self.clone(), id: id as u8, @@ -59,7 +68,9 @@ struct QosLease { } impl Drop for QosLease { fn drop(&mut self) { - *self.ids.used.lock() &= !(1u128 << self.id); + let mut used = self.ids.used.lock(); + *used &= !(1u128 << self.id); + *self.ids.released.lock() |= 1u128 << self.id; } } @@ -690,6 +701,15 @@ impl QosIds { pub(crate) fn used(&self) -> u32 { (*self.used.lock() & !3u128).count_ones() } + /// The IDs released, and not taken again, since the last call: their rows in the + /// firmware's QoS table still name the released context's scheduler page. + pub(crate) fn take_released(&self) -> u128 { + let used = self.used.lock(); + let mut released = self.released.lock(); + let ids = *released & !*used; + *released = 0; + ids + } } /// Memory a context's work list may lead the firmware to stays allocated: a linked node (or diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 079c93d063d447..eab31f896e2b5d 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -451,6 +451,16 @@ impl Port { self.qos.lock().cancel(&view, publication) } + /// Clears the QoS table rows of released QoS IDs (`QosIds::take_released`). + pub(super) fn qos_forget(&self, ids: u128) -> Result { + if ids == 0 { + return Ok(()); + } + let view = self.qos_view()?; + self.qos.lock().forget(&view, ids); + Ok(()) + } + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { let view = self.qos_view()?; self.qos.lock().complete(&view, owners)?; diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index b36656b0174a7b..239ae56e01bb11 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -237,6 +237,26 @@ impl Accounting { self.complete(view, &[ticket.owner]) } + /// Clears the scheduler and share rows of the QoS IDs in `ids`, which no context holds: a + /// released context's row otherwise keeps naming its scheduler page, which the pool may + /// already have given to another process's context. Rows of IDs with work in flight are + /// left alone. + pub(crate) fn forget(&self, view: &View<'_>, ids: u128) { + for qos in 0..fw::QUEUES.min(128) { + if ids & (1u128 << qos) == 0 { + continue; + } + if self.in_flight.iter().zip(view.rows.iter()) + .any(|(&count, row)| count != 0 && usize::from(row[0].load(Ordering::Relaxed)) == qos) + { + continue; + } + view.scheduler[qos].store(0, Ordering::Relaxed); + view.share[qos].store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + /// Preflights the entire set before clearing any final row. QID membership, /// scheduler mappings and monotonically increasing counts remain intact. pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 10557c655194cd..65a2a0f9c8705d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -782,6 +782,9 @@ impl Backend { if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); } + // The new context may take a pooled scheduler page a released context's QoS row + // still names: clear those rows first. + let _ = firmware.port.qos_forget(firmware.queues.qos_ids.take_released()); match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 5b9044b9638b09..6cc53f647a5a16 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -143,6 +143,9 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + // Released contexts' QoS rows stop naming their scheduler pages at the next event + // turn, and at the latest before a new context is created. + let _ = self.port.qos_forget(self.queues.qos_ids.take_released()); let recovery_closed = self.init.recovery_state()? == 0; let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { From 4116a70a561b007b62ea374287bc97b7bebad97f Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:09 -0800 Subject: [PATCH 151/171] drm/asahi: Fix client binds silently replacing the kernel's GPU mappings When a user bind and a kernel mapping claimed the same GPU addresses in one VM, the page-table code logged the collision and overwrote the existing entries, so the GPU silently used the newer mapping's pages in place of the older one's, such as client memory in place of state the firmware reads, and the later unmap of either cleared the survivor's pages. On the A18 Pro a client could reach this by binding where the kernel later places that client's queue aliases. A map operation now fails with EBUSY and leaves the existing mapping untouched, and the A18 Pro arenas are reserved against binds from VM creation. - The structural fix: no mapping path maps over a live entry on purpose (user binds run after the VM manager removed every overlapping mapping, kernel mappings take fresh ranges, a residency restore clears its own range first), so a live entry is always another mapping's. Each map operation checks its whole range before its first store, because its failure path clears the whole range it asked for; the store loop refuses a live entry too, as a backstop. - The A18 Pro instance: the kick and descriptor rings, render generation state and client-private compute and render backing are placed in two arenas when a queue is installed, after the VM exists, and were registered as driver mappings only then. A bind made earlier was not refused, nor one checked at parse time and committed after an alias was installed. With both arenas reserved at creation, the reserved set no longer changes. Mesa already keeps this range out of its allocator on the A18 Pro, so no Mesa bind is refused. Signed-off-by: Jeff Queisser (cherry picked from commit 44bf21e070ff5cc983a25c9db85d7b27f8cf8665 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/hw/t8140.rs | 9 ++++++++- drivers/gpu/drm/asahi/mmu.rs | 18 ++++++++++++++++++ drivers/gpu/drm/asahi/pgtable.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 15de4fd988b0c7..22c96bac5ca51b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -318,7 +318,7 @@ pub(crate) mod resources { ]; pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; - pub(crate) const RESERVED_RANGES: usize = 48; + pub(crate) const RESERVED_RANGES: usize = 50; /// Compact allocations search these shared client arenas in this order. pub(crate) fn compact_ranges() -> [Range; 4] { @@ -365,6 +365,13 @@ pub(crate) mod resources { } add(AUXILIARY); add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + // The dynamic arenas the kernel places a client's queue aliases in (kick and + // descriptor rings, render generation state, client-private compute and render + // backing) stay reserved from creation too, as the fixed windows above are: an alias + // created later is then never mapped over a user bind, and the bind check's view of the + // reserved ranges cannot change between parsing a bind and committing it. + add(super::dynamic::LOWER); + add(super::dynamic::CLIENT_LOWER); Some(ranges) } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 6be617824f3a8f..5d76e9d8987960 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -612,6 +612,11 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + // A live leaf in the range belongs to another mapping, such as a kernel alias in a + // driver window: refuse before writing anything, so the rollback below never clears + // that mapping's leaves. + self.page_table + .ensure_unmapped(map_addr..map_addr + map_length as u64)?; if is_sparse_tile(op.flags()) { // Kernel tile fill: the GEM only anchors the GPUVA. let result = if one_page { @@ -1002,6 +1007,12 @@ impl VmInner { self.page_table.prepare_map(node.start()..end)?; let uat_inner = self.uat_inner.clone(); let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; + // Refuse a range holding another mapping's leaves before writing anything: the + // discard below clears the whole range. + if let Err(error) = self.page_table.ensure_unmapped(node.start()..end) { + drop(handoff); + return Err(error); + } let result = self.map_node_prepared(node, prot); let discard = if result.is_err() { // Rollback still changes live leaves and must remain under the PPL lease. @@ -2896,6 +2907,13 @@ impl Vm { let transaction_uat = inner.uat_inner.clone(); let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); + // As for kernel mappings: refuse another mapping's leaves before the discard below + // could clear them. + if let Err(error) = inner.page_table.ensure_unmapped(iova..end) { + drop(mutation); + drop(handoff); + return Err(error); + } let ret = inner .page_table .map_pages(iova..end, phys as PhysicalAddr, prot, false); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index a996a47ffe8735..847ed49faccc43 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -986,11 +986,15 @@ impl UatPageTable { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); if ptev != 0 { + // Never replace a live leaf: it belongs to another mapping. The callers + // check their whole range first (`ensure_unmapped`), so this is reached + // only by a collision they missed. pr_err!( "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", iova + (idx * UAT_PGSZ) as u64, ptev ); + return Err(EBUSY); } pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); if !one_page { @@ -1001,6 +1005,28 @@ impl UatPageTable { }) } + /// Refuses with `EBUSY`, writing nothing, when any leaf of `iova_range` is live. + /// + /// Every PTE store writes either zero or a complete entry, so a nonzero leaf belongs to + /// another mapping. A map operation checks its whole range with this before its first + /// store: its failure path clears the whole range it asked for, which would otherwise + /// also clear the other mapping's leaves. + pub(crate) fn ensure_unmapped(&mut self, iova_range: Range) -> Result { + self.walk(iova_range, false, false, false, |iova, ptes| { + match ptes.iter().position(|pte| pte.load(Ordering::Relaxed) != 0) { + Some(idx) => { + pr_err!( + "UATPageTable::ensure_unmapped: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptes[idx].load(Ordering::Relaxed) + ); + Err(EBUSY) + } + None => Ok(()), + } + }) + } + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { mod_pr_debug!( "UATPageTable::reprot_pages: {:#x?} {:?}\n", From c8bf2ecc40b8c100f42a99e1dc4f234b0ab4db2a Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:10 -0800 Subject: [PATCH 152/171] drm/asahi: g17: M5: Fix compute jobs overrunning their scratch pool On M5, a compute dispatch with shader-context scratch passes the hardware an auxiliary allocation word, the per-core page count of its scratch storage, unchanged from userspace. The pool behind it was sized for a fixed eight pages per core whatever the word said, so a dispatch asking for more (Mesa's ray-query dispatches ask for 14) let the hardware use storage past what its pool holds. Any value was accepted, also without scratch. The pool now covers the requested count, and a nonzero word without scratch is refused with EINVAL. Render already sizes its pool from the same word: its spill budget adds the published auxiliary page count, which includes the eight-page tail, for every core. Compute now does the same, with the eight-page tail as the floor, so no dispatch gets a smaller pool than before. Mesa's ray-query compute pools grow by six pages per core. Signed-off-by: Jeff Queisser (cherry picked from commit 2b1ebc6f6f8f6b5ea161e9db1ee9b8e88a71359d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/command.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs | 12 +++++++----- drivers/gpu/drm/asahi/g17/render_resources.rs | 7 ++++--- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 +++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 3583d89eb2f89c..b7e3b9555583b6 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -649,6 +649,12 @@ impl Payload { if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { return Err(refuse_ray_traversal()); } + // The auxiliary allocation is the per-USC page count of the dispatch's SCS + // storage, which the pool is sized for (`plan_compute_scs`). Without SCS there + // is no such storage, as render's programs require. + if !m5_scratch.enabled() && cmd.cdm_auxiliary_allocation != 0 { + return Err(EINVAL); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs index 3e5584422364cf..ce0b5dc6129a4c 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/pool_budget.rs @@ -72,15 +72,17 @@ impl RenderBudget { Ok(Self { active_cores, constant_pages, active_kicks }) } - /// Compute SCS storage is private to each enabled USC. The eight-page - /// tail belongs to the compute allocator, in addition to its small/large - /// SCS page inventories and the execution-state spill reservation. + /// Compute SCS storage is private to each enabled USC. The auxiliary + /// allocation the dispatch publishes, per USC and at least the eight-page + /// tail, belongs to the compute allocator, in addition to its small/large + /// SCS page inventories and the execution-state spill reservation. As for + /// render, the pool covers what the descriptor tells the hardware. pub(crate) fn compute_scs_requirements(&self, mut request: Requirements, - pages: u16) -> Result + pages: u16, auxiliary_pages: u32) -> Result { if pages != 0 { let small = u64::from(pages); - let per_core = add(add(small, small.div_ceil(256))?, 8)?; + let per_core = add(add(small, small.div_ceil(256))?, u64::from(auxiliary_pages.max(8)))?; let bytes = mul(mul(per_core, u64::from(self.active_cores))?, 4096)?; request.minimum_bytes = add(request.minimum_bytes, bytes)?; request.ideal_bytes = add(request.ideal_bytes, bytes)?; diff --git a/drivers/gpu/drm/asahi/g17/render_resources.rs b/drivers/gpu/drm/asahi/g17/render_resources.rs index cb4dc51b65a9a9..fb9f6a2bfd5042 100644 --- a/drivers/gpu/drm/asahi/g17/render_resources.rs +++ b/drivers/gpu/drm/asahi/g17/render_resources.rs @@ -239,11 +239,11 @@ impl M5Policy { } pub(crate) fn plan_compute(&self, programs: &ComputePrograms, priority: u8) -> Result { - self.plan_compute_scs(programs, priority, 0) + self.plan_compute_scs(programs, priority, 0, 0) } pub(crate) fn plan_compute_scs(&self, programs: &ComputePrograms, priority: u8, - scs_layout: u64) -> Result { + scs_layout: u64, auxiliary_pages: u32) -> Result { let scratch = super::fw::m5::compute::ScratchRequest::new(scs_layout)?; // UMA pool class 1 is realtime; class 2 covers all other // public priorities. Its resident-kick count also depends on engine. @@ -264,7 +264,8 @@ impl M5Policy { }, 96, 114688).map_err(|_| EINVAL)?; } let request = budget.compute_scs_requirements( - demand.requirements().map_err(|_| EINVAL)?, scratch.pages()).map_err(|_| EINVAL)?; + demand.requirements().map_err(|_| EINVAL)?, scratch.pages(), auxiliary_pages) + .map_err(|_| EINVAL)?; let sizes = budget.sizes(request, History::default()).map_err(|_| EINVAL)?; Ok(Plan { geometry: sizes.geometry().map_err(|_| EINVAL)?, sizes, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index c32bd5012d5661..ca79208c8dc396 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -191,9 +191,10 @@ impl job::Backend for Backend { let programs = resources.as_deref().ok_or(EINVAL)?; *plan = Some(policy.plan(programs, priority)?); } - Validated::Compute { resources, plan, m5_scratch, .. } => { + Validated::Compute { resources, plan, m5_scratch, cdm_auxiliary_allocation, .. } => { if let Some(programs) = resources { - *plan = Some(policy.plan_compute_scs(programs, priority, m5_scratch.layout())?); + *plan = Some(policy.plan_compute_scs(programs, priority, m5_scratch.layout(), + *cdm_auxiliary_allocation)?); } else if m5_scratch.enabled() { // SCS needs the dispatch storage declarations to size its pool. return Err(EINVAL); From d7f31a426331b1e05e989ef908f1badb0a3165fb Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:12 -0800 Subject: [PATCH 153/171] drm/asahi: g17: M5: Fix one client exhausting system memory through GPU pool sizes On M5, the render and compute pools behind a client's queues are sized from that client's own program records, up to the board's whole pool inventory (16 GiB on the M5 Ultra), and allocated and zeroed at once as kernel memory. Kernel memory counts toward the device total but is never refused and is attributed to no process, so one unprivileged client could pin many gigabytes per queue: other clients' allocations then failed with ENOMEM, and the OOM killer could not see the culprit. An M5 pool backing is now admitted against its client's and the device's memory limits before it is allocated, as a user buffer of the same size would be, and a refusal fails the submission with ENOMEM. A client VM now carries the memory accounting of the file that created it, so every place that builds an M5 pool for a context's VM charges that client: compute pools at install, re-arm and growth, and render pools when their backing is built. The charge is released with the backing. Kernel VMs and the other GPUs' pools are unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit 16c04001c3236da56199c6e7e110dca068b20ac1 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/file.rs | 4 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 32 ++++++++++++++++------ drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 ++-- drivers/gpu/drm/asahi/g17/runtime.rs | 9 ++++-- drivers/gpu/drm/asahi/mmu.rs | 15 ++++++++++ 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index ccae606a84a075..0a76f7d989aeaa 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -646,7 +646,9 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + let vm = gpu + .new_vm_for_file(kernel_range.clone(), file_id)? + .with_memory_client(file.inner().mem.clone()); // G16 registers a retained backend owner before returning this VM. Any later // allocation can fail before file::Vm exists to run its normal close path. let failed_vm = ScopeGuard::new_with_data( diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index ab5f9e77a0f458..58780c95ddd4e8 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -329,6 +329,8 @@ fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { pub(crate) struct Pages { backing: KernelObject, layout: PageLayout, + /// The admission of an M5 backing sized by a client's request, released with it. + _charge: Option, } impl Drop for Pages { fn drop(&mut self) { @@ -354,14 +356,17 @@ impl Pages { Ok(Self { backing, layout: PageLayout::Neo { blocks, runs }, + _charge: None, }) } /// Fresh compute backing at the admitted capacity, built off-lock, in the /// capacity's family format. - pub(crate) fn for_compute(alloc: &Allocator<'_>, capacity: ComputeCapacity) -> Result { + /// An M5 backing is charged to `client` when one is given (see [`Self::m5`]). + pub(crate) fn for_compute(alloc: &Allocator<'_>, capacity: ComputeCapacity, + client: Option<&Arc>) -> Result { match capacity { ComputeCapacity::Blocks(blocks) => Self::compute_sized(alloc, blocks), - ComputeCapacity::M5(geometry) => Self::compute_m5(alloc, geometry), + ComputeCapacity::M5(geometry) => Self::compute_m5(alloc, geometry, client), } } fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { @@ -373,19 +378,27 @@ impl Pages { /// Fresh zeroed M5 compute backing, built without the device mutex. M5 keeps /// its directory, block list and initial eight render blocks in one retained /// allocation. - fn compute_m5(alloc: &Allocator<'_>, required: Option) -> Result { + fn compute_m5(alloc: &Allocator<'_>, required: Option, + client: Option<&Arc>) -> Result { use super::fw::m5::pool; let geometry = match required { Some(geometry) => geometry, None => pool::PoolGeometry::g17s(8).map_err(|_| EINVAL)?, }; - Self::m5(alloc, geometry) + Self::m5(alloc, geometry, client) } - fn m5(alloc: &Allocator<'_>, geometry: super::fw::m5::pool::PoolGeometry) -> Result { + /// M5 pool sizes follow the client's program records, up to the board's whole inventory, + /// so the backing is admitted against `client`'s and the device's limits before it is + /// allocated, as a user object of that size would be. A refusal fails with ENOMEM. + fn m5(alloc: &Allocator<'_>, geometry: super::fw::m5::pool::PoolGeometry, + client: Option<&Arc>) -> Result { let tables = geometry.directory_bytes.checked_add(geometry.freelist_bytes).ok_or(EOVERFLOW)?; let size = geometry.blocks.checked_mul(geometry.block_bytes()) .and_then(|n| n.checked_add(tables)).ok_or(EOVERFLOW)?; + let charge = client + .map(|client| crate::memacct::LEDGER.admit(client, size as u64)) + .transpose()?; let backing = alloc.lower(size, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, CpuMap::WriteCombined)?; - let mut pages = Self { backing, layout: PageLayout::M5(geometry) }; + let mut pages = Self { backing, layout: PageLayout::M5(geometry), _charge: charge }; pages.populate_m5()?; Ok(pages) } @@ -522,7 +535,7 @@ impl RenderBacking { /// the canonical and client mappings of these pages. fn m5(alloc: &Allocator<'_>, client: &mmu::Vm, geometry: super::fw::m5::pool::PoolGeometry) -> Result { - let pages = Pages::m5(alloc, geometry)?; + let pages = Pages::m5(alloc, geometry, client.memory_client())?; let client = pages.map_client(client)?; Ok(Self { _client: client, _global: None, pages }) } @@ -1009,11 +1022,12 @@ impl ComputePool { } /// A pool at the admitted capacity, in the capacity's family format. - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, required: ComputeCapacity) -> Result { + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, required: ComputeCapacity, + client: Option<&Arc>) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } - let pages = Pages::for_compute(alloc, required)?; + let pages = Pages::for_compute(alloc, required, client)?; let m5 = pages.is_m5(); Self::with_pages(alloc, qid, m5, pages) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index fa2ddbdfd72dff..97ae8ddefdea49 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -544,8 +544,9 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, required: ComputeCapacity) -> Result { - let pool = ComputePool::new(alloc, id.qid(), required)?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, required: ComputeCapacity, + client: Option<&Arc>) -> Result { + let pool = ComputePool::new(alloc, id.qid(), required, client)?; Ok(Self { pool, id }) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 65a2a0f9c8705d..b9232e9e3f6ba6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1031,7 +1031,8 @@ impl Backend { let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; - let mut pages = Some(super::freelist::Pages::for_compute(&alloc, required)?); + let mut pages = Some(super::freelist::Pages::for_compute(&alloc, required, + self.context.vm().memory_client())?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; let mut growth = None; @@ -1113,7 +1114,8 @@ impl Backend { admission: &Admission<'_>, ) -> Result> { let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::for_compute(alloc, required)?; + let pages = crate::g17::freelist::Pages::for_compute(alloc, required, + self.context.vm().memory_client())?; Ok((pages, compute::DescriptorRing::new(alloc, objects)?)) })(); let (pages, ring) = match built { @@ -1241,7 +1243,8 @@ impl Backend { } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required, + self.context.vm().memory_client())?; { let mut state = self.shared.state.lock(); admission.check()?; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 5d76e9d8987960..f16a4a680fc830 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1129,6 +1129,9 @@ pub(crate) struct Vm { context_bindings: Option>, shared_bindings: Option>, status: Option>, + /// The memory accounting of the client that created this VM, which kernel memory sized + /// by that client's requests is charged to. `None` for kernel VMs. + memory: Option>, gpuvm: ARef>, dummy_obj: ARef, binding: Arc>, @@ -2332,6 +2335,7 @@ impl Vm { context_bindings: None, shared_bindings: None, status: None, + memory: None, dummy_obj: dummy_obj.gem.clone(), binding, }) @@ -2420,6 +2424,17 @@ impl Vm { } /// Status shared by every queue using this VM, if enabled by its GPU. + /// Charges kernel memory sized by this VM's client to `client` ([`Self::memory_client`]). + pub(crate) fn with_memory_client(mut self, client: Arc) -> Self { + self.memory = Some(client); + self + } + + /// The memory accounting of the client that created this VM, if it is a client VM. + pub(crate) fn memory_client(&self) -> Option<&Arc> { + self.memory.as_ref() + } + pub(crate) fn status(&self) -> Option<&Arc> { self.status.as_ref() } From daee276c2b85dbbcdf167afd25223ae7bb79f9a6 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:15 -0800 Subject: [PATCH 154/171] drm/asahi: Fix one client taking every GPU queue from the others On GPUs with independent queues, QUEUE_CREATE had no per-client limit, so one unprivileged client could open queues until the device's execution contexts were gone. Every other process, the compositor included, then failed to create a queue with EBUSY: a GPU denial of service for the whole seat. A client now holds at most half of the device's independent queues, and its next QUEUE_CREATE fails with EBUSY. The limit comes from the same queue limit GET_PARAMS reports, so it follows each GPU's budget (29 of 58 where all execution contexts are available), and leaves the other half to the rest of the seat. The file's queue table hands out the lowest free ID, so bounding the IDs bounds the queues the file holds. GPUs without independent queues are unchanged. Signed-off-by: Jeff Queisser (cherry picked from commit b1994bb3dbc8865836d0581957fd63f0fee35b2d in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/file.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 0a76f7d989aeaa..8c2ee2de966559 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1426,8 +1426,16 @@ impl File { return Err(EACCES); } + // One client holds at most half of the device's independent queues, so it cannot + // take every execution context and leave other clients, the compositor included, + // unable to create one. The table hands out the lowest free ID, so bounding the IDs + // bounds the queues the file holds; a full table refuses with EBUSY. + let per_client = reg_data + .gpu + .queue_limits() + .map_or(u32::MAX, |limits| (limits.max_queues / 2).max(1)); let queues_xa = file.inner().queues(); - let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let resv = queues_xa.lock().reserve_limit(1..=per_client, GFP_KERNEL)?; let vms_xa = file.inner().vms(); let guard = vms_xa.lock(); let file_vm = guard.get(data.vm_id.try_into()?).ok_or(ENOENT)?; From 84b92fcef64c6c629515e360da2484ca2ddfe192 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:17 -0800 Subject: [PATCH 155/171] drm/asahi: mmu: Fix sparse tiles wasting 2 MiB of contiguous memory per GPU VM Each sparse tile was a 64-page physically contiguous allocation, 1 MiB with 16 KiB pages, of which the page tables read only the first UAT page. Every VM that did a sparse-tile bind allocated two, so any client could pin 2 MiB of high-order memory per VM it created, exhausting or fragmenting contiguous memory for the rest of the system. A tile is now one UAT page. The allocation's order is derived from the UAT page size, so the tile stays naturally aligned to a UAT page whatever the kernel page size. Signed-off-by: Jeff Queisser (cherry picked from commit 3f1090a0a72032fa101ee066d475b5369d0dce62 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/mmu/sparse_tile.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/sparse_tile.rs b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs index 55b3807e9af9f1..dc4a5459eff108 100644 --- a/drivers/gpu/drm/asahi/mmu/sparse_tile.rs +++ b/drivers/gpu/drm/asahi/mmu/sparse_tile.rs @@ -19,8 +19,9 @@ unsafe impl Send for SparseTile {} unsafe impl Sync for SparseTile {} impl SparseTile { - /// 64 pages. The page tables read only the first. - const ORDER: u32 = 6; + /// One UAT page, the only part the page tables read: the order of the kernel pages that + /// make up a UAT page, naturally aligned to it. + const ORDER: u32 = (crate::pgtable::UAT_PGSZ / kernel::page::PAGE_SIZE).trailing_zeros(); pub(super) fn new() -> Result> { // SAFETY: A plain zeroed higher-order page allocation. From 23ea331d2c398a2f06525356e4015e3948f41442 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:18 -0800 Subject: [PATCH 156/171] drm/asahi: pgtable: Fix a bad firmware table pointer making the driver overwrite kernel memory On GPUs whose page tables the firmware builds in reserved memory (the T6040, T6041 and T6050), a page-table walk followed any table descriptor the firmware or bootloader had written, and accepted a child table anywhere in System RAM. A corrupted or unexpected descriptor below the shared kernel root would then have the walk store PTEs into an arbitrary kernel page. A table page must now be in the reserved regions or be one the driver allocated; anything else fails the walk with EIO. Freeing already refuses tables the driver does not own, so only the write path was exposed. The firmware's own tables on these boards are reserved no-map regions, and every table the driver adds is recorded when it is allocated, so no legitimate walk is refused. Signed-off-by: Jeff Queisser (cherry picked from commit 5c869c10b83c52065bb8dd272430a15d58c8dea3 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/pgtable.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 847ed49faccc43..6ea0f1ee171c64 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -669,6 +669,7 @@ impl UatPageTable { mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; + let firmware_tree = reserved.as_deref(); let entry = |p: *mut u8| -> Result> { let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is @@ -706,7 +707,21 @@ impl UatPageTable { } } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + let child = upte_val & oas_mask & (!UAT_PGMSK as u64); + // Below a root the driver did not allocate, the descriptors were + // written by the firmware or the bootloader. Follow one only into a + // reserved table page or a table this driver allocated, never into + // other RAM, whose PTEs this walk would then write. + if let (Some(firmware_tree), Some(tables)) = + (firmware_tree, driver_tables.as_ref()) + { + if !firmware_tree.contains(child) && !tables.contains(&child) { + pr_err!("UATPageTable::with_pages: table PTE {:#x} at iova {:#x} level {} names a page outside the reserved and driver tables\n", + upte_val, iova, level + 1); + return Err(EIO); + } + } + Ok(Some(child)) } else if upte_val == 0 || (!alloc && !free) { mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); Ok(None) From a2f2b85da3c175cf597ce511affc62923cecf4a3 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 15:25:30 -0800 Subject: [PATCH 157/171] drm/asahi: g17: M5: Fix GPU jobs being able to write the kernel's completion and scheduler pages On M5, the render completion stamps, each context's scheduler and work-state pages, and each compute queue's support page were mapped for both the firmware and the GPU. The kernel trusts what they hold. They sit in the kernel's upper half, which every execution context includes. Whether user shaders could reach that half was never established. This removes the GPU permission either way: on M5 these pages are now firmware-only. On the Studio, every GPU path ran clean with them firmware-only, so the GPU itself doesn't need them. Scope, by page set: - Render completion stamps (initdata/m5.rs): allocated only on M5. - Scheduler pages (context.rs): M5 takes them from the retained pool, Neo allocates its own. Firmware-only on M5 only; Neo is unchanged. - Work-state pages (context.rs): shared code, allocated on Neo too. Firmware-only on M5 only; Neo is unchanged. - Compute support page (queue/compute.rs): Neo allocates a larger one for its own use. Firmware-only on M5 only; Neo is unchanged. set_fw_cached_prefix now derives the cached prefix from the mapping's own protection, changing only the cache attribute, instead of rewriting it to the firmware-and-GPU protection. Its readback already refused any change other than the cache attribute. So a firmware-only pooled object used to fail queue creation with EFAULT, and every source that succeeded is one for which the derived protection is the same as before. That covers every pooled family on Neo and M5 today, all firmware-and-GPU shared, so the change is identity there. Signed-off-by: Jeff Queisser (cherry picked from commit ff0d367316b29f631940187420c3962dd66939ed in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/context.rs | 11 +++++++++-- drivers/gpu/drm/asahi/g17/initdata/m5.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +++- drivers/gpu/drm/asahi/mmu.rs | 3 ++- drivers/gpu/drm/asahi/pgtable.rs | 4 ++++ 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index ff0e12f3dcdacc..e9a2051fcba45f 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -215,13 +215,20 @@ impl Context { ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; + // On M5 only the firmware reads and writes the scheduler and work-state pages; the GPU + // has no access to them. + let kernel_prot = if alloc.uat.is_m5() { + mmu::PROT_FW_SHARED_RW + } else { + mmu::PROT_GPU_FW_SHARED_RW + }; let mut scheduler = if alloc.uat.is_m5() { SchedulerStorage::Retained(pool.allocate( alloc, Family::Scheduler, UAT_PGSZ, UAT_PGSZ, - mmu::PROT_GPU_FW_SHARED_RW, + kernel_prot, CpuMap::WriteCombined, )?) } else { @@ -244,7 +251,7 @@ impl Context { Family::WorkState, 2 * UAT_PGSZ, UAT_PGSZ, - mmu::PROT_GPU_FW_SHARED_RW, + kernel_prot, CpuMap::WriteCombined, )?; let work_base = object.gpu_va(); diff --git a/drivers/gpu/drm/asahi/g17/initdata/m5.rs b/drivers/gpu/drm/asahi/g17/initdata/m5.rs index 9e76ca5a4f8f94..ec150b54f1bee4 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/m5.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/m5.rs @@ -111,7 +111,7 @@ impl InitData { for stage in 0..2 { self.stamps[stage] = self.add(KernelObject::new(dev, uat.kernel_vm(), Placement::In(uat.kernel_dynamic_range(), mmu::UAT_PGSZ as u64), - mmu::UAT_PGSZ, mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined)?)?; + mmu::UAT_PGSZ, mmu::PROT_FW_SHARED_RW, CpuMap::WriteCombined)?)?; } self.alias(self.pb, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[0], GPU)?; self.alias(self.page_pool, uat.kernel_lower_vm(), m::REGION_VIEW_LOW_ADDRS[1], GPU)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 97ae8ddefdea49..ec269866bf7c3d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -173,7 +173,9 @@ impl Graph { let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; - let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, prot, cpu)?; + // The M5 support page is firmware-only; the GPU has no access to it. + let support_prot = if m5 { mmu::PROT_FW_SHARED_RW } else { prot }; + let mut support = alloc.kernel(if m5 { mmu::UAT_PGSZ } else { 0xc000 }, page, support_prot, cpu)?; if !m5 { let support_va = support.gpu_va(); support.write(0, support_va + page)?; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f16a4a680fc830..d1c9119f6cd65c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1551,9 +1551,10 @@ impl KernelMapping { } let _mutation = owner.mapping_mutation(); mutated = true; + // The prefix keeps the mapping's own access and changes only its cache attribute. owner.page_table.reprot_pages( self.iova()..self.iova() + prefix as u64, - PROT_GPU_FW_PRIV_RW, + self.0.prot.as_cached(), )?; mem::sync(); mem::tlbi_all(); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 6ea0f1ee171c64..dfddfb1298afdb 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -417,6 +417,10 @@ impl Prot { pub(crate) const fn as_uncached(&self) -> Self { self.memattr(MEMATTR_UNCACHED) } + + pub(crate) const fn as_cached(&self) -> Self { + self.memattr(MEMATTR_CACHED) + } } impl Default for Prot { From 5517f34a85b5bbf15f8ffd81e1d1bb0321511399 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 14:39:20 -0800 Subject: [PATCH 158/171] drm/asahi: g17: Fix clients rewriting submitted GPU commands after the kernel validated them Every client VM had a read-write view of its queues' kick rings, the entries the kernel builds from validated state for each command: queue and descriptor addresses, dependencies, QoS slot, event masks and the register programs' addresses. A client's GPU work could therefore rewrite an entry after the kernel had built and checked it, before the firmware consumed it. A render pair's rings also stay mapped in every VM that ran on them. Client views of kick rings are now read-only. The host writes kick entries through the kernel's own mapping and the firmware reads them through its own; neither uses the client's view, and nothing in the driver reads an entry back. The client's view is there for the GPU's reads, as the descriptor rings' client views, already read-only, are. The kernel's lower and firmware mappings keep their permissions. Signed-off-by: Jeff Queisser (cherry picked from commit 9a9e352f065736115349b21ca909737594a8c25c in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/kick.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 860d07628ea410..6ff310656fb23f 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -398,6 +398,11 @@ enum Backing { Render(RenderRing), } +/// A client VM's view of a kick ring. The host writes the entries through the kernel's mapping +/// and the firmware reads them through its own; a client's GPU work only reads them, so its +/// view is read-only: a client cannot rewrite an entry after the kernel built and checked it. +const CLIENT_KICK_PROT: mmu::Prot = mmu::PROT_GPU_SHARED_RO; + /// A render ring's pages and the shared lower reservation clients alias. struct RenderStorage { // The lower alias drops before its backing. @@ -588,7 +593,7 @@ impl Queue { let alias = ring.storage .high - .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + .prepare_alias(client, address, CLIENT_KICK_PROT, false); // On failure the storage returns to the stock and a fresh ring is built. if let Ok(alias) = alias { return Ok(( @@ -620,7 +625,7 @@ impl Queue { .iova() .checked_add(KICK_RING_SIZE as u64) .ok_or(EOVERFLOW)?; - match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + match high.map_alias(client, low.iova(), CLIENT_KICK_PROT) { Ok(client) => break (low, client), Err(error) if error == ENOSPC && end < range.end => { drop(low); @@ -676,7 +681,7 @@ impl Queue { pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let address = self.low_va(); self.object() - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + .map_alias(vm, address, CLIENT_KICK_PROT) } /// Clears an idle compute ring before another VM is given a view of it, so the new owner @@ -709,7 +714,7 @@ impl Queue { Backing::Compute { low, .. } => low, Backing::Render(ring) => &ring.storage.high, }; - object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + object.prepare_alias(vm, self.low_va(), CLIENT_KICK_PROT, false) } /// Both aliases are owned mappings of the same full ring. Publish their From ea0521c5c69e0331f8914011337ee61d1f0a7955 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 18:37:34 -0800 Subject: [PATCH 159/171] drm/asahi: g17: Fix GPU job fences being able to stall memory reclaim under memory pressure Two kinds of G17 work signal job fences: the event work, which completes jobs, and each scheduler's timeout work, which fails timed-out ones. Memory reclaim may wait on those fences, but neither ran on a reclaim-safe workqueue. The event work ran on an M5 driver queue allocated without WQ_MEM_RECLAIM, or on system_highpri_wq; the timeout work ran on system_percpu_wq, drm_sched's default. When a pool has no idle worker and creating one needs memory, that work cannot run, and neither the fences nor reclaim make progress. Send all of it to one reclaim-safe queue, aurora_g17_events. On M5 it is unbound and high priority as before, so the event work still follows the submitting CPU; elsewhere it is per-CPU and high priority, the counterpart of system_highpri_wq, so the event work runs where it did. Every scheduler takes the queue as its timeout queue and keeps a reference until it has been finalized. Their run and free work already used reclaim-safe queues. For that, the Rust OwnedQueue gains reclaim-safe unbound and per-CPU constructors (restrict() refuses a per-CPU queue), and the Rust DRM scheduler binding's new_with_queues() takes an optional submission queue and an optional shared timeout queue, in place of new_with_submit_wq(). Signed-off-by: Jeff Queisser [Eduard Moldovan: the KUnit cases in aurora-silicon's rust_drm_sched test module are left out; integration/aurora has no such module] (cherry picked from commit 37770014197fbd1078b85390c3008686a09966d6 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/job.rs | 13 ++++-- drivers/gpu/drm/asahi/g17/mod.rs | 47 +++++++++++++------- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 ++- rust/helpers/workqueue.c | 4 +- rust/kernel/drm/sched.rs | 33 ++++++++++---- rust/kernel/workqueue.rs | 45 ++++++++++++++++--- 6 files changed, 111 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index da94d77aacaa9f..1a0742f00a4ce4 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -308,6 +308,9 @@ pub(crate) trait Backend: Send + Sync { fn supports_submission(&self) -> bool { true } /// Whether this backend's scheduler workers run in their queue creator's cluster (M5). fn creator_cluster_scheduling(&self) -> bool { false } + /// The reclaim-safe queue for the scheduler's timeout work, whose handler signals the + /// fences of timed-out jobs. + fn fence_queue(&self) -> Arc; fn supports_engine(&self, _engine: Engine) -> bool { true } /// Device policy applied to a validated command before it is ordered (the M5 resource /// plan). @@ -472,15 +475,19 @@ impl Scheduler { Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), }; let device = dev.as_ref().as_ref(); - let scheduler = if backend.creator_cluster_scheduling() { + let submit = if backend.creator_cluster_scheduling() { // The run and free work stays in the creating CPU's cluster, as the event work // follows the submitter, instead of wherever an unbound worker last ran. let queue = workqueue::OwnedQueue::new_ordered_reclaim(name)?; queue.restrict(Cpumask::cluster_of(CpuId::current()))?; - sched::Scheduler::new_with_submit_wq(device, 1, credits, 0, TIMEOUT_MS, name, queue)? + Some(queue) } else { - sched::Scheduler::new(device, 1, credits, 0, TIMEOUT_MS, name)? + // drm_sched allocates its own ordered reclaim-safe queue. + None }; + let scheduler = sched::Scheduler::new_with_queues( + device, 1, credits, 0, TIMEOUT_MS, name, submit, Some(backend.fence_queue()), + )?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; Ok(Self { entity, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 56752e1194db65..227a9b50beb9f0 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1076,10 +1076,15 @@ struct Shared { /// When work was last published while the governor is requested (0: none since its last /// step). Publishers without the device mutex set it too. governor_activity_ns: AtomicU64, - /// M5: the event work's queue. Unbound, so the event work runs in the affinity pod (the L2) - /// of the CPU that last published work, not on whichever CPU the completion IRQ thread first - /// ran on. On a two-die part that keeps completion processing on the submitter's die. - events: Option, + /// The queue of all work that signals job fences: the event work, and every scheduler's + /// timeout work. Memory reclaim may wait on those fences, so it is reclaim-safe. On M5 it is + /// unbound, so the event work runs in the affinity pod (the L2) of the CPU that last + /// published work, not on whichever CPU the completion IRQ thread first ran on; on a two-die + /// part that keeps completion processing on the submitter's die. Elsewhere it is per-CPU and + /// high priority, as `system_highpri_wq`. + events: Arc, + /// M5: the event work and the scheduler workers follow the submitting CPU. + follows_submitter: bool, /// M5: the CPU of the last publication, `u32::MAX` before the first. submitter: AtomicU32, #[pin] @@ -1252,17 +1257,25 @@ impl Shared { return; } // Already queued if this fails, which is just as good. - let _ = match (&self.events, self.submitter()) { - (Some(events), Some(cpu)) => { - events.enqueue_on::, EVENT_WORK>(cpu, self.clone()) - } - _ => workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()), + let _ = match self.submitter() { + Some(cpu) => self.events.enqueue_on::, EVENT_WORK>(cpu, self.clone()), + None => self.events.enqueue::, EVENT_WORK>(self.clone()), }; } + /// The queue for the schedulers' timeout work, which signals the fences of timed-out jobs. + pub(super) fn fence_queue(&self) -> Arc { + self.events.clone() + } + + /// Whether the event work and the scheduler workers follow the submitting CPU (M5). + pub(super) fn follows_submitter(&self) -> bool { + self.follows_submitter + } + /// Records the publishing CPU for the event work's placement (M5). fn note_submitter(&self) { - if self.events.is_some() { + if self.follows_submitter { self.submitter.store(kernel::cpu::CpuId::current().as_u32(), Ordering::Relaxed); } } @@ -1513,11 +1526,14 @@ impl Gpu { let render_policy = if m5 { Some(render_resources::M5Policy::new(&id)?) } else { None }; // The bare-metal M5 boot runs both engines; other M5 profiles take no submission. let submits = !m5 || cfg.profile == FirmwareProfile::M5PublishedRoots; - let events = if m5 { - Some(workqueue::OwnedQueue::new_unbound(c"aurora_g17_events", true)?) - } else { - None - }; + let events = Arc::new( + if m5 { + workqueue::OwnedQueue::new_unbound_reclaim(c"aurora_g17_events", true)? + } else { + workqueue::OwnedQueue::new_percpu_reclaim(c"aurora_g17_events", true)? + }, + GFP_KERNEL, + )?; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1552,6 +1568,7 @@ impl Gpu { governor_armed: AtomicBool::new(false), governor_activity_ns: AtomicU64::new(0), events, + follows_submitter: m5, submitter: AtomicU32::new(u32::MAX), grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index ca79208c8dc396..438f9711f815a2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -119,7 +119,10 @@ impl Backend { impl job::Backend for Backend { fn creator_cluster_scheduling(&self) -> bool { - self.shared.events.is_some() + self.shared.follows_submitter() + } + fn fence_queue(&self) -> Arc { + self.shared.fence_queue() } fn input_sources(&self, inputs: &[crate::file::SyncItem]) -> Option> diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index 657ceb2ccf6d35..c79f38f98f9c2e 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -23,9 +23,9 @@ rust_helper_alloc_ordered_workqueue(const char *name, unsigned int flags) } __rust_helper struct workqueue_struct * -rust_helper_alloc_unbound_workqueue(const char *name, bool highpri) +rust_helper_alloc_workqueue_flags(const char *name, unsigned int flags) { - return alloc_workqueue("%s", WQ_UNBOUND | (highpri ? WQ_HIGHPRI : 0), 0, name); + return alloc_workqueue("%s", flags, 0, name); } __rust_helper int rust_helper_workqueue_restrict(struct workqueue_struct *wq, diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index ad98317711b5de..ae1d3bb9b3246a 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -461,6 +461,9 @@ struct SchedulerOwner { /// The driver's submission queue, if it supplied one. Fields drop after `Drop::drop`, so the /// queue is destroyed only after `drm_sched_fini` has stopped the scheduler's work on it. _submit_wq: Option, + /// The driver's timeout queue, if it supplied one; possibly shared with other schedulers. This + /// reference keeps it alive until `drm_sched_fini` has synchronized the timeout work. + _timeout_wq: Option>, } impl Drop for SchedulerOwner { @@ -518,25 +521,34 @@ impl Scheduler { timeout_ms, name, None, + None, ) } - /// Creates a new DRM Scheduler object whose run and free work goes to `submit_wq`. + /// Creates a new DRM Scheduler object whose run and free work goes to `submit_wq` and whose + /// timeout work goes to `timeout_wq`, each where given. + /// + /// Without `submit_wq`, drm_sched allocates an ordered reclaim-safe queue itself. With one, + /// the scheduler owns it and destroys it after it has been finalized; like drm_sched's own it + /// must be ordered and usable for memory reclaim (see [`OwnedQueue::new_ordered_reclaim`]), so + /// that one scheduler's jobs run and are freed one at a time. /// - /// The scheduler owns the queue and destroys it after it has been finalized. Like the queue - /// drm_sched allocates itself, it must be ordered and usable for memory reclaim (see - /// [`OwnedQueue::new_ordered_reclaim`]), so that one scheduler's jobs run and are freed one at a - /// time. + /// Without `timeout_wq`, the timeout work runs on `system_percpu_wq`, which memory reclaim + /// cannot rely on. A driver whose timeout handler signals job fences passes a queue usable for + /// memory reclaim (such as [`OwnedQueue::new_percpu_reclaim`]); several schedulers may share + /// it, and each keeps a reference until it has been finalized. /// /// [`OwnedQueue::new_ordered_reclaim`]: crate::workqueue::OwnedQueue::new_ordered_reclaim - pub fn new_with_submit_wq( + /// [`OwnedQueue::new_percpu_reclaim`]: crate::workqueue::OwnedQueue::new_percpu_reclaim + pub fn new_with_queues( device: &device::Device, num_rqs: u32, credit_limit: u32, hang_limit: u32, timeout_ms: time::Msecs, name: &'static CStr, - submit_wq: crate::workqueue::OwnedQueue, + submit_wq: Option, + timeout_wq: Option>, ) -> Result> { Self::init( UniqueArc::new_uninit(GFP_KERNEL)?, @@ -546,7 +558,8 @@ impl Scheduler { hang_limit, timeout_ms, name, - Some(submit_wq), + submit_wq, + timeout_wq, ) } @@ -560,6 +573,7 @@ impl Scheduler { timeout_ms: time::Msecs, name: &'static CStr, submit_wq: Option, + timeout_wq: Option>, ) -> Result> { // Allocate the owner before starting C workers: no fallible allocation // may abandon an initialized scheduler without running drm_sched_fini. @@ -575,7 +589,7 @@ impl Scheduler { let init_ops = bindings::drm_sched_init_args { ops: &Self::OPS, submit_wq: submit_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), - timeout_wq: core::ptr::null_mut(), + timeout_wq: timeout_wq.as_ref().map_or(core::ptr::null_mut(), |queue| queue.as_raw()), num_rqs, credit_limit, hang_limit, @@ -602,6 +616,7 @@ impl Scheduler { .write(SchedulerOwner { allocation, _submit_wq: submit_wq, + _timeout_wq: timeout_wq, }) .into(), )) diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 0052a68a873bd0..3e4d97583073ab 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -402,7 +402,7 @@ impl WorkItem for ClosureWork { /// A work queue owned by the caller. /// /// Wraps a `struct workqueue_struct` allocated with `alloc_ordered_workqueue()` or, for an -/// unbound queue, `alloc_workqueue(WQ_UNBOUND)`. It dereferences to a [`Queue`], so work items +/// unbound or per-CPU queue, `alloc_workqueue()`. It dereferences to a [`Queue`], so work items /// are enqueued on it exactly like on the system queues. Dropping it calls `destroy_workqueue()`, /// which waits for every pending work item to finish first. /// @@ -410,12 +410,18 @@ impl WorkItem for ClosureWork { /// one at a time in queueing order, and the driver can wait for all of them at once (see /// [`drain`]) before it tears down the state they use. An unbound owned queue runs its work items /// in the affinity pod of the CPU they are queued for (see [`Queue::enqueue_on`]), and either kind -/// can be restricted to a set of CPUs (see [`restrict`]). +/// can be restricted to a set of CPUs (see [`restrict`]). A per-CPU owned queue runs each work item +/// on the CPU it is queued on, as the per-CPU system queues do. +/// +/// A queue whose work may signal a DMA fence, or that memory reclaim otherwise waits on, must be +/// allocated by a `_reclaim` constructor (`WQ_MEM_RECLAIM`): its rescuer thread runs that work +/// when no new worker can be created because memory is short. /// /// # Invariants /// /// `queue` is valid and points at a workqueue allocated by `alloc_ordered_workqueue()` when -/// `ordered` is set, or by `alloc_workqueue(WQ_UNBOUND)` with `WQ_HIGHPRI` when `highpri` is set. +/// `ordered` is set, or else by `alloc_workqueue()`, with `WQ_UNBOUND` when `unbound` is set +/// (`WQ_PERCPU` otherwise) and `WQ_HIGHPRI` when `highpri` is set. Ordered queues are unbound. /// /// [`restrict`]: OwnedQueue::restrict /// @@ -458,6 +464,7 @@ impl WorkItem for ClosureWork { pub struct OwnedQueue { queue: NonNull, ordered: bool, + unbound: bool, highpri: bool, } @@ -485,6 +492,7 @@ impl OwnedQueue { Ok(Self { queue: NonNull::new(ptr).ok_or(ENOMEM)?, ordered: true, + unbound: true, highpri: false, }) } @@ -493,13 +501,34 @@ impl OwnedQueue { /// /// Its work items run concurrently, each in the affinity pod of the CPU it is queued for. pub fn new_unbound(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, 0) + } + + /// Allocates an unbound workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`. + pub fn new_unbound_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, true, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + /// Allocates a per-CPU workqueue named `name` that may be used on the memory reclaim path + /// (`WQ_MEM_RECLAIM`), with high-priority workers if `highpri`: a driver's own counterpart of + /// `system_highpri_wq` (or `system_percpu_wq`) for work that memory reclaim may wait on. + pub fn new_percpu_reclaim(name: &CStr, highpri: bool) -> Result { + Self::flags(name, false, highpri, bindings::wq_flags_WQ_MEM_RECLAIM) + } + + fn flags(name: &CStr, unbound: bool, highpri: bool, flags: u32) -> Result { + let flags = flags + | if unbound { bindings::wq_flags_WQ_UNBOUND } else { bindings::wq_flags_WQ_PERCPU } + | if highpri { bindings::wq_flags_WQ_HIGHPRI } else { 0 }; // SAFETY: `name` is a valid NUL-terminated string; the C side copies it. - let ptr = unsafe { bindings::alloc_unbound_workqueue(name.as_char_ptr(), highpri) }; + let ptr = unsafe { bindings::alloc_workqueue_flags(name.as_char_ptr(), flags) }; - // INVARIANT: The helper returns a valid unbound workqueue, high priority if asked, or NULL. + // INVARIANT: `alloc_workqueue()` returns a valid workqueue with these flags, or NULL. Ok(Self { queue: NonNull::new(ptr).ok_or(ENOMEM)?, ordered: false, + unbound, highpri, }) } @@ -507,8 +536,12 @@ impl OwnedQueue { /// Restricts the queue's workers to the CPUs in `cpus`. /// /// An ordered queue stays ordered and a high-priority one keeps its priority. Allocates, so it - /// must be called from a context that may sleep. + /// must be called from a context that may sleep. A per-CPU queue cannot be restricted + /// (`EINVAL`). pub fn restrict(&self, cpus: &crate::cpumask::Cpumask) -> Result { + if !self.unbound { + return Err(EINVAL); + } // SAFETY: The queue is a valid unbound workqueue by the type invariant and `cpus` is a // valid mask; the helper copies it. crate::error::to_result(unsafe { From 6803e958fc98c12d1f8a86b858bbab1abfede858 Mon Sep 17 00:00:00 2001 From: Jeff Queisser Date: Thu, 5 Nov 2026 19:07:26 -0800 Subject: [PATCH 160/171] drm/asahi: g17: Fix the driver's KUnit tests failing to build after the report and fault-ownership changes With KUnit enabled, asahi no longer built: two of its tests still called interfaces that later commits had changed. - f04338837f3f ("drm/asahi: g17: M5: install M5 queues and submit through the host command port") gave Reports::record() the family's statistics tag mask and 64 tags per role. The report tests now pass the G17 mask and index the second role at TAGS, and a new case checks that tag 32 counts under M5's mask only and that tags past 63 are refused under any mask. - e7ddf2b8e869 ("drm/asahi: g17: Fix a process keeping access to a GPU queue after another process takes it") added Previous::moved. The same-VM replacement test sets it false. Signed-off-by: Jeff Queisser (cherry picked from commit d078ca18fa45ac38a53e3230c0e593ad8d03ebdd in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 + drivers/gpu/drm/asahi/g17/reports.rs | 33 +++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index ec269866bf7c3d..75eaa5972cb6fb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -3115,6 +3115,7 @@ mod fault_ownership_tests { binding: None, context: Lease::new(&used), publication: Some((9, 3)), + moved: false, }); Previous::retire_witnessed(&mut previous); assert_eq!(used.load(Ordering::Relaxed), 0); diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index a6f66bbcd65a89..daf36b22227856 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -123,21 +123,26 @@ impl kernel::debugfs::Writer for PowerView { mod tests { use super::*; + /// The G17 family's statistics tags, as `InitData::statistics_types` gives them. + const G17: u64 = super::super::fw::channels::STATISTICS_TYPES as u64; + /// M5's, which add tag 32. + const M5: u64 = super::super::fw::m5::STATISTICS_TYPES; + #[test] fn retains_all_words_and_separates_role_and_tag() { let mut reports = Reports::new().unwrap(); let mut words = [0; EVENT_WORDS]; words[0] = 16; words[17] = 0xdeadbeef; - reports.record(Role::Primary, words, 100); - reports.record(Role::Primary, words, 101); - reports.record(Role::Secondary, words, 102); + reports.record(Role::Primary, words, 100, G17); + reports.record(Role::Primary, words, 101, G17); + reports.record(Role::Secondary, words, 102, G17); assert_eq!(reports.samples[16].count, 2); assert_eq!(reports.samples[16].host_ns, 101); assert_eq!(reports.samples[16].words, words); - assert_eq!(reports.samples[48].count, 1); + assert_eq!(reports.samples[TAGS + 16].count, 1); words[0] = 17; - reports.record(Role::Primary, words, 103); + reports.record(Role::Primary, words, 103, G17); assert_eq!(reports.samples[17].count, 1); assert_eq!(reports.samples[16].count, 2); } @@ -145,11 +150,25 @@ mod tests { #[test] fn rejects_invalid_tags_before_indexing() { let mut reports = Reports::new().unwrap(); - for tag in [0, 15, 29, 31, 32, u32::MAX] { + for tag in [0, 15, 29, 31, 32, 63, 64, u32::MAX] { let mut words = [0; EVENT_WORDS]; words[0] = tag; - reports.record(Role::Primary, words, 1); + reports.record(Role::Primary, words, 1, G17); } assert!(reports.samples.iter().all(|s| s.count == 0)); } + + #[test] + fn follows_the_family_mask() { + let mut reports = Reports::new().unwrap(); + let mut words = [0; EVENT_WORDS]; + words[0] = 32; + reports.record(Role::Secondary, words, 1, G17); + assert_eq!(reports.samples[TAGS + 32].count, 0); + reports.record(Role::Secondary, words, 2, M5); + assert_eq!(reports.samples[TAGS + 32].count, 1); + words[0] = 64; + reports.record(Role::Secondary, words, 3, u64::MAX); + assert_eq!(reports.samples.iter().filter(|s| s.count != 0).count(), 1); + } } From 443ee65a96c07256ba04dbef5ea327406727b034 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Fri, 6 Nov 2026 16:18:02 -0500 Subject: [PATCH 161/171] drm/asahi: g17: J716c: write the MCC mask and topology words - HW+0x10a8: the active MCC mask for all eight AMCCs (0xff), matching the slot 3 windows, instead of the inherited 0xf. - HW+0x100c (16) and HW+0x1048 (40, all cores). J714 keeps the inherited values, and J775D its own. Developed with Claude Code. Signed-off-by: Erik Bourget (cherry picked from commit d91195fcb8ffd6e89dac1f33cc77e2d3b47c3278 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index 4d2035a5d9ec85..eb75497d98c442 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1789,6 +1789,8 @@ pub(crate) struct Board { pub(crate) idle_standby_us: Option, /// The active memory cache controller mask, where it differs from the inherited 0xf. mcc_mask: Option, + /// HwData +0x100c and +0x1048 (all cores), where the board sets them. + topology_words: Option<(u32, u32)>, /// The hotspot at which the thermal governor's fan curve starts, millidegrees C. pub(crate) fan_curve_start_mc: i32, /// Whether the host encodings are the G17D ones measured on the M5 Ultra @@ -1813,6 +1815,7 @@ pub(crate) const J714: Board = Board { controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, mcc_mask: None, + topology_words: None, fan_curve_start_mc: 84_000, g17d_forms: false, }; @@ -1834,7 +1837,9 @@ pub(crate) const J716C: Board = Board { calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), controller_defaults: Some(encode_m5_policy_controller_defaults), idle_standby_us: None, - mcc_mask: None, + // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. + mcc_mask: Some(0xff), + topology_words: Some((16, 40)), fan_curve_start_mc: 84_000, g17d_forms: false, }; @@ -1865,6 +1870,7 @@ pub(crate) const J775D: Board = Board { idle_standby_us: Some(3500), // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. mcc_mask: Some(0xff), + topology_words: None, // The SMC raises the fans in its automatic mode from a 97 C hotspot under a sustained // full-chip load, with the CPU and GPU still at their top states: the curve starts there, // above the SMC's own start rather than below it. @@ -2299,6 +2305,10 @@ pub(crate) fn encode_m5_hw_data( if let Some(mask) = board.mcc_mask { put_u64(out, 0x10a8, mask); } + if let Some((word, cores)) = board.topology_words { + put_u32(out, 0x100c, word); + put_u32(out, 0x1048, cores); + } // The inherited J813 minimum parameter-buffer size is replaced per board. put_u32(out, 0x26b0, board.min_parameter_buffer); Ok(()) From 77cb173c900f0ae280f3ab41dda02f79461cc840 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Fri, 6 Nov 2026 16:18:09 -0500 Subject: [PATCH 162/171] drm/asahi: g17: J716c: apply its controller defaults The M5 Max's controller defaults were J714's, applied only to a part with J714's calibration words, which no M5 Max has. Give it its own: the J714 defaults, including the relative power curves, with the leakage of its four clusters, its base and AFR values, a maximum power of 196098 mW, and the found masks of its AON PTD records, applied for the J716c calibration words. As on J714, a unit whose calibration words differ keeps the default budget. Developed with Claude Code. Signed-off-by: Erik Bourget (cherry picked from commit f36da76cf6cafb222ec05619fda23b3b64753c20 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs | 34 +++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs index eb75497d98c442..423ae36c1f0af8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/m5/initdata.rs @@ -1833,9 +1833,9 @@ pub(crate) const J716C: Board = Board { register_windows: &J716C_REGISTER_WINDOWS, min_parameter_buffer: 0x0030_0000, render_pool: (0x80_0000, 1 << 33), - // J714's controller configuration, for a part with J714's calibration only. - calibration: Some([0xd2035701, 0xac3a1280, 0x0a281d81]), - controller_defaults: Some(encode_m5_policy_controller_defaults), + // Its own controller configuration, for a part with this calibration. + calibration: Some([0x7e4a4902, 0x8c4f3362, 0x96be56e5]), + controller_defaults: Some(encode_m5_policy_controller_defaults_j716c), idle_standby_us: None, // The ADT GPU node's active-mcc-bit-vector: eight memory cache controllers. mcc_mask: Some(0xff), @@ -2751,6 +2751,34 @@ pub(crate) fn encode_m5_policy_controller_defaults(out: &mut [u8]) -> Result<(), Ok(()) } +/// Initial controller inputs for J716c with its matched calibration: the J714 +/// defaults, including the relative power curves, with this chip's leakage +/// values (clusters 0-3, base, AFR) and its maximum power (196098 mW), and the +/// found masks of its AON PTD records (RT0g and RT1g, then RT0r) next to their +/// tags. +pub(crate) fn encode_m5_policy_controller_defaults_j716c(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_policy_controller_defaults(out)?; + let policy = &mut out[M5_SHARED_LAYOUT.policy..M5_SHARED_LAYOUT.policy + M5_POLICY_SIZE]; + const MAXIMUM_POWER: u32 = 196_098; + const MAXIMUM_POWER_F32: u32 = 0x483f_8080; + for offset in [M5_POLICY_MAXIMUM_POWER, 0xad0, 0x44a4, 0x44a8, 0x44ac, 0x45e4] { + put_u32(policy, offset, MAXIMUM_POWER); + } + for offset in [0x9f4, 0x45e0] { + put_u32(policy, offset, MAXIMUM_POWER_F32); + } + for (index, value) in [0x4524_9000, 0x451f_9000, 0x451f_9000, 0x4524_9000].into_iter().enumerate() { + put_u32(policy, 0x121b0 + index * 4, value); + } + for index in 0..4 { + put_u32(policy, 0x12230 + index * 4, 0x434d_8000); + } + put_u32(policy, 0x12280, 0x44b1_8000); + put_u32(policy, 0x11a64, 3); + put_u32(policy, 0x11a74, 1); + Ok(()) +} + /// Controller inputs for J775D: the J714 defaults, with the Ultra's power model over them. The /// model covers the main and auxiliary operating points of both dies (64 cores, 110 C) with /// synthetic leakage inputs (1350 per cluster core and per die auxiliary domain, 51 per cluster From 848ffc62a27b42511fcb6ab1315516b6899d2ba6 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Fri, 6 Nov 2026 12:28:32 -0500 Subject: [PATCH 163/171] drm/asahi: g17: Run the M5 Max on bare metal The M5 Max (J716c) already has its board data, but its configuration still names no firmware and probe refuses it as having no bare-metal GPU tables. Give it the M5 Ultra's bare-metal profile: both roles publish their kernel roots, the secondary's at 0x40000 in `pagetables`, each asks for its crash log inside `fw-data`, and the UAT policy is the Ultra's. Admit the G17C firmware build of both roles: primary TEXT 0x50000, then the secondary's 0x54000, with the boot-filled ranges zeroed as on the Ultra. Only the M5 Pro is refused at probe now. The firmware identity test covers the Max's image too. Tested on a J716c: the GPU boots both roles and runs the desktop through zink and Vulkan. Developed with Claude Code. Signed-off-by: Erik Bourget (cherry picked from commit 84d4f9057e9f145a4e648ebec0c558c1eecbedaf in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 2 +- drivers/gpu/drm/asahi/fwid.rs | 14 ++++++---- drivers/gpu/drm/asahi/hw/t6050.rs | 46 ++++++++++++++++++++++--------- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 458ac0933e4294..a8a2ea42bba053 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -448,7 +448,7 @@ impl AsahiPlatformDriver { ENODEV })?; if let g17::FirmwareProfile::M5NoBareMetalTables = cfg.profile { - dev_err!(pdev.as_ref(), "M5 Pro/Max bare-metal GPU tables not yet available\n"); + dev_err!(pdev.as_ref(), "M5 Pro bare-metal GPU tables not yet available\n"); return Err(ENODEV); } Self::probe_g17(pdev, res, cfg) diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index ff82f760d2f976..c9a0ef11d8b166 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -495,11 +495,15 @@ mod tests { #[test] fn t6050_admits_only_its_pinned_image() { - let crate::g17::fw::variant::Firmware::Identified(builds) = - crate::hw::t6050::CONFIG_ULTRA.firmware - else { - panic!("J775D on bare metal must identify its loaded firmware"); - }; + for cfg in [&crate::hw::t6050::CONFIG_ULTRA, &crate::hw::t6050::CONFIG_MAX] { + let crate::g17::fw::variant::Firmware::Identified(builds) = cfg.firmware else { + panic!("an M5 board on bare metal must identify its loaded firmware"); + }; + t6050_admits_only(builds); + } + } + + fn t6050_admits_only(builds: &[crate::g17::fw::variant::Build]) { for image in builds.iter().flat_map(|build| build.images) { // Both roles' records, inside TEXT. assert_eq!(image.entropy.len(), 2); diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs index 3ac6ac6d0837de..7856adc317e037 100644 --- a/drivers/gpu/drm/asahi/hw/t6050.rs +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -57,12 +57,6 @@ pub(crate) const CONFIG: Config = Config { }, }; -// J716c ADT gfx-data-shared-ro-base and gfx1-data-shared-ro-base. -const J716C_CRASH_BUFFERS: [FixedBuffer; 2] = [ - FixedBuffer { va: 0x100_0283_c000, size: 0x8000 }, - FixedBuffer { va: 0x100_028f_8000, size: 0x8000 }, -]; - /// M5 Max: ID variant 4, GPU variant C, HAL generation 300 like the M5 Pro. pub(crate) const CONFIG_MAX: Config = Config { chip_id: 0x6050, @@ -76,16 +70,18 @@ pub(crate) const CONFIG_MAX: Config = Config { }, uat_ias: 42, uat_oas: 42, - // Unknown until the board has bare-metal page tables. - secondary_root_offset: None, - crash_buffers: CrashBuffers::Fixed(J716C_CRASH_BUFFERS), - profile: FirmwareProfile::M5NoBareMetalTables, + // Bare metal, as the M5 Ultra: both roles publish their kernel roots, the secondary's + // 0x40000 into `pagetables`, and each asks for its crash log inside `fw-data`. + secondary_root_offset: Some(0x40000), + crash_buffers: CrashBuffers::FirmwareData, + profile: FirmwareProfile::M5PublishedRoots, pm_metrics: None, windows: &[], reserved_slots: &[], perf: None, opps: 0, - uat_quirks: crate::mmu::UatQuirks::DEFAULT, + // The M5 Ultra's UAT policy, which this board's firmware expects too. + uat_quirks: CONFIG_ULTRA.uat_quirks, region_c: None, chip: None, m5: Some(&crate::g17::fw::m5::initdata::J716C), @@ -93,8 +89,32 @@ pub(crate) const CONFIG_MAX: Config = Config { fabric_bridge: false, recovery_slot_keys: None, gpu_generation: GpuGen::G17, - // No loaded image is admitted: the board has no bare-metal page tables yet. - firmware: crate::g17::fw::variant::Firmware::Identified(&[]), + // As on the M5 Ultra, the firmware in `fw-text` must be this build. The boot loader loads + // the G17C build of both roles back to back: the primary's TEXT 0x50000, then the + // secondary's 0x54000. The digest is that of the firmware's TEXT with the boot-filled ranges + // zeroed. + firmware: crate::g17::fw::variant::Firmware::Identified(&[crate::g17::fw::variant::Build { + os_version: [27, 0, 0], + name: "27.0 (26A428), G17C", + images: &[ + crate::fwid::FirmwareBuild { + name: "26A428 for T6050 G17C, primary then secondary", + text_size: 0xa4000, + data_size: 0x184000, + entropy: &[[0x49e60, 0x49f43], [0x9d918, 0x9d9fb]], + boot_filled: &[ + (0x224, 4), (0x23c, 8), (0x49720, 0x740), (0x49e60, 0x231), + (0x50224, 4), (0x5023c, 8), (0x9d1d8, 0x740), (0x9d918, 0x231), + ], + text_sha256: [ + 0x30, 0x4d, 0x85, 0x44, 0x66, 0x29, 0xbc, 0xdc, 0xa5, 0x95, 0x80, 0xb5, 0xff, + 0x00, 0x6d, 0xd8, 0x9f, 0x2c, 0xd8, 0xb4, 0xca, 0x60, 0x59, 0x9a, 0x0c, 0xb5, + 0x5c, 0x35, 0xec, 0xb0, 0x98, 0xf4, + ], + }, + ], + variant: &crate::g17::fw::variant::V26A428_M5, + }]), // Four clusters of ten cores. The buffers are sized for every core slot of the board, so the // board's cluster and slot counts admit the GPU, whichever cores are fused off. core_layout: CoreLayout { From a6a0c746b727abc08a89b29b4e12ab2ae4ac2c9b Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Fri, 6 Nov 2026 12:55:24 -0500 Subject: [PATCH 164/171] drm/asahi: Match the T6050 GPU by its G17S compatible Bind the T6050 GPU through "apple,agx-g17s", the compatible its M5 Max ("apple,agx-g17c") and M5 Ultra ("apple,agx-g17d") nodes fall back to, in place of "apple,agx-t6050", which no binding documents. The GPU's ID registers still choose the board's configuration. Developed with Claude Code. Signed-off-by: Erik Bourget [Eduard Moldovan: applied to integration/aurora's 7.2.11 OF table] (cherry picked from commit b9dc01a3703813ab6ab25cb25253051b224b03ef in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a8a2ea42bba053..1eac515aa709fa 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -180,7 +180,7 @@ kernel::of_device_table!( ProbeConfig::M5 ), ( - of::DeviceId::new(c"apple,agx-t6050"), + of::DeviceId::new(c"apple,agx-g17s"), ProbeConfig::M5 ), ( From f5deea4a5a7cbe4f4224ab239d69741a672975aa Mon Sep 17 00:00:00 2001 From: Jonathan Goyvaerts Date: Fri, 30 Oct 2026 00:46:28 -0300 Subject: [PATCH 165/171] drm/asahi: g17: log the MMU fault banks on recovery A G17 recovery reports the firmware's reason (3 is a page fault) and the blamed queue, but not which address faulted or which unit faulted on it: the G17 runtime never reads the MMU fault registers. On T6040 a Transport Fever 3 shadow pass page-faults its fragment queue repeatably, and without the address there is no telling whether a depth surface, a tiler heap page or freed client memory is behind it. T6040's GPU is G16S hardware, and G15/G16 expose the fault through two banks: status at SGX +0xd8c0 and address (64-byte units) at +0xd8c8 under the bank selector at +0xd800 (g16::regs::Regs::engine_state). Read both banks while the firmware is halted in the recovery handshake, restoring the selector, and log them decoded by the existing G14X decoder, now shared as regs::decode_fault(). This is diagnostics only: nothing in attribution, cancellation or release depends on it. Signed-off-by: Jonathan Goyvaerts Assisted-by: LLM (cherry picked from commit 8ce1613ac129cbe3f0ebff9b8ac0b58e04d964c2 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/port.rs | 5 + drivers/gpu/drm/asahi/g17/recovery.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 22 ++ drivers/gpu/drm/asahi/regs.rs | 227 +++++++++++--------- 4 files changed, 158 insertions(+), 99 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index eab31f896e2b5d..95bc4893a616ec 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -221,6 +221,11 @@ impl Port { self.views.as_ref().ok_or(ENODEV) } + /// Both MMU fault banks, for recovery diagnostics while the firmware is halted. + pub(super) fn fault_banks(&self) -> Result<[(u64, u64); 2]> { + self.regs.as_ref().ok_or(ENODEV)?.fault_banks() + } + fn ring(&self, priority: u8) -> Result<(usize, RingView)> { let index = usize::from(priority); Ok((index, *self.views()?.compute.get(index).ok_or(EINVAL)?)) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 8d083e755b741d..40f1311d269836 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -113,6 +113,8 @@ pub(super) trait Queues { fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; /// Optional diagnostics only; absence cannot alter attribution or release. fn report_blamed_render(&self, _qid: u8) {} + /// Optional diagnostics: logs the MMU fault banks while the firmware is halted. + fn report_mmu_fault(&self) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -259,6 +261,7 @@ pub(super) fn service( } let mut sources = host.fault_sources()?; sources.report(request.generation, super::queue::compute::retirement::stampless_total()); + host.report_mmu_fault(); if let Some(qid) = request.blamed { host.report_blamed_render(qid); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index b66f1a17444c8e..51144651450815 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -271,6 +271,28 @@ impl recovery::Queues for Service<'_> { .queues .report_render_recovery(qid, &self.firmware.port); } + fn report_mmu_fault(&self) { + let banks = match self.firmware.port.fault_banks() { + Ok(banks) => banks, + Err(error) => { + pr_warn!("G17: recovery MMU fault banks unreadable: {:?}\n", error); + return; + } + }; + for (bank, &(status, address)) in banks.iter().enumerate() { + match crate::regs::decode_fault(status, address, true) { + Some(info) => pr_warn!( + "G17: recovery MMU fault bank {}: address {:#x} {} {:?} unit {:?} ({:#x}) level {} vm_slot {} sideband {:#x} (status {:#x})\n", + bank, info.address, if info.read { "read" } else { "write" }, info.reason, + info.unit, info.unit_code, info.level, info.vm_slot, info.sideband, status + ), + None => pr_info!( + "G17: recovery MMU fault bank {}: no fault (status {:#x})\n", + bank, status + ), + } + } + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 2dacc4c95a16c5..94db751325217d 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -67,6 +67,9 @@ const CORE_MASK_1: usize = 0xd01514; const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; +/// Selects which of the two MMU fault banks `FAULT_INFO_G14X`/`FAULT_ADDR_G14X` show (G15 and +/// later). +const FAULT_BANK_SELECT: usize = 0xd800; /// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; @@ -464,107 +467,117 @@ impl Resources<'_> { fault_info >> 30 }; - let unit_code = ((fault_info >> 9) & 0xff) as u8; - let unit = match unit_code { - 0x00..=0x9f => match unit_code & 0xf { - 0x0 => FaultUnit::DCMP(unit_code >> 4), - 0x1 => FaultUnit::UL1C(unit_code >> 4), - 0x2 => FaultUnit::CMP(unit_code >> 4), - 0x3 => FaultUnit::GSL1(unit_code >> 4), - 0x4 => FaultUnit::IAP(unit_code >> 4), - 0x5 => FaultUnit::VCE(unit_code >> 4), - 0x6 => FaultUnit::TE(unit_code >> 4), - 0x7 => FaultUnit::RAS(unit_code >> 4), - 0x8 => FaultUnit::VDM(unit_code >> 4), - 0x9 => FaultUnit::PPP(unit_code >> 4), - 0xa => FaultUnit::IPF(unit_code >> 4), - 0xb => FaultUnit::IPF_CPF(unit_code >> 4), - 0xc => FaultUnit::VF(unit_code >> 4), - 0xd => FaultUnit::VF_CPF(unit_code >> 4), - 0xe => FaultUnit::ZLS(unit_code >> 4), - _ => FaultUnit::Unknown(unit_code), - }, - 0xa1 => FaultUnit::dPM, - 0xa2 => FaultUnit::dCDM_KS(0), - 0xa3 => FaultUnit::dCDM_KS(1), - 0xa4 => FaultUnit::dCDM_KS(2), - 0xa5 => FaultUnit::dIPP, - 0xa6 => FaultUnit::dIPP_CS, - 0xa7 => FaultUnit::dVDM_CSD, - 0xa8 => FaultUnit::dVDM_SSD, - 0xa9 => FaultUnit::dVDM_ILF, - 0xaa => FaultUnit::dVDM_ILD, - 0xab => FaultUnit::dRDE(0), - 0xac => FaultUnit::dRDE(1), - 0xad => FaultUnit::FC, - 0xae => FaultUnit::GSL2, - 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), - 0xb8 => FaultUnit::GL2CC_MB, - 0xd0..=0xdf if g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gCDM_CS, - 0x1 => FaultUnit::gCDM_ID, - 0x2 => FaultUnit::gCDM_CSR, - 0x3 => FaultUnit::gCDM_CSW, - 0x4 => FaultUnit::gCDM_CTXR, - 0x5 => FaultUnit::gCDM_CTXW, - 0x6 => FaultUnit::gIPP, - 0x7 => FaultUnit::gIPP_CS, - 0x8 => FaultUnit::gKSM_RCE, - _ => FaultUnit::Unknown(unit_code), - }, - 0xe0..=0xff if g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), - 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), - 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), - 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), - 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), - 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), - 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), - _ => FaultUnit::Unknown(unit_code), - }, - 0xe0..=0xff if !g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), - 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), - 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), - 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), - 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), - 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), - 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), - 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), - 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), - 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), - 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), - 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), - 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), - 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), - 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), - 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), - _ => FaultUnit::Unknown(unit_code), - }, - _ => FaultUnit::Unknown(unit_code), - }; - - let reason = match (fault_info >> 1) & 0x7 { - 0 => FaultReason::Unmapped, - 1 => FaultReason::AfFault, - 2 => FaultReason::WriteOnly, - 3 => FaultReason::ReadOnly, - 4 => FaultReason::NoAccess, - a => FaultReason::Unknown(a as u8), - }; + decode_fault(fault_info, fault_addr, g14x) + } +} - Some(FaultInfo { - address: fault_addr << 6, - sideband: ((fault_info >> 23) & 0x7f) as u8, - vm_slot: ((fault_info >> 17) & 0x3f) as u32, - unit_code, - unit, - level: ((fault_info >> 7) & 3) as u8, - unk_5: ((fault_info >> 5) & 3) as u8, - read: (fault_info & (1 << 4)) != 0, - reason, - }) +/// Decodes an MMU fault status word and its raw address register (64-byte units) into a +/// [`FaultInfo`], if the status records a fault. `g14x` selects the G14X-and-later unit numbering. +pub(crate) fn decode_fault(fault_info: u64, fault_addr: u64, g14x: bool) -> Option { + if fault_info & 1 == 0 { + return None; } + + let unit_code = ((fault_info >> 9) & 0xff) as u8; + let unit = match unit_code { + 0x00..=0x9f => match unit_code & 0xf { + 0x0 => FaultUnit::DCMP(unit_code >> 4), + 0x1 => FaultUnit::UL1C(unit_code >> 4), + 0x2 => FaultUnit::CMP(unit_code >> 4), + 0x3 => FaultUnit::GSL1(unit_code >> 4), + 0x4 => FaultUnit::IAP(unit_code >> 4), + 0x5 => FaultUnit::VCE(unit_code >> 4), + 0x6 => FaultUnit::TE(unit_code >> 4), + 0x7 => FaultUnit::RAS(unit_code >> 4), + 0x8 => FaultUnit::VDM(unit_code >> 4), + 0x9 => FaultUnit::PPP(unit_code >> 4), + 0xa => FaultUnit::IPF(unit_code >> 4), + 0xb => FaultUnit::IPF_CPF(unit_code >> 4), + 0xc => FaultUnit::VF(unit_code >> 4), + 0xd => FaultUnit::VF_CPF(unit_code >> 4), + 0xe => FaultUnit::ZLS(unit_code >> 4), + _ => FaultUnit::Unknown(unit_code), + }, + 0xa1 => FaultUnit::dPM, + 0xa2 => FaultUnit::dCDM_KS(0), + 0xa3 => FaultUnit::dCDM_KS(1), + 0xa4 => FaultUnit::dCDM_KS(2), + 0xa5 => FaultUnit::dIPP, + 0xa6 => FaultUnit::dIPP_CS, + 0xa7 => FaultUnit::dVDM_CSD, + 0xa8 => FaultUnit::dVDM_SSD, + 0xa9 => FaultUnit::dVDM_ILF, + 0xaa => FaultUnit::dVDM_ILD, + 0xab => FaultUnit::dRDE(0), + 0xac => FaultUnit::dRDE(1), + 0xad => FaultUnit::FC, + 0xae => FaultUnit::GSL2, + 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), + 0xb8 => FaultUnit::GL2CC_MB, + 0xd0..=0xdf if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gCDM_CS, + 0x1 => FaultUnit::gCDM_ID, + 0x2 => FaultUnit::gCDM_CSR, + 0x3 => FaultUnit::gCDM_CSW, + 0x4 => FaultUnit::gCDM_CTXR, + 0x5 => FaultUnit::gCDM_CTXW, + 0x6 => FaultUnit::gIPP, + 0x7 => FaultUnit::gIPP_CS, + 0x8 => FaultUnit::gKSM_RCE, + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if !g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), + 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), + 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), + 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), + 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), + 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), + 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), + 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), + 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + _ => FaultUnit::Unknown(unit_code), + }; + + let reason = match (fault_info >> 1) & 0x7 { + 0 => FaultReason::Unmapped, + 1 => FaultReason::AfFault, + 2 => FaultReason::WriteOnly, + 3 => FaultReason::ReadOnly, + 4 => FaultReason::NoAccess, + a => FaultReason::Unknown(a as u8), + }; + + Some(FaultInfo { + address: fault_addr << 6, + sideband: ((fault_info >> 23) & 0x7f) as u8, + vm_slot: ((fault_info >> 17) & 0x3f) as u32, + unit_code, + unit, + level: ((fault_info >> 7) & 3) as u8, + unk_5: ((fault_info >> 5) & 3) as u8, + read: (fault_info & (1 << 4)) != 0, + reason, + }) } /// Device resources of a G17 GPU. @@ -705,6 +718,22 @@ impl G17Resources { Ok(()) } + /// Raw status and address words of both MMU fault banks, restoring the bank selector: the + /// G15/G16 layout (`g16::regs::Regs::engine_state`), which T6040's G16S hardware shares. Only + /// while the firmware is halted, since it uses the same selector. + pub(crate) fn fault_banks(&self) -> Result<[(u64, u64); 2]> { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let selector = sgx.try_read64(FAULT_BANK_SELECT)?; + let mut banks = [(0, 0); 2]; + for (bank, words) in banks.iter_mut().enumerate() { + sgx.try_write64(bank as u64, FAULT_BANK_SELECT)?; + *words = (sgx.try_read64(FAULT_INFO_G14X)?, sgx.try_read64(FAULT_ADDR_G14X)?); + } + sgx.try_write64(selector, FAULT_BANK_SELECT)?; + Ok(banks) + } + fn sgx_read32(&self) -> u32 { if let Some(sgx) = self.sgx.try_access() { (&*sgx).as_view().relaxed().read32(OFF) From d76e8cfeb30c2c4f85467738f7e2c8738ead26b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 11:48:37 +1100 Subject: [PATCH 166/171] drm/asahi: introduce the complete T6040 calibration reader Validate version 2 of the per-chip handoff, sixteen raw OPPs and selector slots, eleven populated selectors, zero tails and base state 1. There is no single-unit fallback. The initdata patch wires the reader into probe and firmware construction so this patch adds no unused compiled items. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit a8fb1380b41456833b30014e6e354fba31551cd3 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/calibration.rs | 231 +++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 drivers/gpu/drm/asahi/g17/calibration.rs diff --git a/drivers/gpu/drm/asahi/g17/calibration.rs b/drivers/gpu/drm/asahi/g17/calibration.rs new file mode 100644 index 00000000000000..d973349d35d403 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/calibration.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-chip T6040 power data supplied by the boot loader. + +use super::fw::initdata::{Opp, PerfConfig, PERF_STATES}; +use crate::driver::AsahiDrmDevice; +use kernel::{c_str, of, prelude::*, str::CStr}; + +/// Validated calibration of the running chip. There is no built-in fallback. +pub(super) struct Calibration { + pub(super) perf: PerfConfig, + pub(super) leakage: [u32; 56], + pub(super) max_power_mw: u32, + core_mask: u32, + base: usize, +} + +fn words(node: &of::Node, name: &CStr) -> Result<[u32; N]> { + let values: KVec = node.get_property(name)?; + values.as_slice().try_into().map_err(|_| EINVAL) +} + +// Raw ADT OPP slots and SGX register slots number 16. The supported firmware consumes +// the 11 populated selectors; the remaining handoff entries must all be zero. +const RAW_STATES: usize = 16; + +fn active(values: [u32; RAW_STATES]) -> Result<[u32; PERF_STATES]> { + if values[PERF_STATES..].iter().any(|&v| v != 0) { + return Err(EINVAL); + } + values[..PERF_STATES].try_into().map_err(|_| EINVAL) +} + +fn map(values: [u32; RAW_STATES]) -> Result<[u8; PERF_STATES]> { + let values = active(values)?; + if values[0] != 0 + || values[PERF_STATES - 1] != RAW_STATES as u32 - 1 + || values.iter().any(|&v| v >= RAW_STATES as u32) + || values.windows(2).any(|p| p[0] >= p[1]) + { + return Err(EINVAL); + } + Ok(values.map(|v| v as u8)) +} + +fn relative(values: &[u32; PERF_STATES]) -> bool { + values[0] == 0 + && values[PERF_STATES - 1] == 100 + && values.windows(2).all(|p| p[0] <= p[1] && p[1] <= 100) +} + +impl Calibration { + pub(super) fn read(dev: &AsahiDrmDevice) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + if node.get_property::(c_str!("apple,t6040-handoff-version"))? != 2 { + return Err(EINVAL); + } + let core_mask = node.get_property(c_str!("apple,core-mask"))?; + let max_power_mw = node.get_property(c_str!("apple,max-power-mw"))?; + let base = node.get_property::(c_str!("apple,perf-base-pstate"))? as usize; + if core_mask == 0 + || core_mask & !0xfffff != 0 + || !(1..=200_000).contains(&max_power_mw) + || base != 1 + { + return Err(EINVAL); + } + let perf = PerfConfig { + states_a: map(words(&node, c_str!("apple,perf-map-a"))?)?, + states_b: map(words(&node, c_str!("apple,perf-map-b"))?)?, + relative_a: active(words(&node, c_str!("apple,perf-relative-a"))?)?, + relative_b: active(words(&node, c_str!("apple,perf-relative-b"))?)?, + sram_min_mv: 810, + }; + if !relative(&perf.relative_a) + || !relative(&perf.relative_b) + || perf.relative_b[..=base].iter().any(|&v| v != 0) + { + return Err(EINVAL); + } + let core = words::<2>(&node, c_str!("apple,core-leak-coef"))?; + let sram = words::<2>(&node, c_str!("apple,sram-leak-coef"))?; + let cs = words::<1>(&node, c_str!("apple,cs-leak-coef"))?; + let afr = words::<1>(&node, c_str!("apple,afr-leak-coef"))?; + // Positive finite binary32 values. Integer comparisons preserve their ordering. + if Iterator::chain( + Iterator::chain(Iterator::chain(core.iter(), &sram), &cs), + &afr, + ) + .any(|&v| !(0x3f80_0000..0x4780_0000).contains(&v)) + { + return Err(EINVAL); + } + let mut leakage = [0; 56]; + leakage[..2].copy_from_slice(&core); + leakage[32..34].copy_from_slice(&sram); + leakage[52] = cs[0]; + leakage[54] = afr[0]; + Ok(Self { + perf, + leakage, + max_power_mw, + core_mask, + base, + }) + } + + /// Check the loader's mask against the GPU before starting its firmware. + pub(super) fn check_cores(&self, masks: &[u32]) -> Result { + if masks != [self.core_mask & 0x3ff, self.core_mask >> 10] { + return Err(EINVAL); + } + Ok(()) + } + + /// The performance maps index OPPs in ADT order, rather than sorted frequency order. + pub(super) fn check_opps(&self, opps: &[Opp]) -> Result { + if opps.len() != 16 || opps[0].freq_hz != 0 || opps.iter().any(|opp| opp.cells != 2) { + return Err(EINVAL); + } + for map in [&self.perf.states_a, &self.perf.states_b] { + let mut previous = 0; + for &index in &map[1..] { + let opp = &opps[usize::from(index)]; + let mhz = opp.freq_hz / 1_000_000; + if mhz <= previous + || opp.volt_uv[..2] + .iter() + .any(|v| !(400_000..=1_300_000).contains(v)) + { + return Err(EINVAL); + } + previous = mhz; + } + } + let base = opps[usize::from(self.perf.states_a[self.base])].freq_hz / 1_000_000; + let top = opps[usize::from(self.perf.states_a[PERF_STATES - 1])].freq_hz / 1_000_000; + for (i, &index) in self.perf.states_a.iter().enumerate() { + let freq = opps[usize::from(index)].freq_hz / 1_000_000; + let expected = if i < self.base { + 0 + } else { + 100 * (freq - base) / (top - base) + }; + if u64::from(self.perf.relative_b[i]) != expected { + return Err(EINVAL); + } + } + Ok(()) + } +} + +#[kunit_tests(asahi_t6040_calibration)] +mod tests { + use super::*; + + fn chip() -> Calibration { + Calibration { + perf: PerfConfig { + states_a: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + states_b: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + relative_a: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], + relative_b: [0, 0, 11, 22, 33, 44, 55, 66, 77, 88, 100], + sram_min_mv: 810, + }, + leakage: [0; 56], + max_power_mw: 70_000, + core_mask: 0xfffff, + base: 1, + } + } + + #[test] + fn operating_points_and_mask_must_match_the_handoff() { + let chip = chip(); + let mut opps = [Opp { + freq_hz: 0, + volt_uv: [800_000; super::super::fw::initdata::OPP_CELLS], + cells: 2, + }; 16]; + for (i, opp) in opps.iter_mut().enumerate() { + opp.freq_hz = i as u64 * 100_000_000; + } + assert!(chip.check_opps(&opps).is_ok()); + opps[0].freq_hz = 100_000_000; + assert!(chip.check_opps(&opps).is_err()); + opps[0].freq_hz = 0; + assert!(chip.check_cores(&[0x3ff, 0x3ff]).is_ok()); + assert!(chip.check_cores(&[0x3ff, 0x1ff]).is_err()); + assert!(chip.check_cores(&[0xfffff]).is_err()); + opps[5].volt_uv[1] = 0; + assert!(chip.check_opps(&opps).is_err()); + opps[5].volt_uv[1] = 800_000; + opps[5].freq_hz = opps[4].freq_hz; + assert!(chip.check_opps(&opps).is_err()); + opps[5].freq_hz = 501_000_000; + assert!(chip.check_opps(&opps).is_ok()); + opps[5].freq_hz = 511_000_000; + assert!(chip.check_opps(&opps).is_err()); + } + + #[test] + fn maps_refuse_off_repeats_and_out_of_range() { + let valid = [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15, 0, 0, 0, 0, 0]; + assert!(map(valid).is_ok()); + for (index, value) in [ + (0, 1), + (1, 0), + (5, 4), + (10, 16), + (10, u32::MAX), + (10, 14), + (11, 1), + (15, 15), + ] { + let mut bad = valid; + bad[index] = value; + assert!(map(bad).is_err()); + } + } + + #[test] + fn relative_tables_are_bounded_and_monotonic() { + let mut values = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; + assert!(relative(&values)); + values[5] = 101; + assert!(!relative(&values)); + values[5] = 39; + assert!(!relative(&values)); + } +} From ad90cc614eda39a0ed809920478787d365b6d8c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 26 Oct 2026 22:20:53 +1100 Subject: [PATCH 167/171] drm/asahi: test the complete T6040 performance selector ladder Use valid sixteen-slot handoffs in the calibration KUnit fixture, with both eleven-state maps ending at raw OPP 15. Check the highest state's second voltage lane as well as the intermediate-state ordering and relative-frequency consistency. The former 0..10 fixture could not pass the production map reader and never exercised the final five raw operating points. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 989406b403571dc0e7429fdd1433c8fd72ca12a8 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/gpu/drm/asahi/g17/calibration.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/calibration.rs b/drivers/gpu/drm/asahi/g17/calibration.rs index d973349d35d403..704a91f49b0c17 100644 --- a/drivers/gpu/drm/asahi/g17/calibration.rs +++ b/drivers/gpu/drm/asahi/g17/calibration.rs @@ -157,10 +157,10 @@ mod tests { fn chip() -> Calibration { Calibration { perf: PerfConfig { - states_a: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - states_b: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + states_a: map([0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15, 0, 0, 0, 0, 0]).unwrap(), + states_b: map([0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15, 0, 0, 0, 0, 0]).unwrap(), relative_a: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], - relative_b: [0, 0, 11, 22, 33, 44, 55, 66, 77, 88, 100], + relative_b: [0, 0, 7, 14, 21, 35, 50, 64, 78, 92, 100], sram_min_mv: 810, }, leakage: [0; 56], @@ -188,14 +188,21 @@ mod tests { assert!(chip.check_cores(&[0x3ff, 0x3ff]).is_ok()); assert!(chip.check_cores(&[0x3ff, 0x1ff]).is_err()); assert!(chip.check_cores(&[0xfffff]).is_err()); - opps[5].volt_uv[1] = 0; + // Both eleven-state maps must reach raw OPP 15, including its second voltage lane. + assert_eq!(chip.perf.states_a[PERF_STATES - 1], 15); + assert_eq!(chip.perf.states_b[PERF_STATES - 1], 15); + opps[15].volt_uv[1] = 0; assert!(chip.check_opps(&opps).is_err()); - opps[5].volt_uv[1] = 800_000; - opps[5].freq_hz = opps[4].freq_hz; + opps[15].volt_uv[1] = 800_000; + assert!(chip.check_opps(&opps).is_ok()); + opps[6].volt_uv[1] = 0; + assert!(chip.check_opps(&opps).is_err()); + opps[6].volt_uv[1] = 800_000; + opps[6].freq_hz = opps[4].freq_hz; assert!(chip.check_opps(&opps).is_err()); - opps[5].freq_hz = 501_000_000; + opps[6].freq_hz = 601_000_000; assert!(chip.check_opps(&opps).is_ok()); - opps[5].freq_hz = 511_000_000; + opps[6].freq_hz = 611_000_000; assert!(chip.check_opps(&opps).is_err()); } From 2a41c945330b2daed357e908f165860e92c7adbb Mon Sep 17 00:00:00 2001 From: Eduard Moldovan Date: Tue, 10 Nov 2026 11:38:50 +0100 Subject: [PATCH 168/171] drm/asahi: build the carried G17 and M5 work on integration/aurora The aurora-silicon G17 and M5 commits carried before this one are written for Linux 7.3 and aurora-silicon's driver core. On integration/aurora (Linux 7.2.11): - Module parameters return references: read the M5 mesh storage, MSAA depth compression and thermal target parameters, and stats_export, through them. - float.rs already has F32::to_bits(): drop the second definition. - The M3 T8122 probe takes a device binding owner, as the other probes now do, and its Gpu::new_queue() takes the client usage argument the trait now has. - The G15 backend keeps its identity module, which aurora-silicon moved into the shared fwid code. - The SMC hwmon cell has no kernel fan request: the opt-in M5 thermal governor's fan request returns -EOPNOTSUPP and the SMC keeps the fans. - Without CONFIG_CPU_FREQ, cpufreq_cpu_get() and cpufreq_cpu_put() are inline stubs the Rust bindings lack: the M5 thermal governor takes no CPU frequency requests then. - The recovery KUnit mock drops announce_withheld(), which 8ce1613ac129 adds for a Queues method integration/aurora does not have. - kunit-expected.json lists asahi_fwid.j713_accepts_both_loaded_images as skipped: it is a G16 case and the G16 backend is not built. Signed-off-by: Eduard Moldovan Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/driver.rs | 6 +++++- drivers/gpu/drm/asahi/float.rs | 5 ----- drivers/gpu/drm/asahi/g15/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fans.c | 3 ++- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- .../drm/asahi/g17/queue/render/m5_render_submission.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/render/memory.rs | 2 +- drivers/gpu/drm/asahi/g17/recovery.rs | 1 - drivers/gpu/drm/asahi/g17/thermal.rs | 10 +++++++++- drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- tools/aurora-ci/kunit-expected.json | 4 ++++ 11 files changed, 25 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 1eac515aa709fa..80e988f3e3507a 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -411,6 +411,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -425,6 +428,7 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -489,7 +493,7 @@ impl AsahiPlatformDriver { let raw_stats = match crate::sysfs_exports::register_g17( raw_dev, snapshot, - module_parameters::stats_export.value() != 0, + *module_parameters::stats_export.value() != 0, ) { Ok(handle) => handle, Err(error) => { diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 51cac02624d3d1..52261d10757d18 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -54,11 +54,6 @@ impl F32 { F32(u) } - /// Convert to the raw IEEE 754 bits. - pub(crate) const fn to_bits(self) -> u32 { - self.0 - } - // Convert a `f32` value into an F32 // // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index d55b6385f6214f..7ebc796e809c17 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -59,6 +59,7 @@ pub(crate) mod fw; mod gate; mod graph; pub(crate) mod hw; +mod identity; mod initdata; mod job; mod object; diff --git a/drivers/gpu/drm/asahi/g17/fans.c b/drivers/gpu/drm/asahi/g17/fans.c index 6178a620686084..b41dd4fe2288d9 100644 --- a/drivers/gpu/drm/asahi/g17/fans.c +++ b/drivers/gpu/drm/asahi/g17/fans.c @@ -20,5 +20,6 @@ int asahi_zone_temp(const char *name, int *temp) int asahi_fans_request(struct device *dev, unsigned int permille, unsigned int min_step_rpm) { - return macsmc_hwmon_fans_request(dev, permille, min_step_rpm); + /* The SMC hwmon cell has no kernel fan request here: the SMC keeps the fans. */ + return -EOPNOTSUPP; } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 227a9b50beb9f0..32b99f6fd99990 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -2178,7 +2178,7 @@ impl gpu::Gpu for Gpu { params.features |= crate::file::DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION | crate::file::DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION; } - if crate::module_parameters::m5_mesh_storage_chunks.value() != 0 { + if *crate::module_parameters::m5_mesh_storage_chunks.value() != 0 { params.features |= crate::file::DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs index 935e62593f0163..2f1c6fee6e9a68 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/m5_render_submission.rs @@ -8,7 +8,7 @@ use crate::g17::{buffer::target, command::Validated, fw::render::{RenderArgs, Re /// Compressed single-layer MSAA ZLS admission: D32 depth with optional plain /// stencil, and the compressed Z/S form. pub(crate) fn depth_msaa_compression_enabled() -> bool { - crate::module_parameters::m5_depth_msaa_compression.value() == 1 + *crate::module_parameters::m5_depth_msaa_compression.value() == 1 } pub(crate) fn validate(pass: &RenderPass) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index b129f521cc34ca..1772593a16022b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -658,7 +658,7 @@ impl Memory { /// A zero chunk parameter disables the binding. pub(super) fn mesh_storage_register(&mut self, alloc: &Allocator<'_>) -> Result { if self.mesh_storage.is_none() { - let chunks = crate::module_parameters::m5_mesh_storage_chunks.value(); + let chunks = *crate::module_parameters::m5_mesh_storage_chunks.value(); if chunks == 0 { return Ok(0); } if chunks > 0xffff { return Err(EINVAL); } let bytes = (chunks as usize).checked_mul(MESH_STORAGE_CHUNK).ok_or(EOVERFLOW)?; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 40f1311d269836..c931a8c0570772 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -519,7 +519,6 @@ mod tests { fn settle_cancelled_renders(&mut self, _qids: u128) {} fn rescan_compute(&mut self) -> Result { Ok(()) } fn release_retired_compute(&mut self) {} - fn announce_withheld(&mut self) {} } impl Host for Halted { diff --git a/drivers/gpu/drm/asahi/g17/thermal.rs b/drivers/gpu/drm/asahi/g17/thermal.rs index 0f5c21a77488ae..5ed91a99de1ac3 100644 --- a/drivers/gpu/drm/asahi/g17/thermal.rs +++ b/drivers/gpu/drm/asahi/g17/thermal.rs @@ -126,7 +126,7 @@ pub(super) fn requested() -> bool { enabled() } fn target_mc() -> i32 { - let parameter = crate::module_parameters::m5_thermal_target_mc.value(); + let parameter = *crate::module_parameters::m5_thermal_target_mc.value(); i32::try_from(parameter).unwrap_or(90_000).clamp(70_000, 104_000) } @@ -419,7 +419,14 @@ impl CpuCeiling { (u64::from(max_khz) * u64::from(share) / 100).min(i32::MAX as u64) as i32 } + /// Without cpufreq there is no policy to limit. + #[cfg(not(CONFIG_CPU_FREQ))] + fn acquire(&mut self) -> bool { + false + } + /// Takes a request on every policy (each policy once, through its first CPU). + #[cfg(CONFIG_CPU_FREQ)] fn acquire(&mut self) -> bool { if !self.requests.is_empty() { return true; @@ -474,6 +481,7 @@ impl CpuCeiling { // SAFETY: the request was added in acquire and the policy reference is still held. unsafe { bindings::freq_qos_remove_request(&mut *entry.request); + #[cfg(CONFIG_CPU_FREQ)] bindings::cpufreq_cpu_put(entry.policy); } } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index bcb9fd32c7d255..3b13ca94462f53 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -250,7 +250,7 @@ impl gpu::Gpu for Backend { } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, - usc_exec_base: u64) -> Result> { + usc_exec_base: u64, _usage: Arc) -> Result> { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } diff --git a/tools/aurora-ci/kunit-expected.json b/tools/aurora-ci/kunit-expected.json index 9c7520e8be6c48..01845ae221d7bd 100644 --- a/tools/aurora-ci/kunit-expected.json +++ b/tools/aurora-ci/kunit-expected.json @@ -6,6 +6,10 @@ "reports these cases as passed, so they are tracked here; a new or missing one is a mismatch." ], "results": { + "asahi_fwid.j713_accepts_both_loaded_images": { + "status": "SKIP", + "reason": "By design: the case carries #[cfg(CONFIG_DRM_ASAHI_G16)], and integration/aurora does not build the G16 backend, so #[kunit_tests] reports it as skipped (drivers/gpu/drm/asahi/fwid.rs)." + }, "apple-dcp-fabric.fabric_scenario_test.S35_dual_retry_off": { "status": "SKIP", "reason": "Unconditional: scenario 35 of the fabric scenario table only calls kunit_skip(\"TB match-data retry policy is tested by W3\") (drivers/gpu/drm/apple/dcp-fabric-test.c). No other code in the tree is named W3; the retry policy is not exercised by this case." From 90b9e228300749ddb908a316b217e12402d45889 Mon Sep 17 00:00:00 2001 From: Jahnavi MN Date: Thu, 16 Jul 2026 08:37:43 +0000 Subject: [PATCH 169/171] rust: task: add Task::tgid() Returns the TGID (Thread Group ID / Process ID) of the given task. This is the rust/kernel/task.rs part of upstream commit 2847d9ab088b ("rust_binder: Add dynamic debug logging mask"), backported to 7.2.11 on its own for the asahi driver. Signed-off-by: Jahnavi MN Link: https://patch.msgid.link/20260716-rust_binder_debug_mask-v4-1-3d7436c2d2f2@google.com Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 2847d9ab088bb559f4f03070f6ad103070362b25) Signed-off-by: Eduard Moldovan Co-Authored-By: Claude Opus 5.5 --- rust/kernel/task.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 38273f4eedb513..1b290c61714db9 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -210,6 +210,13 @@ impl Task { unsafe { *ptr::addr_of!((*self.as_ptr()).pid) } } + /// Returns the TGID (Thread Group ID / Process ID) of the given task. + pub fn tgid(&self) -> Pid { + // SAFETY: The tgid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).tgid) } + } + /// Returns the UID of the given task. #[inline] pub fn uid(&self) -> Kuid { From 6a2b96d76b823c866bc303b4a8831184c933584e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 21 Oct 2026 16:42:05 +1100 Subject: [PATCH 170/171] drm/asahi: bound repeated timeout recovery and G17 mailbox stalls Prevent compute replay from waiting indefinitely for event-serviced input fences while holding the device mutex. Bound initial dependency waits and recovery preparation joins. Queue workload-triggered stop from scheduler callbacks; halt firmware producers before a bounded stop join and retain all graph/lease ownership when a borrower has not returned. Stop on a per-device burst of eight watchdog expirations or eight restart handshakes within thirty seconds. New VMs, generations and replay cannot renew that budget or undo terminal loss. Charge render borrowed-graph and queued-retirement deferrals against the same finite eight-poll bound as compute attribution; a deferred timeout cannot evade eventual device loss. G17 RX now uses an oneshot threaded interrupt with a 64-message pass and an off-lock sleep on exhaustion. G17 non-atomic TX without an IRQ sleeps instead of polling for 500 ms with interrupts disabled; atomic sends have a 1 ms cap. Share one absolute TX deadline across contention/retries and preserve provider/removal/PM/cookie gates. Add lockdep KUnit for 64 consecutive timeouts during pending recovery, late borrower return and latched failure; inject permanently full/empty register models through production mailbox poll/send functions. Document the audited bounds and the unverified hardware attribution. Assisted-by: LLM Signed-off-by: Ryan Murray [Eduard Moldovan: the drm/asahi part is already in integration/aurora through DjDeveloperr's G17 sync (4b7aa809ff54) and Erik Bourget's tree; this carries the mailbox part, its KUnit header and the safety notes] (cherry picked from commit 9827b6bb277af5912490229f6fe56d3c575dafe3 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- .../gpu/asahi-g17-recovery-safety.rst | 72 +++++++++++++++++++ drivers/soc/apple/mailbox-g17-test.h | 66 +++++++++++++++++ drivers/soc/apple/mailbox.c | 71 +++++++++++++++--- 3 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 Documentation/gpu/asahi-g17-recovery-safety.rst create mode 100644 drivers/soc/apple/mailbox-g17-test.h diff --git a/Documentation/gpu/asahi-g17-recovery-safety.rst b/Documentation/gpu/asahi-g17-recovery-safety.rst new file mode 100644 index 00000000000000..f56d08ac2e570f --- /dev/null +++ b/Documentation/gpu/asahi-g17-recovery-safety.rst @@ -0,0 +1,72 @@ +G17 repeated-timeout containment +=============================== + +The 73f audit found workload-reachable host-liveness hazards. The journal +alone cannot attribute the observed machine reset to one of them; no hardware +reproduction is claimed here. + +* Compute replay called Packet::check_dependencies with the device mutex + held. That routine used MAX_SCHEDULE_TIMEOUT. If an input needed completion + service taking the same mutex, the event worker could deadlock. Replay now + checks statuses without sleeping and fails an unexpectedly unmet dependency. + Initial publication waits off-lock in 10 ms cancellation/status-check slices, + with one absolute 2 s budget (10 s for a selected long submission). +* G17 mailbox receive drained an indefinitely replenished FIFO under rx_lock + with IRQs disabled. G17 now uses a oneshot threaded receive IRQ, consumes at + most 64 messages per pass, then sleeps 1-2 ms off-lock with the IRQ masked + when its budget is exhausted. The level condition retriggers without + discarding messages. Explicit receive polling has the same finite budget. +* G17 mailbox transmit without a send-empty IRQ could busy-poll for 500 ms + under tx_lock with IRQs disabled. Non-atomic G17 callers now sleep off-lock + between checks; an atomic G17 send polls for at most 1 ms. All sleeping + retries share a single 500 ms deadline, including contention between senders. + Removal, cookie, provider lifecycle and PM ownership gates are retained. +* Scheduler failure callbacks no longer synchronously join preparation leases + and RTKit receive work. They mark the runtime lost and queue the event worker. + Recovery joins preparation for at most 2 s. Runtime stop halts both firmware + producers first and joins preparation for at most 2 s before detachment. + If a builder remains stalled, its stopped firmware, graphs and reservations + remain owned in Shared::state. Timeout is never an ownership witness. That + path does not re-arm a polling loop; a later queued completion/owner return + may finish the stopped teardown. Explicit device shutdown still joins every + borrower before returning or releasing provider resources. +* Borrowed render graphs and off-lock queued retirements previously renewed + the watchdog without incrementing a render deferral counter. Both engines + now charge deferrals, and render transaction grace ends after eight polls. + Existing three replay credits remain finite and are never reset here. +* A per-device burst limit stops the runtime on the eighth actual scheduler + timeout in a 30 s window or the eighth restart handshake in a 30 s window. + VM changes, new publications, recovery generations and replay do not reset + those counters; terminal loss latches. The 30 s window contains eight + two-second expirations, including a stalled deferred timeout that cannot + complete its normal per-client settlement. This limits host recovery load even + when each individual handshake succeeds. Legitimate long jobs only consume + this budget after their independent bounded execution allowance expires. + +Lock order remains stop -> state and state -> preparation wait/reclaim. All +preparation joins and deferred fence completion/destruction run outside state. +Runtime stop uses try_lock for the stop owner, avoiding a recursive join from +an off-lock fence callback. Receive callbacks only retain/copy ingress and +queue work; no firmware ring is polled while an IRQ spinlock is held without +a finite iteration/time bound on the G17 paths. RX acknowledgement is still +level-sensitive, and TX publication/removal remains serialized by tx_lock. + +The existing control backpressure timer has 1,000 finite retries; its event +probes do not consume or reset that budget. Event-ring ingress is bounded by +EVENT_SLOTS; trace/statistics consume a snapshot producer. Render retirement +loops remove one retained tracker per successful iteration (or mark one as +accounted); compute replay consumes a retained replay per iteration. Shared +capacity deadlines remain absolute and failed fences do not release mappings +or return physical reservations. The new storm limits additionally bound +repeated recoveries and timeouts across those paths. + +KUnit exercises 64 consecutive watchdog timeouts with recovery pending under +real state/preparation mutexes, a retained lease that must time out rather +than be freed, late lease return with failed admission still closed, restart +storms and nonblocking replay status checks. The mailbox tests use normal RAM +as never-empty RX and never-draining TX registers, invoking the production +poll/send functions for 64 RX passes and eight atomic TX timeouts plus a +sleeping TX timeout/removal control. The isolated Asahi scenario runs with +lockdep, PROVE_LOCKING and panic_on_warn. Software tests cannot establish +that physical fabric/firmware or the original reset is fixed; separately +controller-gated Neo qualification remains necessary. diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h new file mode 100644 index 00000000000000..33365864f167a0 --- /dev/null +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* CPU-only transport fault injection: normal RAM stands in for the registers. + * The receive FIFO never empties and the send FIFO never gains space. + */ +#include + +static void apple_g17_test_rx(struct apple_mbox *mbox, + struct apple_mbox_msg msg, void *cookie) +{ + unsigned int *count = cookie; + + (*count)++; +} + +static void apple_g17_rx_flood_bounded_test(struct kunit *test) +{ + const struct apple_mbox_hw hw = { + .i2a_control = 0, .i2a_recv0 = 8, .i2a_recv1 = 16, + .control_empty = BIT(17), + }; + struct apple_mbox mbox = { .hw = &hw, .rx = apple_g17_test_rx }; + unsigned int count = 0; + void *regs = kunit_kzalloc(test, 32, GFP_KERNEL); + int n; + + KUNIT_ASSERT_NOT_NULL(test, regs); + mbox.regs = (void __iomem *)regs; + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + mbox.cookie = &count; + spin_lock_init(&mbox.rx_lock); + for (n = 0; n < 64; n++) + KUNIT_ASSERT_EQ(test, apple_mbox_poll(&mbox), APPLE_G17_MBOX_RX_BUDGET); + KUNIT_EXPECT_EQ(test, count, 64U * APPLE_G17_MBOX_RX_BUDGET); +} + +static void apple_g17_tx_stall_bounded_test(struct kunit *test) +{ + const struct apple_mbox_hw hw = { .control_full = BIT(16) }; + struct apple_mbox mbox = { .hw = &hw, .irq_send_empty = -1 }; + struct apple_mbox_msg msg = { 0 }; + void *regs = kunit_kzalloc(test, 32, GFP_KERNEL); + int n; + + KUNIT_ASSERT_NOT_NULL(test, regs); + mbox.regs = (void __iomem *)regs; + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + writel(BIT(16), mbox.regs); + spin_lock_init(&mbox.tx_lock); + for (n = 0; n < 8; n++) + KUNIT_ASSERT_EQ(test, apple_mbox_send(&mbox, msg, true), -ETIMEDOUT); + /* The G17 no-IRQ, non-atomic branch must sleep off-lock and expire. */ + KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ETIMEDOUT); + mbox.removing = true; + KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ENODEV); +} + +static struct kunit_case apple_g17_stall_cases[] = { + KUNIT_CASE(apple_g17_rx_flood_bounded_test), + KUNIT_CASE(apple_g17_tx_stall_bounded_test), + { } +}; +static struct kunit_suite apple_g17_stall_suite = { + .name = "apple_mbox_g17_stall", + .test_cases = apple_g17_stall_cases, +}; +kunit_test_suite(apple_g17_stall_suite); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5be648a40fe03c..b23b556a6ae77d 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -112,6 +113,8 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_MBOX_MSG1_MSG GENMASK(31, 0) #define APPLE_MBOX_TX_TIMEOUT 500 +#define APPLE_G17_MBOX_RX_BUDGET 64 +#define APPLE_G17_MBOX_ATOMIC_TX_US 1000 struct apple_mbox_hw { unsigned int reg_offset; @@ -266,10 +269,17 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_stop_cpu); +static bool apple_mbox_is_g17(const struct apple_mbox *mbox) +{ + return mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX || + mbox->ascwrap_v6.role == APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; +} + int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { - unsigned long flags; + unsigned long flags, remaining, now; + unsigned long deadline = jiffies + msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT); int ret; u32 mbox_ctrl; long t; @@ -282,10 +292,32 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); while (mbox_ctrl & mbox->hw->control_full) { + now = jiffies; + if (time_after_eq(now, deadline)) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ETIMEDOUT; + } + remaining = deadline - now; + if (apple_mbox_is_g17(mbox) && !atomic && mbox->irq_send_empty < 0) { + /* The GPU has no send-empty IRQ. Do not busy wait with IRQs + * off while dead firmware keeps its FIFO full. Recheck both + * the removal fence and the one absolute deadline on retry. + */ + spin_unlock_irqrestore(&mbox->tx_lock, flags); + usleep_range(1000, 2000); + spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + continue; + } if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, !(mbox_ctrl & mbox->hw->control_full), 100, + apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_ATOMIC_TX_US : APPLE_MBOX_TX_TIMEOUT * 1000); if (ret) { @@ -316,8 +348,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, spin_unlock_irqrestore(&mbox->tx_lock, flags); t = wait_for_completion_interruptible_timeout( - &mbox->tx_empty, - msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT)); + &mbox->tx_empty, remaining); if (t <= 0) { /* A late IRQ may already have masked this under tx_lock. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -374,13 +405,14 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) struct apple_mbox_msg msg; int ret = 0; u32 mbox_ctrl; + u32 budget = apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_RX_BUDGET : U32_MAX; /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - while (!(mbox_ctrl & mbox->hw->control_empty)) { + while (!(mbox_ctrl & mbox->hw->control_empty) && budget--) { msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); msg.msg1 = FIELD_GET( APPLE_MBOX_MSG1_MSG, @@ -410,11 +442,20 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; + unsigned long flags; + int consumed = 0; - spin_lock(&mbox->rx_lock); + spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) - apple_mbox_poll_locked(mbox); - spin_unlock(&mbox->rx_lock); + consumed = apple_mbox_poll_locked(mbox); + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep + * so a firmware flood cannot monopolize a CPU at IRQ-thread priority. + * The level condition retriggers after return; no message is discarded. + */ + if (apple_mbox_is_g17(mbox) && consumed == APPLE_G17_MBOX_RX_BUDGET) + usleep_range(1000, 2000); return IRQ_HANDLED; } @@ -740,9 +781,15 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!irqname) return -ENOMEM; - ret = devm_request_irq(dev, mbox->irq_recv_not_empty, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (apple_mbox_is_g17(mbox)) + ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, + apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, + irqname, mbox); + else + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); if (ret) return ret; @@ -910,3 +957,7 @@ module_platform_driver(apple_mbox_driver); MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple Mailbox driver"); + +#if IS_ENABLED(CONFIG_KUNIT) +#include "mailbox-g17-test.h" +#endif From 76767e347ea925e8ac23eb44f288e847c26e1113 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 14:23:35 -0400 Subject: [PATCH 171/171] soc: apple: mailbox: let a consumer set the receive thread's minimum utilization The G17 mailboxes receive in a threaded IRQ handler, which runs as an RT thread. RT threads run with the RT default minimum utilization, 1024 unless the system lowers sched_util_clamp_min_rt_default, so schedutil raises the thread's CPU cluster to its top frequency for every message. For the GPU completion mailbox that means one top-frequency burst per GPU completion, also while the GPU itself is the bottleneck and the CPU has nothing to do quickly. Add apple_mbox_set_rx_util_min(), which records the minimum utilization a consumer wants for the receive thread, or -1 for the scheduler's default, and have the threaded handler apply it to itself (sched_setattr_nocheck() on current, keeping its policy and priority) the next time it runs. The request is a single store, so it may be made from any context. Only mailboxes with a threaded receive handler support it; until a consumer asks, the thread keeps the default. Wrap it for RTKit consumers as apple_rtkit_set_rx_util_min() and RtKit::set_rx_util_min(). RtKit::rx_util_min_handle() returns a handle that makes the same request without borrowing the RtKit, for consumers that update it from their submission and completion paths. A KUnit case covers the request checks, and applying a request to a FIFO thread without changing its policy or priority. (cherry picked from commit 564d91c812880812cead8b98dad4eb34ba914029 in aurora-silicon/linux) Signed-off-by: Eduard Moldovan --- drivers/soc/apple/mailbox-g17-test.h | 40 ++++++++++++++++++++++ drivers/soc/apple/mailbox.c | 41 ++++++++++++++++++++++ drivers/soc/apple/rtkit.c | 6 ++++ include/linux/soc/apple/mailbox.h | 14 ++++++++ include/linux/soc/apple/rtkit.h | 6 ++++ rust/kernel/soc/apple/rtkit.rs | 51 ++++++++++++++++++++++++++++ 6 files changed, 158 insertions(+) diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h index 33365864f167a0..10690d1d27bf78 100644 --- a/drivers/soc/apple/mailbox-g17-test.h +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -54,9 +54,49 @@ static void apple_g17_tx_stall_bounded_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_mbox_send(&mbox, msg, false), -ENODEV); } +/* The receive thread's minimum utilization: request validation, and applying + * it to an RT thread (the test thread stands in) without losing its policy. + */ +static void apple_g17_rx_util_min_test(struct kunit *test) +{ + struct apple_mbox mbox = { + .rx_util_min = -1, .rx_util_min_applied = -1, + }; + + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP); + mbox.ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + if (!IS_ENABLED(CONFIG_UCLAMP_TASK)) { + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 0), -EOPNOTSUPP); + kunit_skip(test, "needs CONFIG_UCLAMP_TASK"); + } + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -2), -EINVAL); + KUNIT_EXPECT_EQ(test, apple_mbox_set_rx_util_min(&mbox, SCHED_CAPACITY_SCALE + 1), -EINVAL); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min, -1); +#ifdef CONFIG_UCLAMP_TASK + sched_set_fifo(current); + KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, 256), 0); + apple_mbox_apply_rx_util_min(&mbox); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, 256); + KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO); + KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2); + KUNIT_EXPECT_EQ(test, (unsigned int)current->uclamp_req[UCLAMP_MIN].value, 256U); + KUNIT_EXPECT_TRUE(test, current->uclamp_req[UCLAMP_MIN].user_defined); + + /* -1 hands the clamp back to the scheduler's RT default. */ + KUNIT_ASSERT_EQ(test, apple_mbox_set_rx_util_min(&mbox, -1), 0); + apple_mbox_apply_rx_util_min(&mbox); + KUNIT_EXPECT_EQ(test, mbox.rx_util_min_applied, -1); + KUNIT_EXPECT_EQ(test, current->policy, SCHED_FIFO); + KUNIT_EXPECT_EQ(test, current->rt_priority, MAX_RT_PRIO / 2); + KUNIT_EXPECT_FALSE(test, current->uclamp_req[UCLAMP_MIN].user_defined); + sched_set_normal(current, 0); +#endif +} + static struct kunit_case apple_g17_stall_cases[] = { KUNIT_CASE(apple_g17_rx_flood_bounded_test), KUNIT_CASE(apple_g17_tx_stall_bounded_test), + KUNIT_CASE(apple_g17_rx_util_min_test), { } }; static struct kunit_suite apple_g17_stall_suite = { diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index b23b556a6ae77d..20659792173b0d 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -29,9 +29,11 @@ #include #include #include +#include #include #include #include +#include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) @@ -439,12 +441,49 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) return ret; } +/* + * Threaded receive handler only: apply the consumer's minimum utilization to + * this thread. As an RT thread it otherwise runs with the RT default, which + * makes schedutil raise its CPU cluster to the top frequency on every message. + */ +static void apple_mbox_apply_rx_util_min(struct apple_mbox *mbox) +{ + int util_min = READ_ONCE(mbox->rx_util_min); + struct sched_attr attr = { + .sched_policy = current->policy, + .sched_priority = current->rt_priority, + .sched_flags = SCHED_FLAG_KEEP_ALL | SCHED_FLAG_UTIL_CLAMP_MIN, + }; + + if (util_min == mbox->rx_util_min_applied) + return; + /* -1 resets the clamp to the default, which follows the RT sysctl. */ + attr.sched_util_min = util_min; + if (!sched_setattr_nocheck(current, &attr)) + mbox->rx_util_min_applied = util_min; +} + +int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) +{ + if (!IS_ENABLED(CONFIG_UCLAMP_TASK) || !apple_mbox_is_g17(mbox)) + return -EOPNOTSUPP; + if (util_min < -1 || util_min > SCHED_CAPACITY_SCALE) + return -EINVAL; + WRITE_ONCE(mbox->rx_util_min, util_min); + return 0; +} +EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); + static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; unsigned long flags; int consumed = 0; + /* Only the G17 receive handler is threaded (see apple_mbox_probe()). */ + if (apple_mbox_is_g17(mbox)) + apple_mbox_apply_rx_util_min(mbox); + spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); @@ -772,6 +811,8 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->irq_send_empty < 0 && mbox->irq_send_empty != -ENXIO) return mbox->irq_send_empty; + mbox->rx_util_min = -1; + mbox->rx_util_min_applied = -1; spin_lock_init(&mbox->rx_lock); spin_lock_init(&mbox->tx_lock); mutex_init(&mbox->lifecycle_lock); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 11334b3f68faa5..4d6fd5393185a8 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -994,6 +994,12 @@ void apple_rtkit_resume_rx(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); +int apple_rtkit_set_rx_util_min(struct apple_rtkit *rtk, int util_min) +{ + return apple_mbox_set_rx_util_min(rtk->mbox, util_min); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_rx_util_min); + void apple_rtkit_flush_rx(struct apple_rtkit *rtk) { flush_workqueue(rtk->wq); diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index fcb17eaf2d5a33..b5b723f0a215ee 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -77,6 +77,14 @@ struct apple_mbox { void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); void *cookie; + /* + * Minimum utilization (uclamp_min) a consumer asked the threaded receive + * handler to run with, and the value it last applied to itself; -1 is the + * scheduler's default. See apple_mbox_set_rx_util_min(). + */ + int rx_util_min; + int rx_util_min_applied; + /* * Protected by tx_lock: set while irq_send_empty is unmasked, so that * the IRQ handler, a failed send and provider removal mask it exactly @@ -97,6 +105,12 @@ int apple_mbox_poll(struct apple_mbox *mbox); */ void apple_mbox_hold_rx(struct apple_mbox *mbox); void apple_mbox_release_rx(struct apple_mbox *mbox); +/* + * Set the minimum utilization (0..1024) the threaded receive handler runs with, + * or -1 to restore the scheduler's default; the handler applies it to itself + * when it next runs. Only mailboxes with a threaded receive handler support it. + */ +int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 1ba27a92e2b144..2e6f53815f0701 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -246,6 +246,12 @@ int apple_rtkit_poll(struct apple_rtkit *rtk); void apple_rtkit_hold_rx(struct apple_rtkit *rtk); void apple_rtkit_resume_rx(struct apple_rtkit *rtk); +/* + * Set the minimum utilization (0..1024, or -1 for the scheduler's default) of + * the coprocessor's receive thread; see apple_mbox_set_rx_util_min(). + */ +int apple_rtkit_set_rx_util_min(struct apple_rtkit *rtk, int util_min); + /* * Wait until the messages already received have been handled, including the * recv_message, crashed and shmem_setup callbacks the worker thread ran for diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 87ec3bd73658bb..c3fb4c2f6bc53a 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -334,6 +334,57 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } + + /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or + /// restores the scheduler's default with `None`. + /// + /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a + /// mailbox without a threaded receive handler. + pub fn set_rx_util_min(&self, util_min: Option) -> Result { + // INVARIANT: `rtk` is valid per the type invariant. + RxUtilMin { rtk: self.rtk }.set(util_min) + } + + /// Returns a handle that sets the receive thread's minimum utilization (see + /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this [`RtKit`] is dropped. + pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { + // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. + RxUtilMin { rtk: self.rtk } + } +} + +/// Sets the minimum utilization of an RTKit coprocessor's receive thread. +/// +/// The request is a single store that the receive thread applies to itself when it next runs, +/// so it may be made from any context. +/// +/// # Invariants +/// +/// `rtk` is valid while the handle is used. +#[derive(Clone, Copy)] +pub struct RxUtilMin { + rtk: *mut bindings::apple_rtkit, +} + +// SAFETY: The request is one store into the mailbox, valid from any thread. +unsafe impl Send for RxUtilMin {} +// SAFETY: As above; the handle has no state of its own. +unsafe impl Sync for RxUtilMin {} + +impl RxUtilMin { + /// See [`RtKit::set_rx_util_min`]. + pub fn set(&self, util_min: Option) -> Result { + let util_min = match util_min { + Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, + None => -1, + }; + // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. + to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) + } } // SAFETY: `RtKit` operations require a mutable reference